Line data Source code
1 : //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements the AArch64TargetLowering class.
11 : //
12 : //===----------------------------------------------------------------------===//
13 :
14 : #include "AArch64ISelLowering.h"
15 : #include "AArch64CallingConvention.h"
16 : #include "AArch64MachineFunctionInfo.h"
17 : #include "AArch64PerfectShuffle.h"
18 : #include "AArch64RegisterInfo.h"
19 : #include "AArch64Subtarget.h"
20 : #include "MCTargetDesc/AArch64AddressingModes.h"
21 : #include "Utils/AArch64BaseInfo.h"
22 : #include "llvm/ADT/APFloat.h"
23 : #include "llvm/ADT/APInt.h"
24 : #include "llvm/ADT/ArrayRef.h"
25 : #include "llvm/ADT/STLExtras.h"
26 : #include "llvm/ADT/SmallVector.h"
27 : #include "llvm/ADT/Statistic.h"
28 : #include "llvm/ADT/StringRef.h"
29 : #include "llvm/ADT/StringSwitch.h"
30 : #include "llvm/ADT/Triple.h"
31 : #include "llvm/ADT/Twine.h"
32 : #include "llvm/Analysis/VectorUtils.h"
33 : #include "llvm/CodeGen/CallingConvLower.h"
34 : #include "llvm/CodeGen/MachineBasicBlock.h"
35 : #include "llvm/CodeGen/MachineFrameInfo.h"
36 : #include "llvm/CodeGen/MachineFunction.h"
37 : #include "llvm/CodeGen/MachineInstr.h"
38 : #include "llvm/CodeGen/MachineInstrBuilder.h"
39 : #include "llvm/CodeGen/MachineMemOperand.h"
40 : #include "llvm/CodeGen/MachineRegisterInfo.h"
41 : #include "llvm/CodeGen/RuntimeLibcalls.h"
42 : #include "llvm/CodeGen/SelectionDAG.h"
43 : #include "llvm/CodeGen/SelectionDAGNodes.h"
44 : #include "llvm/CodeGen/TargetCallingConv.h"
45 : #include "llvm/CodeGen/TargetInstrInfo.h"
46 : #include "llvm/CodeGen/ValueTypes.h"
47 : #include "llvm/IR/Attributes.h"
48 : #include "llvm/IR/Constants.h"
49 : #include "llvm/IR/DataLayout.h"
50 : #include "llvm/IR/DebugLoc.h"
51 : #include "llvm/IR/DerivedTypes.h"
52 : #include "llvm/IR/Function.h"
53 : #include "llvm/IR/GetElementPtrTypeIterator.h"
54 : #include "llvm/IR/GlobalValue.h"
55 : #include "llvm/IR/IRBuilder.h"
56 : #include "llvm/IR/Instruction.h"
57 : #include "llvm/IR/Instructions.h"
58 : #include "llvm/IR/Intrinsics.h"
59 : #include "llvm/IR/Module.h"
60 : #include "llvm/IR/OperandTraits.h"
61 : #include "llvm/IR/Type.h"
62 : #include "llvm/IR/Use.h"
63 : #include "llvm/IR/Value.h"
64 : #include "llvm/MC/MCRegisterInfo.h"
65 : #include "llvm/Support/Casting.h"
66 : #include "llvm/Support/CodeGen.h"
67 : #include "llvm/Support/CommandLine.h"
68 : #include "llvm/Support/Compiler.h"
69 : #include "llvm/Support/Debug.h"
70 : #include "llvm/Support/ErrorHandling.h"
71 : #include "llvm/Support/KnownBits.h"
72 : #include "llvm/Support/MachineValueType.h"
73 : #include "llvm/Support/MathExtras.h"
74 : #include "llvm/Support/raw_ostream.h"
75 : #include "llvm/Target/TargetMachine.h"
76 : #include "llvm/Target/TargetOptions.h"
77 : #include <algorithm>
78 : #include <bitset>
79 : #include <cassert>
80 : #include <cctype>
81 : #include <cstdint>
82 : #include <cstdlib>
83 : #include <iterator>
84 : #include <limits>
85 : #include <tuple>
86 : #include <utility>
87 : #include <vector>
88 :
89 : using namespace llvm;
90 :
91 : #define DEBUG_TYPE "aarch64-lower"
92 :
93 : STATISTIC(NumTailCalls, "Number of tail calls");
94 : STATISTIC(NumShiftInserts, "Number of vector shift inserts");
95 : STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
96 :
97 : static cl::opt<bool>
98 : EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden,
99 : cl::desc("Allow AArch64 SLI/SRI formation"),
100 : cl::init(false));
101 :
102 : // FIXME: The necessary dtprel relocations don't seem to be supported
103 : // well in the GNU bfd and gold linkers at the moment. Therefore, by
104 : // default, for now, fall back to GeneralDynamic code generation.
105 : cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
106 : "aarch64-elf-ldtls-generation", cl::Hidden,
107 : cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
108 : cl::init(false));
109 :
110 : static cl::opt<bool>
111 : EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
112 : cl::desc("Enable AArch64 logical imm instruction "
113 : "optimization"),
114 : cl::init(true));
115 :
116 : /// Value type used for condition codes.
117 : static const MVT MVT_CC = MVT::i32;
118 :
119 1570 : AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
120 1570 : const AArch64Subtarget &STI)
121 1570 : : TargetLowering(TM), Subtarget(&STI) {
122 : // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
123 : // we have to make something up. Arbitrarily, choose ZeroOrOne.
124 : setBooleanContents(ZeroOrOneBooleanContent);
125 : // When comparing vectors the result sets the different elements in the
126 : // vector to all-one or all-zero.
127 : setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
128 :
129 : // Set up the register classes.
130 : addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
131 : addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
132 :
133 1570 : if (Subtarget->hasFPARMv8()) {
134 : addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
135 : addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
136 : addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
137 : addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
138 : }
139 :
140 1570 : if (Subtarget->hasNEON()) {
141 : addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
142 : addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
143 : // Someone set us up the NEON.
144 1547 : addDRTypeForNEON(MVT::v2f32);
145 1547 : addDRTypeForNEON(MVT::v8i8);
146 1547 : addDRTypeForNEON(MVT::v4i16);
147 1547 : addDRTypeForNEON(MVT::v2i32);
148 1547 : addDRTypeForNEON(MVT::v1i64);
149 1547 : addDRTypeForNEON(MVT::v1f64);
150 1547 : addDRTypeForNEON(MVT::v4f16);
151 :
152 1547 : addQRTypeForNEON(MVT::v4f32);
153 1547 : addQRTypeForNEON(MVT::v2f64);
154 1547 : addQRTypeForNEON(MVT::v16i8);
155 1547 : addQRTypeForNEON(MVT::v8i16);
156 1547 : addQRTypeForNEON(MVT::v4i32);
157 1547 : addQRTypeForNEON(MVT::v2i64);
158 1547 : addQRTypeForNEON(MVT::v8f16);
159 : }
160 :
161 : // Compute derived properties from the register classes
162 3140 : computeRegisterProperties(Subtarget->getRegisterInfo());
163 :
164 : // Provide all sorts of operation actions
165 : setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
166 : setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
167 : setOperationAction(ISD::SETCC, MVT::i32, Custom);
168 : setOperationAction(ISD::SETCC, MVT::i64, Custom);
169 : setOperationAction(ISD::SETCC, MVT::f16, Custom);
170 : setOperationAction(ISD::SETCC, MVT::f32, Custom);
171 : setOperationAction(ISD::SETCC, MVT::f64, Custom);
172 : setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
173 : setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
174 : setOperationAction(ISD::BRCOND, MVT::Other, Expand);
175 : setOperationAction(ISD::BR_CC, MVT::i32, Custom);
176 : setOperationAction(ISD::BR_CC, MVT::i64, Custom);
177 : setOperationAction(ISD::BR_CC, MVT::f16, Custom);
178 : setOperationAction(ISD::BR_CC, MVT::f32, Custom);
179 : setOperationAction(ISD::BR_CC, MVT::f64, Custom);
180 : setOperationAction(ISD::SELECT, MVT::i32, Custom);
181 : setOperationAction(ISD::SELECT, MVT::i64, Custom);
182 : setOperationAction(ISD::SELECT, MVT::f16, Custom);
183 : setOperationAction(ISD::SELECT, MVT::f32, Custom);
184 : setOperationAction(ISD::SELECT, MVT::f64, Custom);
185 : setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
186 : setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
187 : setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
188 : setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
189 : setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
190 : setOperationAction(ISD::BR_JT, MVT::Other, Expand);
191 : setOperationAction(ISD::JumpTable, MVT::i64, Custom);
192 :
193 : setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
194 : setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
195 : setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
196 :
197 : setOperationAction(ISD::FREM, MVT::f32, Expand);
198 : setOperationAction(ISD::FREM, MVT::f64, Expand);
199 : setOperationAction(ISD::FREM, MVT::f80, Expand);
200 :
201 : setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
202 :
203 : // Custom lowering hooks are needed for XOR
204 : // to fold it into CSINC/CSINV.
205 : setOperationAction(ISD::XOR, MVT::i32, Custom);
206 : setOperationAction(ISD::XOR, MVT::i64, Custom);
207 :
208 : // Virtually no operation on f128 is legal, but LLVM can't expand them when
209 : // there's a valid register class, so we need custom operations in most cases.
210 : setOperationAction(ISD::FABS, MVT::f128, Expand);
211 : setOperationAction(ISD::FADD, MVT::f128, Custom);
212 : setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
213 : setOperationAction(ISD::FCOS, MVT::f128, Expand);
214 : setOperationAction(ISD::FDIV, MVT::f128, Custom);
215 : setOperationAction(ISD::FMA, MVT::f128, Expand);
216 : setOperationAction(ISD::FMUL, MVT::f128, Custom);
217 : setOperationAction(ISD::FNEG, MVT::f128, Expand);
218 : setOperationAction(ISD::FPOW, MVT::f128, Expand);
219 : setOperationAction(ISD::FREM, MVT::f128, Expand);
220 : setOperationAction(ISD::FRINT, MVT::f128, Expand);
221 : setOperationAction(ISD::FSIN, MVT::f128, Expand);
222 : setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
223 : setOperationAction(ISD::FSQRT, MVT::f128, Expand);
224 : setOperationAction(ISD::FSUB, MVT::f128, Custom);
225 : setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
226 : setOperationAction(ISD::SETCC, MVT::f128, Custom);
227 : setOperationAction(ISD::BR_CC, MVT::f128, Custom);
228 : setOperationAction(ISD::SELECT, MVT::f128, Custom);
229 : setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
230 : setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
231 :
232 : // Lowering for many of the conversions is actually specified by the non-f128
233 : // type. The LowerXXX function will be trivial when f128 isn't involved.
234 : setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
235 : setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
236 : setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
237 : setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
238 : setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
239 : setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
240 : setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
241 : setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
242 : setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
243 : setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
244 : setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
245 : setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
246 : setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
247 : setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
248 :
249 : // Variable arguments.
250 : setOperationAction(ISD::VASTART, MVT::Other, Custom);
251 : setOperationAction(ISD::VAARG, MVT::Other, Custom);
252 : setOperationAction(ISD::VACOPY, MVT::Other, Custom);
253 : setOperationAction(ISD::VAEND, MVT::Other, Expand);
254 :
255 : // Variable-sized objects.
256 : setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
257 : setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
258 :
259 3140 : if (Subtarget->isTargetWindows())
260 : setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
261 : else
262 : setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
263 :
264 : // Constant pool entries
265 : setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
266 :
267 : // BlockAddress
268 : setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
269 :
270 : // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
271 : setOperationAction(ISD::ADDC, MVT::i32, Custom);
272 : setOperationAction(ISD::ADDE, MVT::i32, Custom);
273 : setOperationAction(ISD::SUBC, MVT::i32, Custom);
274 : setOperationAction(ISD::SUBE, MVT::i32, Custom);
275 : setOperationAction(ISD::ADDC, MVT::i64, Custom);
276 : setOperationAction(ISD::ADDE, MVT::i64, Custom);
277 : setOperationAction(ISD::SUBC, MVT::i64, Custom);
278 : setOperationAction(ISD::SUBE, MVT::i64, Custom);
279 :
280 : // AArch64 lacks both left-rotate and popcount instructions.
281 : setOperationAction(ISD::ROTL, MVT::i32, Expand);
282 : setOperationAction(ISD::ROTL, MVT::i64, Expand);
283 150720 : for (MVT VT : MVT::vector_valuetypes()) {
284 : setOperationAction(ISD::ROTL, VT, Expand);
285 : setOperationAction(ISD::ROTR, VT, Expand);
286 : }
287 :
288 : // AArch64 doesn't have {U|S}MUL_LOHI.
289 : setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
290 : setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
291 :
292 : setOperationAction(ISD::CTPOP, MVT::i32, Custom);
293 : setOperationAction(ISD::CTPOP, MVT::i64, Custom);
294 :
295 : setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
296 : setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
297 150720 : for (MVT VT : MVT::vector_valuetypes()) {
298 : setOperationAction(ISD::SDIVREM, VT, Expand);
299 : setOperationAction(ISD::UDIVREM, VT, Expand);
300 : }
301 : setOperationAction(ISD::SREM, MVT::i32, Expand);
302 : setOperationAction(ISD::SREM, MVT::i64, Expand);
303 : setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
304 : setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
305 : setOperationAction(ISD::UREM, MVT::i32, Expand);
306 : setOperationAction(ISD::UREM, MVT::i64, Expand);
307 :
308 : // Custom lower Add/Sub/Mul with overflow.
309 : setOperationAction(ISD::SADDO, MVT::i32, Custom);
310 : setOperationAction(ISD::SADDO, MVT::i64, Custom);
311 : setOperationAction(ISD::UADDO, MVT::i32, Custom);
312 : setOperationAction(ISD::UADDO, MVT::i64, Custom);
313 : setOperationAction(ISD::SSUBO, MVT::i32, Custom);
314 : setOperationAction(ISD::SSUBO, MVT::i64, Custom);
315 : setOperationAction(ISD::USUBO, MVT::i32, Custom);
316 : setOperationAction(ISD::USUBO, MVT::i64, Custom);
317 : setOperationAction(ISD::SMULO, MVT::i32, Custom);
318 : setOperationAction(ISD::SMULO, MVT::i64, Custom);
319 : setOperationAction(ISD::UMULO, MVT::i32, Custom);
320 : setOperationAction(ISD::UMULO, MVT::i64, Custom);
321 :
322 : setOperationAction(ISD::FSIN, MVT::f32, Expand);
323 : setOperationAction(ISD::FSIN, MVT::f64, Expand);
324 : setOperationAction(ISD::FCOS, MVT::f32, Expand);
325 : setOperationAction(ISD::FCOS, MVT::f64, Expand);
326 : setOperationAction(ISD::FPOW, MVT::f32, Expand);
327 : setOperationAction(ISD::FPOW, MVT::f64, Expand);
328 : setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
329 : setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
330 1570 : if (Subtarget->hasFullFP16())
331 : setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
332 : else
333 : setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
334 :
335 : setOperationAction(ISD::FREM, MVT::f16, Promote);
336 : setOperationAction(ISD::FREM, MVT::v4f16, Promote);
337 : setOperationAction(ISD::FREM, MVT::v8f16, Promote);
338 : setOperationAction(ISD::FPOW, MVT::f16, Promote);
339 : setOperationAction(ISD::FPOW, MVT::v4f16, Promote);
340 : setOperationAction(ISD::FPOW, MVT::v8f16, Promote);
341 : setOperationAction(ISD::FPOWI, MVT::f16, Promote);
342 : setOperationAction(ISD::FCOS, MVT::f16, Promote);
343 : setOperationAction(ISD::FCOS, MVT::v4f16, Promote);
344 : setOperationAction(ISD::FCOS, MVT::v8f16, Promote);
345 : setOperationAction(ISD::FSIN, MVT::f16, Promote);
346 : setOperationAction(ISD::FSIN, MVT::v4f16, Promote);
347 : setOperationAction(ISD::FSIN, MVT::v8f16, Promote);
348 : setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
349 : setOperationAction(ISD::FSINCOS, MVT::v4f16, Promote);
350 : setOperationAction(ISD::FSINCOS, MVT::v8f16, Promote);
351 : setOperationAction(ISD::FEXP, MVT::f16, Promote);
352 : setOperationAction(ISD::FEXP, MVT::v4f16, Promote);
353 : setOperationAction(ISD::FEXP, MVT::v8f16, Promote);
354 : setOperationAction(ISD::FEXP2, MVT::f16, Promote);
355 : setOperationAction(ISD::FEXP2, MVT::v4f16, Promote);
356 : setOperationAction(ISD::FEXP2, MVT::v8f16, Promote);
357 : setOperationAction(ISD::FLOG, MVT::f16, Promote);
358 : setOperationAction(ISD::FLOG, MVT::v4f16, Promote);
359 : setOperationAction(ISD::FLOG, MVT::v8f16, Promote);
360 : setOperationAction(ISD::FLOG2, MVT::f16, Promote);
361 : setOperationAction(ISD::FLOG2, MVT::v4f16, Promote);
362 : setOperationAction(ISD::FLOG2, MVT::v8f16, Promote);
363 : setOperationAction(ISD::FLOG10, MVT::f16, Promote);
364 : setOperationAction(ISD::FLOG10, MVT::v4f16, Promote);
365 : setOperationAction(ISD::FLOG10, MVT::v8f16, Promote);
366 :
367 1570 : if (!Subtarget->hasFullFP16()) {
368 : setOperationAction(ISD::SELECT, MVT::f16, Promote);
369 : setOperationAction(ISD::SELECT_CC, MVT::f16, Promote);
370 : setOperationAction(ISD::SETCC, MVT::f16, Promote);
371 : setOperationAction(ISD::BR_CC, MVT::f16, Promote);
372 : setOperationAction(ISD::FADD, MVT::f16, Promote);
373 : setOperationAction(ISD::FSUB, MVT::f16, Promote);
374 : setOperationAction(ISD::FMUL, MVT::f16, Promote);
375 : setOperationAction(ISD::FDIV, MVT::f16, Promote);
376 : setOperationAction(ISD::FMA, MVT::f16, Promote);
377 : setOperationAction(ISD::FNEG, MVT::f16, Promote);
378 : setOperationAction(ISD::FABS, MVT::f16, Promote);
379 : setOperationAction(ISD::FCEIL, MVT::f16, Promote);
380 : setOperationAction(ISD::FSQRT, MVT::f16, Promote);
381 : setOperationAction(ISD::FFLOOR, MVT::f16, Promote);
382 : setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
383 : setOperationAction(ISD::FRINT, MVT::f16, Promote);
384 : setOperationAction(ISD::FROUND, MVT::f16, Promote);
385 : setOperationAction(ISD::FTRUNC, MVT::f16, Promote);
386 : setOperationAction(ISD::FMINNUM, MVT::f16, Promote);
387 : setOperationAction(ISD::FMAXNUM, MVT::f16, Promote);
388 : setOperationAction(ISD::FMINNAN, MVT::f16, Promote);
389 : setOperationAction(ISD::FMAXNAN, MVT::f16, Promote);
390 :
391 : // promote v4f16 to v4f32 when that is known to be safe.
392 : setOperationAction(ISD::FADD, MVT::v4f16, Promote);
393 : setOperationAction(ISD::FSUB, MVT::v4f16, Promote);
394 : setOperationAction(ISD::FMUL, MVT::v4f16, Promote);
395 : setOperationAction(ISD::FDIV, MVT::v4f16, Promote);
396 : setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote);
397 : setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote);
398 : AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32);
399 : AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32);
400 : AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32);
401 : AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32);
402 : AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32);
403 : AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32);
404 :
405 : setOperationAction(ISD::FABS, MVT::v4f16, Expand);
406 : setOperationAction(ISD::FNEG, MVT::v4f16, Expand);
407 : setOperationAction(ISD::FROUND, MVT::v4f16, Expand);
408 : setOperationAction(ISD::FMA, MVT::v4f16, Expand);
409 : setOperationAction(ISD::SETCC, MVT::v4f16, Expand);
410 : setOperationAction(ISD::BR_CC, MVT::v4f16, Expand);
411 : setOperationAction(ISD::SELECT, MVT::v4f16, Expand);
412 : setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand);
413 : setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand);
414 : setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand);
415 : setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand);
416 : setOperationAction(ISD::FCEIL, MVT::v4f16, Expand);
417 : setOperationAction(ISD::FRINT, MVT::v4f16, Expand);
418 : setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand);
419 : setOperationAction(ISD::FSQRT, MVT::v4f16, Expand);
420 :
421 : setOperationAction(ISD::FABS, MVT::v8f16, Expand);
422 : setOperationAction(ISD::FADD, MVT::v8f16, Expand);
423 : setOperationAction(ISD::FCEIL, MVT::v8f16, Expand);
424 : setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand);
425 : setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
426 : setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand);
427 : setOperationAction(ISD::FMA, MVT::v8f16, Expand);
428 : setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
429 : setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand);
430 : setOperationAction(ISD::FNEG, MVT::v8f16, Expand);
431 : setOperationAction(ISD::FROUND, MVT::v8f16, Expand);
432 : setOperationAction(ISD::FRINT, MVT::v8f16, Expand);
433 : setOperationAction(ISD::FSQRT, MVT::v8f16, Expand);
434 : setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
435 : setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand);
436 : setOperationAction(ISD::SETCC, MVT::v8f16, Expand);
437 : setOperationAction(ISD::BR_CC, MVT::v8f16, Expand);
438 : setOperationAction(ISD::SELECT, MVT::v8f16, Expand);
439 : setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand);
440 : setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand);
441 : }
442 :
443 : // AArch64 has implementations of a lot of rounding-like FP operations.
444 4710 : for (MVT Ty : {MVT::f32, MVT::f64}) {
445 : setOperationAction(ISD::FFLOOR, Ty, Legal);
446 : setOperationAction(ISD::FNEARBYINT, Ty, Legal);
447 : setOperationAction(ISD::FCEIL, Ty, Legal);
448 : setOperationAction(ISD::FRINT, Ty, Legal);
449 : setOperationAction(ISD::FTRUNC, Ty, Legal);
450 : setOperationAction(ISD::FROUND, Ty, Legal);
451 : setOperationAction(ISD::FMINNUM, Ty, Legal);
452 : setOperationAction(ISD::FMAXNUM, Ty, Legal);
453 : setOperationAction(ISD::FMINNAN, Ty, Legal);
454 : setOperationAction(ISD::FMAXNAN, Ty, Legal);
455 : }
456 :
457 1570 : if (Subtarget->hasFullFP16()) {
458 : setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
459 : setOperationAction(ISD::FFLOOR, MVT::f16, Legal);
460 : setOperationAction(ISD::FCEIL, MVT::f16, Legal);
461 : setOperationAction(ISD::FRINT, MVT::f16, Legal);
462 : setOperationAction(ISD::FTRUNC, MVT::f16, Legal);
463 : setOperationAction(ISD::FROUND, MVT::f16, Legal);
464 : setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
465 : setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
466 : setOperationAction(ISD::FMINNAN, MVT::f16, Legal);
467 : setOperationAction(ISD::FMAXNAN, MVT::f16, Legal);
468 : }
469 :
470 : setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
471 :
472 : setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
473 :
474 : setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
475 : setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
476 : setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
477 : setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
478 : setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
479 :
480 : // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
481 : // This requires the Performance Monitors extension.
482 1570 : if (Subtarget->hasPerfMon())
483 : setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
484 :
485 1570 : if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
486 : getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
487 : // Issue __sincos_stret if available.
488 : setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
489 : setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
490 : } else {
491 : setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
492 : setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
493 : }
494 :
495 : // Make floating-point constants legal for the large code model, so they don't
496 : // become loads from the constant pool.
497 3140 : if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
498 : setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
499 : setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
500 : }
501 :
502 : // AArch64 does not have floating-point extending loads, i1 sign-extending
503 : // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
504 10990 : for (MVT VT : MVT::fp_valuetypes()) {
505 : setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
506 : setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
507 : setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
508 : setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
509 : }
510 10990 : for (MVT VT : MVT::integer_valuetypes())
511 : setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
512 :
513 : setTruncStoreAction(MVT::f32, MVT::f16, Expand);
514 : setTruncStoreAction(MVT::f64, MVT::f32, Expand);
515 : setTruncStoreAction(MVT::f64, MVT::f16, Expand);
516 : setTruncStoreAction(MVT::f128, MVT::f80, Expand);
517 : setTruncStoreAction(MVT::f128, MVT::f64, Expand);
518 : setTruncStoreAction(MVT::f128, MVT::f32, Expand);
519 : setTruncStoreAction(MVT::f128, MVT::f16, Expand);
520 :
521 : setOperationAction(ISD::BITCAST, MVT::i16, Custom);
522 : setOperationAction(ISD::BITCAST, MVT::f16, Custom);
523 :
524 : // Indexed loads and stores are supported.
525 6280 : for (unsigned im = (unsigned)ISD::PRE_INC;
526 7850 : im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
527 : setIndexedLoadAction(im, MVT::i8, Legal);
528 : setIndexedLoadAction(im, MVT::i16, Legal);
529 : setIndexedLoadAction(im, MVT::i32, Legal);
530 : setIndexedLoadAction(im, MVT::i64, Legal);
531 : setIndexedLoadAction(im, MVT::f64, Legal);
532 : setIndexedLoadAction(im, MVT::f32, Legal);
533 : setIndexedLoadAction(im, MVT::f16, Legal);
534 : setIndexedStoreAction(im, MVT::i8, Legal);
535 : setIndexedStoreAction(im, MVT::i16, Legal);
536 : setIndexedStoreAction(im, MVT::i32, Legal);
537 : setIndexedStoreAction(im, MVT::i64, Legal);
538 : setIndexedStoreAction(im, MVT::f64, Legal);
539 : setIndexedStoreAction(im, MVT::f32, Legal);
540 : setIndexedStoreAction(im, MVT::f16, Legal);
541 : }
542 :
543 : // Trap.
544 : setOperationAction(ISD::TRAP, MVT::Other, Legal);
545 :
546 : // We combine OR nodes for bitfield operations.
547 : setTargetDAGCombine(ISD::OR);
548 :
549 : // Vector add and sub nodes may conceal a high-half opportunity.
550 : // Also, try to fold ADD into CSINC/CSINV..
551 : setTargetDAGCombine(ISD::ADD);
552 : setTargetDAGCombine(ISD::SUB);
553 : setTargetDAGCombine(ISD::SRL);
554 : setTargetDAGCombine(ISD::XOR);
555 : setTargetDAGCombine(ISD::SINT_TO_FP);
556 : setTargetDAGCombine(ISD::UINT_TO_FP);
557 :
558 : setTargetDAGCombine(ISD::FP_TO_SINT);
559 : setTargetDAGCombine(ISD::FP_TO_UINT);
560 : setTargetDAGCombine(ISD::FDIV);
561 :
562 : setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
563 :
564 : setTargetDAGCombine(ISD::ANY_EXTEND);
565 : setTargetDAGCombine(ISD::ZERO_EXTEND);
566 : setTargetDAGCombine(ISD::SIGN_EXTEND);
567 : setTargetDAGCombine(ISD::BITCAST);
568 : setTargetDAGCombine(ISD::CONCAT_VECTORS);
569 : setTargetDAGCombine(ISD::STORE);
570 1570 : if (Subtarget->supportsAddressTopByteIgnored())
571 : setTargetDAGCombine(ISD::LOAD);
572 :
573 : setTargetDAGCombine(ISD::MUL);
574 :
575 : setTargetDAGCombine(ISD::SELECT);
576 : setTargetDAGCombine(ISD::VSELECT);
577 :
578 : setTargetDAGCombine(ISD::INTRINSIC_VOID);
579 : setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
580 : setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
581 :
582 : setTargetDAGCombine(ISD::GlobalAddress);
583 :
584 : // In case of strict alignment, avoid an excessive number of byte wide stores.
585 1570 : MaxStoresPerMemsetOptSize = 8;
586 3140 : MaxStoresPerMemset = Subtarget->requiresStrictAlign()
587 1570 : ? MaxStoresPerMemsetOptSize : 32;
588 :
589 1570 : MaxGluedStoresPerMemcpy = 4;
590 1570 : MaxStoresPerMemcpyOptSize = 4;
591 3140 : MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
592 1570 : ? MaxStoresPerMemcpyOptSize : 16;
593 :
594 1570 : MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
595 :
596 : setStackPointerRegisterToSaveRestore(AArch64::SP);
597 :
598 : setSchedulingPreference(Sched::Hybrid);
599 :
600 1570 : EnableExtLdPromotion = true;
601 :
602 : // Set required alignment.
603 : setMinFunctionAlignment(2);
604 : // Set preferred alignments.
605 1570 : setPrefFunctionAlignment(STI.getPrefFunctionAlignment());
606 1570 : setPrefLoopAlignment(STI.getPrefLoopAlignment());
607 :
608 : // Only change the limit for entries in a jump table if specified by
609 : // the subtarget, but not at the command line.
610 1570 : unsigned MaxJT = STI.getMaximumJumpTableSize();
611 1570 : if (MaxJT && getMaximumJumpTableSize() == 0)
612 37 : setMaximumJumpTableSize(MaxJT);
613 :
614 : setHasExtractBitsInsn(true);
615 :
616 : setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
617 :
618 1570 : if (Subtarget->hasNEON()) {
619 : // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
620 : // silliness like this:
621 : setOperationAction(ISD::FABS, MVT::v1f64, Expand);
622 : setOperationAction(ISD::FADD, MVT::v1f64, Expand);
623 : setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
624 : setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
625 : setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
626 : setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
627 : setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
628 : setOperationAction(ISD::FMA, MVT::v1f64, Expand);
629 : setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
630 : setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
631 : setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
632 : setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
633 : setOperationAction(ISD::FREM, MVT::v1f64, Expand);
634 : setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
635 : setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
636 : setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
637 : setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
638 : setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
639 : setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
640 : setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
641 : setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
642 : setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
643 : setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
644 : setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
645 : setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
646 :
647 : setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
648 : setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
649 : setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
650 : setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
651 : setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
652 :
653 : setOperationAction(ISD::MUL, MVT::v1i64, Expand);
654 :
655 : // AArch64 doesn't have a direct vector ->f32 conversion instructions for
656 : // elements smaller than i32, so promote the input to i32 first.
657 : setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
658 : setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
659 : setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
660 : setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
661 : // i8 and i16 vector elements also need promotion to i32 for v8i8 or v8i16
662 : // -> v8f16 conversions.
663 : setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
664 : setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
665 : setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
666 : setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
667 : // Similarly, there is no direct i32 -> f64 vector conversion instruction.
668 : setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
669 : setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
670 : setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
671 : setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
672 : // Or, direct i32 -> f16 vector conversion. Set it so custom, so the
673 : // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
674 : setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
675 : setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
676 :
677 : setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
678 : setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
679 :
680 : setOperationAction(ISD::CTTZ, MVT::v2i8, Expand);
681 : setOperationAction(ISD::CTTZ, MVT::v4i16, Expand);
682 : setOperationAction(ISD::CTTZ, MVT::v2i32, Expand);
683 : setOperationAction(ISD::CTTZ, MVT::v1i64, Expand);
684 : setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);
685 : setOperationAction(ISD::CTTZ, MVT::v8i16, Expand);
686 : setOperationAction(ISD::CTTZ, MVT::v4i32, Expand);
687 : setOperationAction(ISD::CTTZ, MVT::v2i64, Expand);
688 :
689 : // AArch64 doesn't have MUL.2d:
690 : setOperationAction(ISD::MUL, MVT::v2i64, Expand);
691 : // Custom handling for some quad-vector types to detect MULL.
692 : setOperationAction(ISD::MUL, MVT::v8i16, Custom);
693 : setOperationAction(ISD::MUL, MVT::v4i32, Custom);
694 : setOperationAction(ISD::MUL, MVT::v2i64, Custom);
695 :
696 : // Vector reductions
697 10829 : for (MVT VT : MVT::integer_valuetypes()) {
698 : setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
699 : setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
700 : setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
701 : setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
702 : setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
703 : }
704 10829 : for (MVT VT : MVT::fp_valuetypes()) {
705 : setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
706 : setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
707 : }
708 :
709 : setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
710 : setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
711 : // Likewise, narrowing and extending vector loads/stores aren't handled
712 : // directly.
713 148512 : for (MVT VT : MVT::vector_valuetypes()) {
714 : setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
715 :
716 146965 : if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
717 : setOperationAction(ISD::MULHS, VT, Custom);
718 : setOperationAction(ISD::MULHU, VT, Custom);
719 : } else {
720 : setOperationAction(ISD::MULHS, VT, Expand);
721 : setOperationAction(ISD::MULHU, VT, Expand);
722 : }
723 : setOperationAction(ISD::SMUL_LOHI, VT, Expand);
724 : setOperationAction(ISD::UMUL_LOHI, VT, Expand);
725 :
726 : setOperationAction(ISD::BSWAP, VT, Expand);
727 :
728 14108640 : for (MVT InnerVT : MVT::vector_valuetypes()) {
729 : setTruncStoreAction(VT, InnerVT, Expand);
730 : setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
731 : setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
732 : setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
733 : }
734 : }
735 :
736 : // AArch64 has implementations of a lot of rounding-like FP operations.
737 6188 : for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
738 : setOperationAction(ISD::FFLOOR, Ty, Legal);
739 : setOperationAction(ISD::FNEARBYINT, Ty, Legal);
740 : setOperationAction(ISD::FCEIL, Ty, Legal);
741 : setOperationAction(ISD::FRINT, Ty, Legal);
742 : setOperationAction(ISD::FTRUNC, Ty, Legal);
743 : setOperationAction(ISD::FROUND, Ty, Legal);
744 : }
745 :
746 : setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
747 : }
748 :
749 1570 : PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
750 1570 : }
751 :
752 21658 : void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
753 : assert(VT.isVector() && "VT should be a vector type");
754 :
755 43316 : if (VT.isFloatingPoint()) {
756 9282 : MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
757 : setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
758 : setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
759 : }
760 :
761 : // Mark vector float intrinsics as expand.
762 21658 : if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
763 : setOperationAction(ISD::FSIN, VT, Expand);
764 : setOperationAction(ISD::FCOS, VT, Expand);
765 : setOperationAction(ISD::FPOW, VT, Expand);
766 : setOperationAction(ISD::FLOG, VT, Expand);
767 : setOperationAction(ISD::FLOG2, VT, Expand);
768 : setOperationAction(ISD::FLOG10, VT, Expand);
769 : setOperationAction(ISD::FEXP, VT, Expand);
770 : setOperationAction(ISD::FEXP2, VT, Expand);
771 :
772 : // But we do support custom-lowering for FCOPYSIGN.
773 : setOperationAction(ISD::FCOPYSIGN, VT, Custom);
774 : }
775 :
776 : setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
777 : setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
778 : setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
779 : setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
780 : setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
781 : setOperationAction(ISD::SRA, VT, Custom);
782 : setOperationAction(ISD::SRL, VT, Custom);
783 : setOperationAction(ISD::SHL, VT, Custom);
784 : setOperationAction(ISD::AND, VT, Custom);
785 : setOperationAction(ISD::OR, VT, Custom);
786 : setOperationAction(ISD::SETCC, VT, Custom);
787 : setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
788 :
789 : setOperationAction(ISD::SELECT, VT, Expand);
790 : setOperationAction(ISD::SELECT_CC, VT, Expand);
791 : setOperationAction(ISD::VSELECT, VT, Expand);
792 2469012 : for (MVT InnerVT : MVT::all_valuetypes())
793 : setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
794 :
795 : // CNT supports only B element sizes, then use UADDLP to widen.
796 21658 : if (VT != MVT::v8i8 && VT != MVT::v16i8)
797 : setOperationAction(ISD::CTPOP, VT, Custom);
798 :
799 : setOperationAction(ISD::UDIV, VT, Expand);
800 : setOperationAction(ISD::SDIV, VT, Expand);
801 : setOperationAction(ISD::UREM, VT, Expand);
802 : setOperationAction(ISD::SREM, VT, Expand);
803 : setOperationAction(ISD::FREM, VT, Expand);
804 :
805 : setOperationAction(ISD::FP_TO_SINT, VT, Custom);
806 : setOperationAction(ISD::FP_TO_UINT, VT, Custom);
807 :
808 21658 : if (!VT.isFloatingPoint())
809 : setOperationAction(ISD::ABS, VT, Legal);
810 :
811 : // [SU][MIN|MAX] are available for all NEON types apart from i64.
812 21658 : if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
813 46410 : for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
814 : setOperationAction(Opcode, VT, Legal);
815 :
816 : // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
817 43316 : if (VT.isFloatingPoint() &&
818 21658 : (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
819 49824 : for (unsigned Opcode : {ISD::FMINNAN, ISD::FMAXNAN,
820 31140 : ISD::FMINNUM, ISD::FMAXNUM})
821 : setOperationAction(Opcode, VT, Legal);
822 :
823 21658 : if (Subtarget->isLittleEndian()) {
824 85008 : for (unsigned im = (unsigned)ISD::PRE_INC;
825 106260 : im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
826 : setIndexedLoadAction(im, VT, Legal);
827 : setIndexedStoreAction(im, VT, Legal);
828 : }
829 : }
830 21658 : }
831 :
832 10829 : void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
833 : addRegisterClass(VT, &AArch64::FPR64RegClass);
834 10829 : addTypeForNEON(VT, MVT::v2i32);
835 10829 : }
836 :
837 10829 : void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
838 : addRegisterClass(VT, &AArch64::FPR128RegClass);
839 10829 : addTypeForNEON(VT, MVT::v4i32);
840 10829 : }
841 :
842 5239 : EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
843 : EVT VT) const {
844 5239 : if (!VT.isVector())
845 4561 : return MVT::i32;
846 678 : return VT.changeVectorElementTypeToInteger();
847 : }
848 :
849 742 : static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
850 : const APInt &Demanded,
851 : TargetLowering::TargetLoweringOpt &TLO,
852 : unsigned NewOpc) {
853 : uint64_t OldImm = Imm, NewImm, Enc;
854 742 : uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
855 :
856 : // Return if the immediate is already all zeros, all ones, a bimm32 or a
857 : // bimm64.
858 1484 : if (Imm == 0 || Imm == Mask ||
859 742 : AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
860 719 : return false;
861 :
862 : unsigned EltSize = Size;
863 : uint64_t DemandedBits = Demanded.getZExtValue();
864 :
865 : // Clear bits that are not demanded.
866 23 : Imm &= DemandedBits;
867 :
868 : while (true) {
869 : // The goal here is to set the non-demanded bits in a way that minimizes
870 : // the number of switching between 0 and 1. In order to achieve this goal,
871 : // we set the non-demanded bits to the value of the preceding demanded bits.
872 : // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
873 : // non-demanded bit), we copy bit0 (1) to the least significant 'x',
874 : // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
875 : // The final result is 0b11000011.
876 28 : uint64_t NonDemandedBits = ~DemandedBits;
877 28 : uint64_t InvertedImm = ~Imm & DemandedBits;
878 28 : uint64_t RotatedImm =
879 28 : ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
880 : NonDemandedBits;
881 28 : uint64_t Sum = RotatedImm + NonDemandedBits;
882 28 : bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
883 28 : uint64_t Ones = (Sum + Carry) & NonDemandedBits;
884 28 : NewImm = (Imm | Ones) & Mask;
885 :
886 : // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
887 : // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
888 : // we halve the element size and continue the search.
889 28 : if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
890 : break;
891 :
892 : // We cannot shrink the element size any further if it is 2-bits.
893 23 : if (EltSize == 2)
894 : return false;
895 :
896 23 : EltSize /= 2;
897 23 : Mask >>= EltSize;
898 23 : uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
899 :
900 : // Return if there is mismatch in any of the demanded bits of Imm and Hi.
901 23 : if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
902 : return false;
903 :
904 : // Merge the upper and lower halves of Imm and DemandedBits.
905 5 : Imm |= Hi;
906 5 : DemandedBits |= DemandedBitsHi;
907 5 : }
908 :
909 : ++NumOptimizedImms;
910 :
911 : // Replicate the element across the register width.
912 10 : while (EltSize < Size) {
913 5 : NewImm |= NewImm << EltSize;
914 5 : EltSize *= 2;
915 : }
916 :
917 : (void)OldImm;
918 : assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
919 : "demanded bits should never be altered");
920 : assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
921 :
922 : // Create the new constant immediate node.
923 5 : EVT VT = Op.getValueType();
924 : SDLoc DL(Op);
925 : SDValue New;
926 :
927 : // If the new constant immediate is all-zeros or all-ones, let the target
928 : // independent DAG combine optimize this node.
929 5 : if (NewImm == 0 || NewImm == OrigMask) {
930 1 : New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
931 1 : TLO.DAG.getConstant(NewImm, DL, VT));
932 : // Otherwise, create a machine node so that target independent DAG combine
933 : // doesn't undo this optimization.
934 : } else {
935 : Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
936 4 : SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
937 : New = SDValue(
938 8 : TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
939 : }
940 :
941 5 : return TLO.CombineTo(Op, New);
942 : }
943 :
944 9892 : bool AArch64TargetLowering::targetShrinkDemandedConstant(
945 : SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const {
946 : // Delay this optimization to as late as possible.
947 9892 : if (!TLO.LegalOps)
948 : return false;
949 :
950 4991 : if (!EnableOptimizeLogicalImm)
951 : return false;
952 :
953 9982 : EVT VT = Op.getValueType();
954 4991 : if (VT.isVector())
955 : return false;
956 :
957 3706 : unsigned Size = VT.getSizeInBits();
958 : assert((Size == 32 || Size == 64) &&
959 : "i32 or i64 is expected after legalization.");
960 :
961 : // Exit early if we demand all bits.
962 3706 : if (Demanded.countPopulation() == Size)
963 : return false;
964 :
965 : unsigned NewOpc;
966 831 : switch (Op.getOpcode()) {
967 : default:
968 : return false;
969 760 : case ISD::AND:
970 760 : NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
971 : break;
972 67 : case ISD::OR:
973 67 : NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
974 : break;
975 2 : case ISD::XOR:
976 2 : NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
977 : break;
978 : }
979 : ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
980 : if (!C)
981 : return false;
982 742 : uint64_t Imm = C->getZExtValue();
983 742 : return optimizeLogicalImm(Op, Size, Imm, Demanded, TLO, NewOpc);
984 : }
985 :
986 : /// computeKnownBitsForTargetNode - Determine which of the bits specified in
987 : /// Mask are known to be either zero or one and return them Known.
988 10058 : void AArch64TargetLowering::computeKnownBitsForTargetNode(
989 : const SDValue Op, KnownBits &Known,
990 : const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
991 10058 : switch (Op.getOpcode()) {
992 : default:
993 : break;
994 : case AArch64ISD::CSEL: {
995 413 : KnownBits Known2;
996 826 : DAG.computeKnownBits(Op->getOperand(0), Known, Depth + 1);
997 826 : DAG.computeKnownBits(Op->getOperand(1), Known2, Depth + 1);
998 413 : Known.Zero &= Known2.Zero;
999 413 : Known.One &= Known2.One;
1000 : break;
1001 : }
1002 1193 : case ISD::INTRINSIC_W_CHAIN: {
1003 1193 : ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1004 1193 : Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1005 1193 : switch (IntID) {
1006 : default: return;
1007 : case Intrinsic::aarch64_ldaxr:
1008 : case Intrinsic::aarch64_ldxr: {
1009 : unsigned BitWidth = Known.getBitWidth();
1010 1013 : EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1011 : unsigned MemBits = VT.getScalarSizeInBits();
1012 2026 : Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1013 : return;
1014 : }
1015 : }
1016 : break;
1017 : }
1018 1453 : case ISD::INTRINSIC_WO_CHAIN:
1019 : case ISD::INTRINSIC_VOID: {
1020 2906 : unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1021 : switch (IntNo) {
1022 : default:
1023 : break;
1024 25 : case Intrinsic::aarch64_neon_umaxv:
1025 : case Intrinsic::aarch64_neon_uminv: {
1026 : // Figure out the datatype of the vector operand. The UMINV instruction
1027 : // will zero extend the result, so we can mark as known zero all the
1028 : // bits larger than the element datatype. 32-bit or larget doesn't need
1029 : // this as those are legal types and will be handled by isel directly.
1030 25 : MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1031 : unsigned BitWidth = Known.getBitWidth();
1032 25 : if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1033 : assert(BitWidth >= 8 && "Unexpected width!");
1034 16 : APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1035 16 : Known.Zero |= Mask;
1036 9 : } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1037 : assert(BitWidth >= 16 && "Unexpected width!");
1038 9 : APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1039 9 : Known.Zero |= Mask;
1040 : }
1041 : break;
1042 : } break;
1043 : }
1044 : }
1045 : }
1046 : }
1047 :
1048 4464 : MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1049 : EVT) const {
1050 4464 : return MVT::i64;
1051 : }
1052 :
1053 2644 : bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
1054 : unsigned AddrSpace,
1055 : unsigned Align,
1056 : bool *Fast) const {
1057 2644 : if (Subtarget->requiresStrictAlign())
1058 : return false;
1059 :
1060 2152 : if (Fast) {
1061 : // Some CPUs are fine with unaligned stores except for 128-bit ones.
1062 266 : *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1063 : // See comments in performSTORECombine() for more details about
1064 : // these conditions.
1065 :
1066 : // Code that uses clang vector extensions can mark that it
1067 : // wants unaligned accesses to be treated as fast by
1068 : // underspecifying alignment to be 1 or 2.
1069 240 : Align <= 2 ||
1070 :
1071 : // Disregard v2i64. Memcpy lowering produces those and splitting
1072 : // them regresses performance on micro-benchmarks and olden/bh.
1073 : VT == MVT::v2i64;
1074 : }
1075 : return true;
1076 : }
1077 :
1078 : FastISel *
1079 1222 : AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1080 : const TargetLibraryInfo *libInfo) const {
1081 1222 : return AArch64::createFastISel(funcInfo, libInfo);
1082 : }
1083 :
1084 0 : const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1085 0 : switch ((AArch64ISD::NodeType)Opcode) {
1086 : case AArch64ISD::FIRST_NUMBER: break;
1087 : case AArch64ISD::CALL: return "AArch64ISD::CALL";
1088 0 : case AArch64ISD::ADRP: return "AArch64ISD::ADRP";
1089 0 : case AArch64ISD::ADR: return "AArch64ISD::ADR";
1090 0 : case AArch64ISD::ADDlow: return "AArch64ISD::ADDlow";
1091 0 : case AArch64ISD::LOADgot: return "AArch64ISD::LOADgot";
1092 0 : case AArch64ISD::RET_FLAG: return "AArch64ISD::RET_FLAG";
1093 0 : case AArch64ISD::BRCOND: return "AArch64ISD::BRCOND";
1094 0 : case AArch64ISD::CSEL: return "AArch64ISD::CSEL";
1095 0 : case AArch64ISD::FCSEL: return "AArch64ISD::FCSEL";
1096 0 : case AArch64ISD::CSINV: return "AArch64ISD::CSINV";
1097 0 : case AArch64ISD::CSNEG: return "AArch64ISD::CSNEG";
1098 0 : case AArch64ISD::CSINC: return "AArch64ISD::CSINC";
1099 0 : case AArch64ISD::THREAD_POINTER: return "AArch64ISD::THREAD_POINTER";
1100 0 : case AArch64ISD::TLSDESC_CALLSEQ: return "AArch64ISD::TLSDESC_CALLSEQ";
1101 0 : case AArch64ISD::ADC: return "AArch64ISD::ADC";
1102 0 : case AArch64ISD::SBC: return "AArch64ISD::SBC";
1103 0 : case AArch64ISD::ADDS: return "AArch64ISD::ADDS";
1104 0 : case AArch64ISD::SUBS: return "AArch64ISD::SUBS";
1105 0 : case AArch64ISD::ADCS: return "AArch64ISD::ADCS";
1106 0 : case AArch64ISD::SBCS: return "AArch64ISD::SBCS";
1107 0 : case AArch64ISD::ANDS: return "AArch64ISD::ANDS";
1108 0 : case AArch64ISD::CCMP: return "AArch64ISD::CCMP";
1109 0 : case AArch64ISD::CCMN: return "AArch64ISD::CCMN";
1110 0 : case AArch64ISD::FCCMP: return "AArch64ISD::FCCMP";
1111 0 : case AArch64ISD::FCMP: return "AArch64ISD::FCMP";
1112 0 : case AArch64ISD::DUP: return "AArch64ISD::DUP";
1113 0 : case AArch64ISD::DUPLANE8: return "AArch64ISD::DUPLANE8";
1114 0 : case AArch64ISD::DUPLANE16: return "AArch64ISD::DUPLANE16";
1115 0 : case AArch64ISD::DUPLANE32: return "AArch64ISD::DUPLANE32";
1116 0 : case AArch64ISD::DUPLANE64: return "AArch64ISD::DUPLANE64";
1117 0 : case AArch64ISD::MOVI: return "AArch64ISD::MOVI";
1118 0 : case AArch64ISD::MOVIshift: return "AArch64ISD::MOVIshift";
1119 0 : case AArch64ISD::MOVIedit: return "AArch64ISD::MOVIedit";
1120 0 : case AArch64ISD::MOVImsl: return "AArch64ISD::MOVImsl";
1121 0 : case AArch64ISD::FMOV: return "AArch64ISD::FMOV";
1122 0 : case AArch64ISD::MVNIshift: return "AArch64ISD::MVNIshift";
1123 0 : case AArch64ISD::MVNImsl: return "AArch64ISD::MVNImsl";
1124 0 : case AArch64ISD::BICi: return "AArch64ISD::BICi";
1125 0 : case AArch64ISD::ORRi: return "AArch64ISD::ORRi";
1126 0 : case AArch64ISD::BSL: return "AArch64ISD::BSL";
1127 0 : case AArch64ISD::NEG: return "AArch64ISD::NEG";
1128 0 : case AArch64ISD::EXTR: return "AArch64ISD::EXTR";
1129 0 : case AArch64ISD::ZIP1: return "AArch64ISD::ZIP1";
1130 0 : case AArch64ISD::ZIP2: return "AArch64ISD::ZIP2";
1131 0 : case AArch64ISD::UZP1: return "AArch64ISD::UZP1";
1132 0 : case AArch64ISD::UZP2: return "AArch64ISD::UZP2";
1133 0 : case AArch64ISD::TRN1: return "AArch64ISD::TRN1";
1134 0 : case AArch64ISD::TRN2: return "AArch64ISD::TRN2";
1135 0 : case AArch64ISD::REV16: return "AArch64ISD::REV16";
1136 0 : case AArch64ISD::REV32: return "AArch64ISD::REV32";
1137 0 : case AArch64ISD::REV64: return "AArch64ISD::REV64";
1138 0 : case AArch64ISD::EXT: return "AArch64ISD::EXT";
1139 0 : case AArch64ISD::VSHL: return "AArch64ISD::VSHL";
1140 0 : case AArch64ISD::VLSHR: return "AArch64ISD::VLSHR";
1141 0 : case AArch64ISD::VASHR: return "AArch64ISD::VASHR";
1142 0 : case AArch64ISD::CMEQ: return "AArch64ISD::CMEQ";
1143 0 : case AArch64ISD::CMGE: return "AArch64ISD::CMGE";
1144 0 : case AArch64ISD::CMGT: return "AArch64ISD::CMGT";
1145 0 : case AArch64ISD::CMHI: return "AArch64ISD::CMHI";
1146 0 : case AArch64ISD::CMHS: return "AArch64ISD::CMHS";
1147 0 : case AArch64ISD::FCMEQ: return "AArch64ISD::FCMEQ";
1148 0 : case AArch64ISD::FCMGE: return "AArch64ISD::FCMGE";
1149 0 : case AArch64ISD::FCMGT: return "AArch64ISD::FCMGT";
1150 0 : case AArch64ISD::CMEQz: return "AArch64ISD::CMEQz";
1151 0 : case AArch64ISD::CMGEz: return "AArch64ISD::CMGEz";
1152 0 : case AArch64ISD::CMGTz: return "AArch64ISD::CMGTz";
1153 0 : case AArch64ISD::CMLEz: return "AArch64ISD::CMLEz";
1154 0 : case AArch64ISD::CMLTz: return "AArch64ISD::CMLTz";
1155 0 : case AArch64ISD::FCMEQz: return "AArch64ISD::FCMEQz";
1156 0 : case AArch64ISD::FCMGEz: return "AArch64ISD::FCMGEz";
1157 0 : case AArch64ISD::FCMGTz: return "AArch64ISD::FCMGTz";
1158 0 : case AArch64ISD::FCMLEz: return "AArch64ISD::FCMLEz";
1159 0 : case AArch64ISD::FCMLTz: return "AArch64ISD::FCMLTz";
1160 0 : case AArch64ISD::SADDV: return "AArch64ISD::SADDV";
1161 0 : case AArch64ISD::UADDV: return "AArch64ISD::UADDV";
1162 0 : case AArch64ISD::SMINV: return "AArch64ISD::SMINV";
1163 0 : case AArch64ISD::UMINV: return "AArch64ISD::UMINV";
1164 0 : case AArch64ISD::SMAXV: return "AArch64ISD::SMAXV";
1165 0 : case AArch64ISD::UMAXV: return "AArch64ISD::UMAXV";
1166 0 : case AArch64ISD::NOT: return "AArch64ISD::NOT";
1167 0 : case AArch64ISD::BIT: return "AArch64ISD::BIT";
1168 0 : case AArch64ISD::CBZ: return "AArch64ISD::CBZ";
1169 0 : case AArch64ISD::CBNZ: return "AArch64ISD::CBNZ";
1170 0 : case AArch64ISD::TBZ: return "AArch64ISD::TBZ";
1171 0 : case AArch64ISD::TBNZ: return "AArch64ISD::TBNZ";
1172 0 : case AArch64ISD::TC_RETURN: return "AArch64ISD::TC_RETURN";
1173 0 : case AArch64ISD::PREFETCH: return "AArch64ISD::PREFETCH";
1174 0 : case AArch64ISD::SITOF: return "AArch64ISD::SITOF";
1175 0 : case AArch64ISD::UITOF: return "AArch64ISD::UITOF";
1176 0 : case AArch64ISD::NVCAST: return "AArch64ISD::NVCAST";
1177 0 : case AArch64ISD::SQSHL_I: return "AArch64ISD::SQSHL_I";
1178 0 : case AArch64ISD::UQSHL_I: return "AArch64ISD::UQSHL_I";
1179 0 : case AArch64ISD::SRSHR_I: return "AArch64ISD::SRSHR_I";
1180 0 : case AArch64ISD::URSHR_I: return "AArch64ISD::URSHR_I";
1181 0 : case AArch64ISD::SQSHLU_I: return "AArch64ISD::SQSHLU_I";
1182 0 : case AArch64ISD::WrapperLarge: return "AArch64ISD::WrapperLarge";
1183 0 : case AArch64ISD::LD2post: return "AArch64ISD::LD2post";
1184 0 : case AArch64ISD::LD3post: return "AArch64ISD::LD3post";
1185 0 : case AArch64ISD::LD4post: return "AArch64ISD::LD4post";
1186 0 : case AArch64ISD::ST2post: return "AArch64ISD::ST2post";
1187 0 : case AArch64ISD::ST3post: return "AArch64ISD::ST3post";
1188 0 : case AArch64ISD::ST4post: return "AArch64ISD::ST4post";
1189 0 : case AArch64ISD::LD1x2post: return "AArch64ISD::LD1x2post";
1190 0 : case AArch64ISD::LD1x3post: return "AArch64ISD::LD1x3post";
1191 0 : case AArch64ISD::LD1x4post: return "AArch64ISD::LD1x4post";
1192 0 : case AArch64ISD::ST1x2post: return "AArch64ISD::ST1x2post";
1193 0 : case AArch64ISD::ST1x3post: return "AArch64ISD::ST1x3post";
1194 0 : case AArch64ISD::ST1x4post: return "AArch64ISD::ST1x4post";
1195 0 : case AArch64ISD::LD1DUPpost: return "AArch64ISD::LD1DUPpost";
1196 0 : case AArch64ISD::LD2DUPpost: return "AArch64ISD::LD2DUPpost";
1197 0 : case AArch64ISD::LD3DUPpost: return "AArch64ISD::LD3DUPpost";
1198 0 : case AArch64ISD::LD4DUPpost: return "AArch64ISD::LD4DUPpost";
1199 0 : case AArch64ISD::LD1LANEpost: return "AArch64ISD::LD1LANEpost";
1200 0 : case AArch64ISD::LD2LANEpost: return "AArch64ISD::LD2LANEpost";
1201 0 : case AArch64ISD::LD3LANEpost: return "AArch64ISD::LD3LANEpost";
1202 0 : case AArch64ISD::LD4LANEpost: return "AArch64ISD::LD4LANEpost";
1203 0 : case AArch64ISD::ST2LANEpost: return "AArch64ISD::ST2LANEpost";
1204 0 : case AArch64ISD::ST3LANEpost: return "AArch64ISD::ST3LANEpost";
1205 0 : case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost";
1206 0 : case AArch64ISD::SMULL: return "AArch64ISD::SMULL";
1207 0 : case AArch64ISD::UMULL: return "AArch64ISD::UMULL";
1208 0 : case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE";
1209 0 : case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS";
1210 0 : case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE";
1211 0 : case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS";
1212 : }
1213 0 : return nullptr;
1214 : }
1215 :
1216 : MachineBasicBlock *
1217 3 : AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
1218 : MachineBasicBlock *MBB) const {
1219 : // We materialise the F128CSEL pseudo-instruction as some control flow and a
1220 : // phi node:
1221 :
1222 : // OrigBB:
1223 : // [... previous instrs leading to comparison ...]
1224 : // b.ne TrueBB
1225 : // b EndBB
1226 : // TrueBB:
1227 : // ; Fallthrough
1228 : // EndBB:
1229 : // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
1230 :
1231 3 : MachineFunction *MF = MBB->getParent();
1232 3 : const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1233 3 : const BasicBlock *LLVM_BB = MBB->getBasicBlock();
1234 : DebugLoc DL = MI.getDebugLoc();
1235 3 : MachineFunction::iterator It = ++MBB->getIterator();
1236 :
1237 3 : unsigned DestReg = MI.getOperand(0).getReg();
1238 3 : unsigned IfTrueReg = MI.getOperand(1).getReg();
1239 3 : unsigned IfFalseReg = MI.getOperand(2).getReg();
1240 3 : unsigned CondCode = MI.getOperand(3).getImm();
1241 : bool NZCVKilled = MI.getOperand(4).isKill();
1242 :
1243 3 : MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
1244 3 : MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
1245 : MF->insert(It, TrueBB);
1246 : MF->insert(It, EndBB);
1247 :
1248 : // Transfer rest of current basic-block to EndBB
1249 3 : EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
1250 : MBB->end());
1251 3 : EndBB->transferSuccessorsAndUpdatePHIs(MBB);
1252 :
1253 3 : BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
1254 3 : BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
1255 3 : MBB->addSuccessor(TrueBB);
1256 3 : MBB->addSuccessor(EndBB);
1257 :
1258 : // TrueBB falls through to the end.
1259 3 : TrueBB->addSuccessor(EndBB);
1260 :
1261 3 : if (!NZCVKilled) {
1262 : TrueBB->addLiveIn(AArch64::NZCV);
1263 : EndBB->addLiveIn(AArch64::NZCV);
1264 : }
1265 :
1266 6 : BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
1267 3 : .addReg(IfTrueReg)
1268 : .addMBB(TrueBB)
1269 3 : .addReg(IfFalseReg)
1270 : .addMBB(MBB);
1271 :
1272 3 : MI.eraseFromParent();
1273 3 : return EndBB;
1274 : }
1275 :
1276 67 : MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
1277 : MachineInstr &MI, MachineBasicBlock *BB) const {
1278 134 : switch (MI.getOpcode()) {
1279 0 : default:
1280 : #ifndef NDEBUG
1281 : MI.dump();
1282 : #endif
1283 0 : llvm_unreachable("Unexpected instruction for custom inserter!");
1284 :
1285 3 : case AArch64::F128CSEL:
1286 3 : return EmitF128CSEL(MI, BB);
1287 :
1288 64 : case TargetOpcode::STACKMAP:
1289 : case TargetOpcode::PATCHPOINT:
1290 64 : return emitPatchPoint(MI, BB);
1291 : }
1292 : }
1293 :
1294 : //===----------------------------------------------------------------------===//
1295 : // AArch64 Lowering private implementation.
1296 : //===----------------------------------------------------------------------===//
1297 :
1298 : //===----------------------------------------------------------------------===//
1299 : // Lowering Code
1300 : //===----------------------------------------------------------------------===//
1301 :
1302 : /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
1303 : /// CC
1304 1291 : static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
1305 1291 : switch (CC) {
1306 0 : default:
1307 0 : llvm_unreachable("Unknown condition code!");
1308 : case ISD::SETNE:
1309 : return AArch64CC::NE;
1310 290 : case ISD::SETEQ:
1311 290 : return AArch64CC::EQ;
1312 97 : case ISD::SETGT:
1313 97 : return AArch64CC::GT;
1314 65 : case ISD::SETGE:
1315 65 : return AArch64CC::GE;
1316 146 : case ISD::SETLT:
1317 146 : return AArch64CC::LT;
1318 89 : case ISD::SETLE:
1319 89 : return AArch64CC::LE;
1320 124 : case ISD::SETUGT:
1321 124 : return AArch64CC::HI;
1322 66 : case ISD::SETUGE:
1323 66 : return AArch64CC::HS;
1324 63 : case ISD::SETULT:
1325 63 : return AArch64CC::LO;
1326 75 : case ISD::SETULE:
1327 75 : return AArch64CC::LS;
1328 : }
1329 : }
1330 :
1331 : /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
1332 618 : static void changeFPCCToAArch64CC(ISD::CondCode CC,
1333 : AArch64CC::CondCode &CondCode,
1334 : AArch64CC::CondCode &CondCode2) {
1335 618 : CondCode2 = AArch64CC::AL;
1336 618 : switch (CC) {
1337 0 : default:
1338 0 : llvm_unreachable("Unknown FP condition!");
1339 83 : case ISD::SETEQ:
1340 : case ISD::SETOEQ:
1341 83 : CondCode = AArch64CC::EQ;
1342 83 : break;
1343 50 : case ISD::SETGT:
1344 : case ISD::SETOGT:
1345 50 : CondCode = AArch64CC::GT;
1346 50 : break;
1347 50 : case ISD::SETGE:
1348 : case ISD::SETOGE:
1349 50 : CondCode = AArch64CC::GE;
1350 50 : break;
1351 68 : case ISD::SETOLT:
1352 68 : CondCode = AArch64CC::MI;
1353 68 : break;
1354 47 : case ISD::SETOLE:
1355 47 : CondCode = AArch64CC::LS;
1356 47 : break;
1357 45 : case ISD::SETONE:
1358 45 : CondCode = AArch64CC::MI;
1359 45 : CondCode2 = AArch64CC::GT;
1360 45 : break;
1361 27 : case ISD::SETO:
1362 27 : CondCode = AArch64CC::VC;
1363 27 : break;
1364 27 : case ISD::SETUO:
1365 27 : CondCode = AArch64CC::VS;
1366 27 : break;
1367 31 : case ISD::SETUEQ:
1368 31 : CondCode = AArch64CC::EQ;
1369 31 : CondCode2 = AArch64CC::VS;
1370 31 : break;
1371 27 : case ISD::SETUGT:
1372 27 : CondCode = AArch64CC::HI;
1373 27 : break;
1374 33 : case ISD::SETUGE:
1375 33 : CondCode = AArch64CC::PL;
1376 33 : break;
1377 29 : case ISD::SETLT:
1378 : case ISD::SETULT:
1379 29 : CondCode = AArch64CC::LT;
1380 29 : break;
1381 36 : case ISD::SETLE:
1382 : case ISD::SETULE:
1383 36 : CondCode = AArch64CC::LE;
1384 36 : break;
1385 65 : case ISD::SETNE:
1386 : case ISD::SETUNE:
1387 65 : CondCode = AArch64CC::NE;
1388 65 : break;
1389 : }
1390 618 : }
1391 :
1392 : /// Convert a DAG fp condition code to an AArch64 CC.
1393 : /// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1394 : /// should be AND'ed instead of OR'ed.
1395 30 : static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
1396 : AArch64CC::CondCode &CondCode,
1397 : AArch64CC::CondCode &CondCode2) {
1398 30 : CondCode2 = AArch64CC::AL;
1399 30 : switch (CC) {
1400 19 : default:
1401 19 : changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1402 : assert(CondCode2 == AArch64CC::AL);
1403 19 : break;
1404 7 : case ISD::SETONE:
1405 : // (a one b)
1406 : // == ((a olt b) || (a ogt b))
1407 : // == ((a ord b) && (a une b))
1408 7 : CondCode = AArch64CC::VC;
1409 7 : CondCode2 = AArch64CC::NE;
1410 7 : break;
1411 4 : case ISD::SETUEQ:
1412 : // (a ueq b)
1413 : // == ((a uno b) || (a oeq b))
1414 : // == ((a ule b) && (a uge b))
1415 4 : CondCode = AArch64CC::PL;
1416 4 : CondCode2 = AArch64CC::LE;
1417 4 : break;
1418 : }
1419 30 : }
1420 :
1421 : /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
1422 : /// CC usable with the vector instructions. Fewer operations are available
1423 : /// without a real NZCV register, so we have to use less efficient combinations
1424 : /// to get the same effect.
1425 149 : static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
1426 : AArch64CC::CondCode &CondCode,
1427 : AArch64CC::CondCode &CondCode2,
1428 : bool &Invert) {
1429 149 : Invert = false;
1430 149 : switch (CC) {
1431 93 : default:
1432 : // Mostly the scalar mappings work fine.
1433 93 : changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1434 93 : break;
1435 8 : case ISD::SETUO:
1436 8 : Invert = true;
1437 : LLVM_FALLTHROUGH;
1438 16 : case ISD::SETO:
1439 16 : CondCode = AArch64CC::MI;
1440 16 : CondCode2 = AArch64CC::GE;
1441 16 : break;
1442 40 : case ISD::SETUEQ:
1443 : case ISD::SETULT:
1444 : case ISD::SETULE:
1445 : case ISD::SETUGT:
1446 : case ISD::SETUGE:
1447 : // All of the compare-mask comparisons are ordered, but we can switch
1448 : // between the two by a double inversion. E.g. ULE == !OGT.
1449 40 : Invert = true;
1450 40 : changeFPCCToAArch64CC(getSetCCInverse(CC, false), CondCode, CondCode2);
1451 40 : break;
1452 : }
1453 149 : }
1454 :
1455 : static bool isLegalArithImmed(uint64_t C) {
1456 : // Matches AArch64DAGToDAGISel::SelectArithImmed().
1457 966 : bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
1458 : LLVM_DEBUG(dbgs() << "Is imm " << C
1459 : << " legal: " << (IsLegal ? "yes\n" : "no\n"));
1460 : return IsLegal;
1461 : }
1462 :
1463 : // Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
1464 : // the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
1465 : // can be set differently by this operation. It comes down to whether
1466 : // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
1467 : // everything is fine. If not then the optimization is wrong. Thus general
1468 : // comparisons are only valid if op2 != 0.
1469 : //
1470 : // So, finally, the only LLVM-native comparisons that don't mention C and V
1471 : // are SETEQ and SETNE. They're the only ones we can safely use CMN for in
1472 : // the absence of information about op2.
1473 0 : static bool isCMN(SDValue Op, ISD::CondCode CC) {
1474 0 : return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
1475 0 : (CC == ISD::SETEQ || CC == ISD::SETNE);
1476 : }
1477 :
1478 1321 : static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1479 : const SDLoc &dl, SelectionDAG &DAG) {
1480 1321 : EVT VT = LHS.getValueType();
1481 : const bool FullFP16 =
1482 1321 : static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1483 :
1484 1321 : if (VT.isFloatingPoint()) {
1485 : assert(VT != MVT::f128);
1486 199 : if (VT == MVT::f16 && !FullFP16) {
1487 3 : LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
1488 3 : RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
1489 3 : VT = MVT::f32;
1490 : }
1491 443 : return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
1492 : }
1493 :
1494 : // The CMP instruction is just an alias for SUBS, and representing it as
1495 : // SUBS means that it's possible to get CSE with subtract operations.
1496 : // A later phase can perform the optimization of setting the destination
1497 : // register to WZR/XZR if it ends up being unused.
1498 : unsigned Opcode = AArch64ISD::SUBS;
1499 :
1500 878 : if (isCMN(RHS, CC)) {
1501 : // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
1502 : Opcode = AArch64ISD::ADDS;
1503 32 : RHS = RHS.getOperand(1);
1504 846 : } else if (LHS.getOpcode() == ISD::AND && isNullConstant(RHS) &&
1505 : !isUnsignedIntSetCC(CC)) {
1506 : // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
1507 : // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
1508 : // of the signed comparisons.
1509 : Opcode = AArch64ISD::ANDS;
1510 42 : RHS = LHS.getOperand(1);
1511 42 : LHS = LHS.getOperand(0);
1512 : }
1513 :
1514 878 : return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
1515 878 : .getValue(1);
1516 : }
1517 :
1518 : /// \defgroup AArch64CCMP CMP;CCMP matching
1519 : ///
1520 : /// These functions deal with the formation of CMP;CCMP;... sequences.
1521 : /// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
1522 : /// a comparison. They set the NZCV flags to a predefined value if their
1523 : /// predicate is false. This allows to express arbitrary conjunctions, for
1524 : /// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B))))"
1525 : /// expressed as:
1526 : /// cmp A
1527 : /// ccmp B, inv(CB), CA
1528 : /// check for CB flags
1529 : ///
1530 : /// In general we can create code for arbitrary "... (and (and A B) C)"
1531 : /// sequences. We can also implement some "or" expressions, because "(or A B)"
1532 : /// is equivalent to "not (and (not A) (not B))" and we can implement some
1533 : /// negation operations:
1534 : /// We can negate the results of a single comparison by inverting the flags
1535 : /// used when the predicate fails and inverting the flags tested in the next
1536 : /// instruction; We can also negate the results of the whole previous
1537 : /// conditional compare sequence by inverting the flags tested in the next
1538 : /// instruction. However there is no way to negate the result of a partial
1539 : /// sequence.
1540 : ///
1541 : /// Therefore on encountering an "or" expression we can negate the subtree on
1542 : /// one side and have to be able to push the negate to the leafs of the subtree
1543 : /// on the other side (see also the comments in code). As complete example:
1544 : /// "or (or (setCA (cmp A)) (setCB (cmp B)))
1545 : /// (and (setCC (cmp C)) (setCD (cmp D)))"
1546 : /// is transformed to
1547 : /// "not (and (not (and (setCC (cmp C)) (setCC (cmp D))))
1548 : /// (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
1549 : /// and implemented as:
1550 : /// cmp C
1551 : /// ccmp D, inv(CD), CC
1552 : /// ccmp A, CA, inv(CD)
1553 : /// ccmp B, CB, inv(CA)
1554 : /// check for CB flags
1555 : /// A counterexample is "or (and A B) (and C D)" which cannot be implemented
1556 : /// by conditional compare sequences.
1557 : /// @{
1558 :
1559 : /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
1560 39 : static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1561 : ISD::CondCode CC, SDValue CCOp,
1562 : AArch64CC::CondCode Predicate,
1563 : AArch64CC::CondCode OutCC,
1564 : const SDLoc &DL, SelectionDAG &DAG) {
1565 : unsigned Opcode = 0;
1566 : const bool FullFP16 =
1567 78 : static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1568 :
1569 39 : if (LHS.getValueType().isFloatingPoint()) {
1570 : assert(LHS.getValueType() != MVT::f128);
1571 5 : if (LHS.getValueType() == MVT::f16 && !FullFP16) {
1572 4 : LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
1573 4 : RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
1574 : }
1575 : Opcode = AArch64ISD::FCCMP;
1576 12 : } else if (RHS.getOpcode() == ISD::SUB) {
1577 0 : SDValue SubOp0 = RHS.getOperand(0);
1578 0 : if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
1579 : // See emitComparison() on why we can only do this for SETEQ and SETNE.
1580 : Opcode = AArch64ISD::CCMN;
1581 0 : RHS = RHS.getOperand(1);
1582 : }
1583 : }
1584 : if (Opcode == 0)
1585 : Opcode = AArch64ISD::CCMP;
1586 :
1587 39 : SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
1588 : AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
1589 : unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
1590 39 : SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
1591 39 : return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
1592 : }
1593 :
1594 : /// Returns true if @p Val is a tree of AND/OR/SETCC operations.
1595 : /// CanPushNegate is set to true if we can push a negate operation through
1596 : /// the tree in a was that we are left with AND operations and negate operations
1597 : /// at the leafs only. i.e. "not (or (or x y) z)" can be changed to
1598 : /// "and (and (not x) (not y)) (not z)"; "not (or (and x y) z)" cannot be
1599 : /// brought into such a form.
1600 0 : static bool isConjunctionDisjunctionTree(const SDValue Val, bool &CanNegate,
1601 : unsigned Depth = 0) {
1602 0 : if (!Val.hasOneUse())
1603 0 : return false;
1604 0 : unsigned Opcode = Val->getOpcode();
1605 0 : if (Opcode == ISD::SETCC) {
1606 0 : if (Val->getOperand(0).getValueType() == MVT::f128)
1607 0 : return false;
1608 0 : CanNegate = true;
1609 0 : return true;
1610 : }
1611 : // Protect against exponential runtime and stack overflow.
1612 0 : if (Depth > 6)
1613 0 : return false;
1614 0 : if (Opcode == ISD::AND || Opcode == ISD::OR) {
1615 0 : SDValue O0 = Val->getOperand(0);
1616 0 : SDValue O1 = Val->getOperand(1);
1617 : bool CanNegateL;
1618 0 : if (!isConjunctionDisjunctionTree(O0, CanNegateL, Depth+1))
1619 0 : return false;
1620 : bool CanNegateR;
1621 0 : if (!isConjunctionDisjunctionTree(O1, CanNegateR, Depth+1))
1622 0 : return false;
1623 :
1624 0 : if (Opcode == ISD::OR) {
1625 : // For an OR expression we need to be able to negate at least one side or
1626 : // we cannot do the transformation at all.
1627 0 : if (!CanNegateL && !CanNegateR)
1628 0 : return false;
1629 : // We can however change a (not (or x y)) to (and (not x) (not y)) if we
1630 : // can negate the x and y subtrees.
1631 0 : CanNegate = CanNegateL && CanNegateR;
1632 : } else {
1633 : // If the operands are OR expressions then we finally need to negate their
1634 : // outputs, we can only do that for the operand with emitted last by
1635 : // negating OutCC, not for both operands.
1636 0 : bool NeedsNegOutL = O0->getOpcode() == ISD::OR;
1637 0 : bool NeedsNegOutR = O1->getOpcode() == ISD::OR;
1638 0 : if (NeedsNegOutL && NeedsNegOutR)
1639 0 : return false;
1640 : // We cannot negate an AND operation (it would become an OR),
1641 0 : CanNegate = false;
1642 : }
1643 0 : return true;
1644 : }
1645 : return false;
1646 : }
1647 :
1648 : /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
1649 : /// of CCMP/CFCMP ops. See @ref AArch64CCMP.
1650 : /// Tries to transform the given i1 producing node @p Val to a series compare
1651 : /// and conditional compare operations. @returns an NZCV flags producing node
1652 : /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
1653 : /// transformation was not possible.
1654 : /// On recursive invocations @p PushNegate may be set to true to have negation
1655 : /// effects pushed to the tree leafs; @p Predicate is an NZCV flag predicate
1656 : /// for the comparisons in the current subtree; @p Depth limits the search
1657 : /// depth to avoid stack overflow.
1658 78 : static SDValue emitConjunctionDisjunctionTreeRec(SelectionDAG &DAG, SDValue Val,
1659 : AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
1660 : AArch64CC::CondCode Predicate) {
1661 : // We're at a tree leaf, produce a conditional comparison operation.
1662 78 : unsigned Opcode = Val->getOpcode();
1663 78 : if (Opcode == ISD::SETCC) {
1664 50 : SDValue LHS = Val->getOperand(0);
1665 50 : SDValue RHS = Val->getOperand(1);
1666 50 : ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
1667 50 : bool isInteger = LHS.getValueType().isInteger();
1668 50 : if (Negate)
1669 17 : CC = getSetCCInverse(CC, isInteger);
1670 : SDLoc DL(Val);
1671 : // Determine OutCC and handle FP special case.
1672 50 : if (isInteger) {
1673 20 : OutCC = changeIntCCToAArch64CC(CC);
1674 : } else {
1675 : assert(LHS.getValueType().isFloatingPoint());
1676 : AArch64CC::CondCode ExtraCC;
1677 30 : changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
1678 : // Some floating point conditions can't be tested with a single condition
1679 : // code. Construct an additional comparison in this case.
1680 30 : if (ExtraCC != AArch64CC::AL) {
1681 : SDValue ExtraCmp;
1682 11 : if (!CCOp.getNode())
1683 4 : ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
1684 : else
1685 7 : ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
1686 7 : ExtraCC, DL, DAG);
1687 11 : CCOp = ExtraCmp;
1688 11 : Predicate = ExtraCC;
1689 : }
1690 : }
1691 :
1692 : // Produce a normal comparison if we are first in the chain
1693 50 : if (!CCOp)
1694 18 : return emitComparison(LHS, RHS, CC, DL, DAG);
1695 : // Otherwise produce a ccmp.
1696 : return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
1697 32 : DAG);
1698 : }
1699 : assert((Opcode == ISD::AND || (Opcode == ISD::OR && Val->hasOneUse())) &&
1700 : "Valid conjunction/disjunction tree");
1701 :
1702 : // Check if both sides can be transformed.
1703 28 : SDValue LHS = Val->getOperand(0);
1704 28 : SDValue RHS = Val->getOperand(1);
1705 :
1706 : // In case of an OR we need to negate our operands and the result.
1707 : // (A v B) <=> not(not(A) ^ not(B))
1708 28 : bool NegateOpsAndResult = Opcode == ISD::OR;
1709 : // We can negate the results of all previous operations by inverting the
1710 : // predicate flags giving us a free negation for one side. The other side
1711 : // must be negatable by itself.
1712 28 : if (NegateOpsAndResult) {
1713 : // See which side we can negate.
1714 : bool CanNegateL;
1715 17 : bool isValidL = isConjunctionDisjunctionTree(LHS, CanNegateL);
1716 : assert(isValidL && "Valid conjunction/disjunction tree");
1717 : (void)isValidL;
1718 :
1719 : #ifndef NDEBUG
1720 : bool CanNegateR;
1721 : bool isValidR = isConjunctionDisjunctionTree(RHS, CanNegateR);
1722 : assert(isValidR && "Valid conjunction/disjunction tree");
1723 : assert((CanNegateL || CanNegateR) && "Valid conjunction/disjunction tree");
1724 : #endif
1725 :
1726 : // Order the side which we cannot negate to RHS so we can emit it first.
1727 17 : if (!CanNegateL)
1728 : std::swap(LHS, RHS);
1729 : } else {
1730 11 : bool NeedsNegOutL = LHS->getOpcode() == ISD::OR;
1731 : assert((!NeedsNegOutL || RHS->getOpcode() != ISD::OR) &&
1732 : "Valid conjunction/disjunction tree");
1733 : // Order the side where we need to negate the output flags to RHS so it
1734 : // gets emitted first.
1735 11 : if (NeedsNegOutL)
1736 : std::swap(LHS, RHS);
1737 : }
1738 :
1739 : // Emit RHS. If we want to negate the tree we only need to push a negate
1740 : // through if we are already in a PushNegate case, otherwise we can negate
1741 : // the "flags to test" afterwards.
1742 : AArch64CC::CondCode RHSCC;
1743 : SDValue CmpR = emitConjunctionDisjunctionTreeRec(DAG, RHS, RHSCC, Negate,
1744 28 : CCOp, Predicate);
1745 28 : if (NegateOpsAndResult && !Negate)
1746 32 : RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
1747 : // Emit LHS. We may need to negate it.
1748 : SDValue CmpL = emitConjunctionDisjunctionTreeRec(DAG, LHS, OutCC,
1749 : NegateOpsAndResult, CmpR,
1750 28 : RHSCC);
1751 : // If we transformed an OR to and AND then we have to negate the result
1752 : // (or absorb the Negate parameter).
1753 28 : if (NegateOpsAndResult && !Negate)
1754 32 : OutCC = AArch64CC::getInvertedCondCode(OutCC);
1755 28 : return CmpL;
1756 : }
1757 :
1758 : /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
1759 : /// of CCMP/CFCMP ops. See @ref AArch64CCMP.
1760 : /// \see emitConjunctionDisjunctionTreeRec().
1761 145 : static SDValue emitConjunctionDisjunctionTree(SelectionDAG &DAG, SDValue Val,
1762 : AArch64CC::CondCode &OutCC) {
1763 : bool CanNegate;
1764 145 : if (!isConjunctionDisjunctionTree(Val, CanNegate))
1765 123 : return SDValue();
1766 :
1767 : return emitConjunctionDisjunctionTreeRec(DAG, Val, OutCC, false, SDValue(),
1768 22 : AArch64CC::AL);
1769 : }
1770 :
1771 : /// @}
1772 :
1773 : /// Returns how profitable it is to fold a comparison's operand's shift and/or
1774 : /// extension operations.
1775 0 : static unsigned getCmpOperandFoldingProfit(SDValue Op) {
1776 : auto isSupportedExtend = [&](SDValue V) {
1777 : if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
1778 : return true;
1779 :
1780 : if (V.getOpcode() == ISD::AND)
1781 : if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
1782 : uint64_t Mask = MaskCst->getZExtValue();
1783 : return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
1784 : }
1785 :
1786 : return false;
1787 : };
1788 :
1789 0 : if (!Op.hasOneUse())
1790 0 : return 0;
1791 :
1792 0 : if (isSupportedExtend(Op))
1793 0 : return 1;
1794 :
1795 : unsigned Opc = Op.getOpcode();
1796 0 : if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
1797 : if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1798 0 : uint64_t Shift = ShiftCst->getZExtValue();
1799 0 : if (isSupportedExtend(Op.getOperand(0)))
1800 0 : return (Shift <= 4) ? 2 : 1;
1801 : EVT VT = Op.getValueType();
1802 0 : if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
1803 0 : return 1;
1804 : }
1805 :
1806 : return 0;
1807 : }
1808 :
1809 886 : static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1810 : SDValue &AArch64cc, SelectionDAG &DAG,
1811 : const SDLoc &dl) {
1812 : if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
1813 474 : EVT VT = RHS.getValueType();
1814 474 : uint64_t C = RHSC->getZExtValue();
1815 : if (!isLegalArithImmed(C)) {
1816 : // Constant does not fit, try adjusting it by one?
1817 36 : switch (CC) {
1818 : default:
1819 : break;
1820 : case ISD::SETLT:
1821 : case ISD::SETGE:
1822 4 : if ((VT == MVT::i32 && C != 0x80000000 &&
1823 4 : isLegalArithImmed((uint32_t)(C - 1))) ||
1824 0 : (VT == MVT::i64 && C != 0x80000000ULL &&
1825 0 : isLegalArithImmed(C - 1ULL))) {
1826 0 : CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
1827 0 : C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1828 0 : RHS = DAG.getConstant(C, dl, VT);
1829 : }
1830 : break;
1831 : case ISD::SETULT:
1832 : case ISD::SETUGE:
1833 : if ((VT == MVT::i32 && C != 0 &&
1834 3 : isLegalArithImmed((uint32_t)(C - 1))) ||
1835 1 : (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
1836 0 : CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
1837 0 : C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1838 0 : RHS = DAG.getConstant(C, dl, VT);
1839 : }
1840 : break;
1841 : case ISD::SETLE:
1842 : case ISD::SETGT:
1843 9 : if ((VT == MVT::i32 && C != INT32_MAX &&
1844 9 : isLegalArithImmed((uint32_t)(C + 1))) ||
1845 3 : (VT == MVT::i64 && C != INT64_MAX &&
1846 3 : isLegalArithImmed(C + 1ULL))) {
1847 6 : CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
1848 6 : C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1849 6 : RHS = DAG.getConstant(C, dl, VT);
1850 : }
1851 : break;
1852 : case ISD::SETULE:
1853 : case ISD::SETUGT:
1854 4 : if ((VT == MVT::i32 && C != UINT32_MAX &&
1855 4 : isLegalArithImmed((uint32_t)(C + 1))) ||
1856 2 : (VT == MVT::i64 && C != UINT64_MAX &&
1857 0 : isLegalArithImmed(C + 1ULL))) {
1858 0 : CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
1859 0 : C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1860 0 : RHS = DAG.getConstant(C, dl, VT);
1861 : }
1862 : break;
1863 : }
1864 : }
1865 : }
1866 :
1867 : // Comparisons are canonicalized so that the RHS operand is simpler than the
1868 : // LHS one, the extreme case being when RHS is an immediate. However, AArch64
1869 : // can fold some shift+extend operations on the RHS operand, so swap the
1870 : // operands if that can be done.
1871 : //
1872 : // For example:
1873 : // lsl w13, w11, #1
1874 : // cmp w13, w12
1875 : // can be turned into:
1876 : // cmp w12, w11, lsl #1
1877 : if (!isa<ConstantSDNode>(RHS) ||
1878 474 : !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
1879 442 : SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
1880 :
1881 442 : if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
1882 : std::swap(LHS, RHS);
1883 97 : CC = ISD::getSetCCSwappedOperands(CC);
1884 : }
1885 : }
1886 :
1887 : SDValue Cmp;
1888 : AArch64CC::CondCode AArch64CC;
1889 886 : if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
1890 : const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
1891 :
1892 : // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
1893 : // For the i8 operand, the largest immediate is 255, so this can be easily
1894 : // encoded in the compare instruction. For the i16 operand, however, the
1895 : // largest immediate cannot be encoded in the compare.
1896 : // Therefore, use a sign extending load and cmn to avoid materializing the
1897 : // -1 constant. For example,
1898 : // movz w1, #65535
1899 : // ldrh w0, [x0, #0]
1900 : // cmp w0, w1
1901 : // >
1902 : // ldrsh w0, [x0, #0]
1903 : // cmn w0, #1
1904 : // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
1905 : // if and only if (sext LHS) == (sext RHS). The checks are in place to
1906 : // ensure both the LHS and RHS are truly zero extended and to make sure the
1907 : // transformation is profitable.
1908 481 : if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
1909 : cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
1910 245 : cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
1911 2 : LHS.getNode()->hasNUsesOfValue(1, 0)) {
1912 2 : int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
1913 2 : if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
1914 : SDValue SExt =
1915 : DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
1916 2 : DAG.getValueType(MVT::i16));
1917 2 : Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
1918 : RHS.getValueType()),
1919 2 : CC, dl, DAG);
1920 2 : AArch64CC = changeIntCCToAArch64CC(CC);
1921 : }
1922 : }
1923 :
1924 605 : if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
1925 145 : if ((Cmp = emitConjunctionDisjunctionTree(DAG, LHS, AArch64CC))) {
1926 44 : if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
1927 2 : AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
1928 : }
1929 : }
1930 : }
1931 :
1932 886 : if (!Cmp) {
1933 862 : Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
1934 862 : AArch64CC = changeIntCCToAArch64CC(CC);
1935 : }
1936 886 : AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
1937 886 : return Cmp;
1938 : }
1939 :
1940 : static std::pair<SDValue, SDValue>
1941 0 : getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
1942 : assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
1943 : "Unsupported value type");
1944 : SDValue Value, Overflow;
1945 0 : SDLoc DL(Op);
1946 0 : SDValue LHS = Op.getOperand(0);
1947 0 : SDValue RHS = Op.getOperand(1);
1948 : unsigned Opc = 0;
1949 0 : switch (Op.getOpcode()) {
1950 0 : default:
1951 0 : llvm_unreachable("Unknown overflow instruction!");
1952 0 : case ISD::SADDO:
1953 : Opc = AArch64ISD::ADDS;
1954 0 : CC = AArch64CC::VS;
1955 : break;
1956 0 : case ISD::UADDO:
1957 : Opc = AArch64ISD::ADDS;
1958 0 : CC = AArch64CC::HS;
1959 : break;
1960 0 : case ISD::SSUBO:
1961 : Opc = AArch64ISD::SUBS;
1962 0 : CC = AArch64CC::VS;
1963 : break;
1964 0 : case ISD::USUBO:
1965 : Opc = AArch64ISD::SUBS;
1966 0 : CC = AArch64CC::LO;
1967 : break;
1968 : // Multiply needs a little bit extra work.
1969 0 : case ISD::SMULO:
1970 : case ISD::UMULO: {
1971 0 : CC = AArch64CC::NE;
1972 : bool IsSigned = Op.getOpcode() == ISD::SMULO;
1973 0 : if (Op.getValueType() == MVT::i32) {
1974 0 : unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
1975 : // For a 32 bit multiply with overflow check we want the instruction
1976 : // selector to generate a widening multiply (SMADDL/UMADDL). For that we
1977 : // need to generate the following pattern:
1978 : // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
1979 0 : LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
1980 0 : RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
1981 0 : SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
1982 : SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
1983 0 : DAG.getConstant(0, DL, MVT::i64));
1984 : // On AArch64 the upper 32 bits are always zero extended for a 32 bit
1985 : // operation. We need to clear out the upper 32 bits, because we used a
1986 : // widening multiply that wrote all 64 bits. In the end this should be a
1987 : // noop.
1988 0 : Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
1989 0 : if (IsSigned) {
1990 : // The signed overflow check requires more than just a simple check for
1991 : // any bit set in the upper 32 bits of the result. These bits could be
1992 : // just the sign bits of a negative number. To perform the overflow
1993 : // check we have to arithmetic shift right the 32nd bit of the result by
1994 : // 31 bits. Then we compare the result to the upper 32 bits.
1995 : SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
1996 0 : DAG.getConstant(32, DL, MVT::i64));
1997 0 : UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
1998 : SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
1999 0 : DAG.getConstant(31, DL, MVT::i64));
2000 : // It is important that LowerBits is last, otherwise the arithmetic
2001 : // shift will not be folded into the compare (SUBS).
2002 0 : SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2003 0 : Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2004 0 : .getValue(1);
2005 : } else {
2006 : // The overflow check for unsigned multiply is easy. We only need to
2007 : // check if any of the upper 32 bits are set. This can be done with a
2008 : // CMP (shifted register). For that we need to generate the following
2009 : // pattern:
2010 : // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2011 : SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2012 0 : DAG.getConstant(32, DL, MVT::i64));
2013 0 : SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2014 : Overflow =
2015 0 : DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2016 : DAG.getConstant(0, DL, MVT::i64),
2017 0 : UpperBits).getValue(1);
2018 : }
2019 : break;
2020 : }
2021 : assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
2022 : // For the 64 bit multiply
2023 0 : Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2024 0 : if (IsSigned) {
2025 0 : SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2026 : SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2027 0 : DAG.getConstant(63, DL, MVT::i64));
2028 : // It is important that LowerBits is last, otherwise the arithmetic
2029 : // shift will not be folded into the compare (SUBS).
2030 0 : SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2031 0 : Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2032 0 : .getValue(1);
2033 : } else {
2034 0 : SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2035 0 : SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2036 : Overflow =
2037 0 : DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2038 : DAG.getConstant(0, DL, MVT::i64),
2039 0 : UpperBits).getValue(1);
2040 : }
2041 : break;
2042 : }
2043 : } // switch (...)
2044 :
2045 : if (Opc) {
2046 0 : SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2047 :
2048 : // Emit the AArch64 operation with overflow check.
2049 0 : Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2050 0 : Overflow = Value.getValue(1);
2051 : }
2052 0 : return std::make_pair(Value, Overflow);
2053 : }
2054 :
2055 93 : SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2056 : RTLIB::Libcall Call) const {
2057 93 : SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2058 93 : return makeLibCall(DAG, Call, MVT::f128, Ops, false, SDLoc(Op)).first;
2059 : }
2060 :
2061 : // Returns true if the given Op is the overflow flag result of an overflow
2062 : // intrinsic operation.
2063 0 : static bool isOverflowIntrOpRes(SDValue Op) {
2064 0 : unsigned Opc = Op.getOpcode();
2065 1377 : return (Op.getResNo() == 1 &&
2066 : (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2067 57 : Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2068 : }
2069 :
2070 951 : static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2071 951 : SDValue Sel = Op.getOperand(0);
2072 951 : SDValue Other = Op.getOperand(1);
2073 : SDLoc dl(Sel);
2074 :
2075 : // If the operand is an overflow checking operation, invert the condition
2076 : // code and kill the Not operation. I.e., transform:
2077 : // (xor (overflow_op_bool, 1))
2078 : // -->
2079 : // (csel 1, 0, invert(cc), overflow_op_bool)
2080 : // ... which later gets transformed to just a cset instruction with an
2081 : // inverted condition code, rather than a cset + eor sequence.
2082 951 : if (isOneConstant(Other) && isOverflowIntrOpRes(Sel)) {
2083 : // Only lower legal XALUO ops.
2084 24 : if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
2085 0 : return SDValue();
2086 :
2087 24 : SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2088 24 : SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2089 : AArch64CC::CondCode CC;
2090 : SDValue Value, Overflow;
2091 24 : std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
2092 48 : SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2093 : return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
2094 24 : CCVal, Overflow);
2095 : }
2096 : // If neither operand is a SELECT_CC, give up.
2097 927 : if (Sel.getOpcode() != ISD::SELECT_CC)
2098 : std::swap(Sel, Other);
2099 927 : if (Sel.getOpcode() != ISD::SELECT_CC)
2100 926 : return Op;
2101 :
2102 : // The folding we want to perform is:
2103 : // (xor x, (select_cc a, b, cc, 0, -1) )
2104 : // -->
2105 : // (csel x, (xor x, -1), cc ...)
2106 : //
2107 : // The latter will get matched to a CSINV instruction.
2108 :
2109 1 : ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
2110 1 : SDValue LHS = Sel.getOperand(0);
2111 1 : SDValue RHS = Sel.getOperand(1);
2112 1 : SDValue TVal = Sel.getOperand(2);
2113 1 : SDValue FVal = Sel.getOperand(3);
2114 :
2115 : // FIXME: This could be generalized to non-integer comparisons.
2116 : if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
2117 0 : return Op;
2118 :
2119 : ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
2120 : ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
2121 :
2122 : // The values aren't constants, this isn't the pattern we're looking for.
2123 1 : if (!CFVal || !CTVal)
2124 0 : return Op;
2125 :
2126 : // We can commute the SELECT_CC by inverting the condition. This
2127 : // might be needed to make this fit into a CSINV pattern.
2128 3 : if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
2129 : std::swap(TVal, FVal);
2130 : std::swap(CTVal, CFVal);
2131 1 : CC = ISD::getSetCCInverse(CC, true);
2132 : }
2133 :
2134 : // If the constants line up, perform the transform!
2135 3 : if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
2136 1 : SDValue CCVal;
2137 1 : SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2138 :
2139 1 : FVal = Other;
2140 1 : TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
2141 1 : DAG.getConstant(-1ULL, dl, Other.getValueType()));
2142 :
2143 : return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
2144 1 : CCVal, Cmp);
2145 : }
2146 :
2147 0 : return Op;
2148 : }
2149 :
2150 0 : static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2151 0 : EVT VT = Op.getValueType();
2152 :
2153 : // Let legalize expand this if it isn't a legal type yet.
2154 0 : if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
2155 0 : return SDValue();
2156 :
2157 0 : SDVTList VTs = DAG.getVTList(VT, MVT::i32);
2158 :
2159 : unsigned Opc;
2160 : bool ExtraOp = false;
2161 0 : switch (Op.getOpcode()) {
2162 0 : default:
2163 0 : llvm_unreachable("Invalid code");
2164 : case ISD::ADDC:
2165 : Opc = AArch64ISD::ADDS;
2166 : break;
2167 0 : case ISD::SUBC:
2168 : Opc = AArch64ISD::SUBS;
2169 0 : break;
2170 0 : case ISD::ADDE:
2171 : Opc = AArch64ISD::ADCS;
2172 : ExtraOp = true;
2173 0 : break;
2174 0 : case ISD::SUBE:
2175 : Opc = AArch64ISD::SBCS;
2176 : ExtraOp = true;
2177 0 : break;
2178 : }
2179 :
2180 0 : if (!ExtraOp)
2181 0 : return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
2182 0 : return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
2183 0 : Op.getOperand(2));
2184 : }
2185 :
2186 0 : static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2187 : // Let legalize expand this if it isn't a legal type yet.
2188 0 : if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
2189 0 : return SDValue();
2190 :
2191 0 : SDLoc dl(Op);
2192 : AArch64CC::CondCode CC;
2193 : // The actual operation that sets the overflow or carry flag.
2194 : SDValue Value, Overflow;
2195 0 : std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
2196 :
2197 : // We use 0 and 1 as false and true values.
2198 0 : SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2199 0 : SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2200 :
2201 : // We use an inverted condition, because the conditional select is inverted
2202 : // too. This will allow it to be selected to a single instruction:
2203 : // CSINC Wd, WZR, WZR, invert(cond).
2204 0 : SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2205 0 : Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
2206 0 : CCVal, Overflow);
2207 :
2208 0 : SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
2209 0 : return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
2210 : }
2211 :
2212 : // Prefetch operands are:
2213 : // 1: Address to prefetch
2214 : // 2: bool isWrite
2215 : // 3: int locality (0 = no locality ... 3 = extreme locality)
2216 : // 4: bool isDataCache
2217 0 : static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2218 0 : SDLoc DL(Op);
2219 0 : unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
2220 0 : unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
2221 0 : unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
2222 :
2223 0 : bool IsStream = !Locality;
2224 : // When the locality number is set
2225 0 : if (Locality) {
2226 : // The front-end should have filtered out the out-of-range values
2227 : assert(Locality <= 3 && "Prefetch locality out-of-range");
2228 : // The locality degree is the opposite of the cache speed.
2229 : // Put the number the other way around.
2230 : // The encoding starts at 0 for level 1
2231 0 : Locality = 3 - Locality;
2232 : }
2233 :
2234 : // built the mask value encoding the expected behavior.
2235 0 : unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
2236 0 : (!IsData << 3) | // IsDataCache bit
2237 0 : (Locality << 1) | // Cache level bits
2238 0 : (unsigned)IsStream; // Stream bit
2239 : return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
2240 0 : DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
2241 : }
2242 :
2243 2 : SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2244 : SelectionDAG &DAG) const {
2245 : assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
2246 :
2247 : RTLIB::Libcall LC;
2248 6 : LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
2249 :
2250 2 : return LowerF128Call(Op, DAG, LC);
2251 : }
2252 :
2253 36 : SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2254 : SelectionDAG &DAG) const {
2255 36 : if (Op.getOperand(0).getValueType() != MVT::f128) {
2256 : // It's legal except when f128 is involved
2257 31 : return Op;
2258 : }
2259 :
2260 : RTLIB::Libcall LC;
2261 5 : LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
2262 :
2263 : // FP_ROUND node has a second operand indicating whether it is known to be
2264 : // precise. That doesn't take part in the LibCall so we can't directly use
2265 : // LowerF128Call.
2266 5 : SDValue SrcVal = Op.getOperand(0);
2267 5 : return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
2268 5 : SDLoc(Op)).first;
2269 : }
2270 :
2271 240 : static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
2272 : // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2273 : // Any additional optimization in this function should be recorded
2274 : // in the cost tables.
2275 480 : EVT InVT = Op.getOperand(0).getValueType();
2276 240 : EVT VT = Op.getValueType();
2277 : unsigned NumElts = InVT.getVectorNumElements();
2278 :
2279 : // f16 vectors are promoted to f32 before a conversion.
2280 256 : if (InVT.getVectorElementType() == MVT::f16) {
2281 16 : MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
2282 : SDLoc dl(Op);
2283 : return DAG.getNode(
2284 : Op.getOpcode(), dl, Op.getValueType(),
2285 16 : DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
2286 : }
2287 :
2288 224 : if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2289 : SDLoc dl(Op);
2290 : SDValue Cv =
2291 : DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
2292 36 : Op.getOperand(0));
2293 36 : return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
2294 : }
2295 :
2296 188 : if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2297 : SDLoc dl(Op);
2298 : MVT ExtVT =
2299 : MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
2300 3 : VT.getVectorNumElements());
2301 3 : SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
2302 3 : return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
2303 : }
2304 :
2305 : // Type changing conversions are illegal.
2306 185 : return Op;
2307 : }
2308 :
2309 461 : SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2310 : SelectionDAG &DAG) const {
2311 1383 : if (Op.getOperand(0).getValueType().isVector())
2312 240 : return LowerVectorFP_TO_INT(Op, DAG);
2313 :
2314 : // f16 conversions are promoted to f32 when full fp16 is not supported.
2315 23 : if (Op.getOperand(0).getValueType() == MVT::f16 &&
2316 23 : !Subtarget->hasFullFP16()) {
2317 : SDLoc dl(Op);
2318 : return DAG.getNode(
2319 : Op.getOpcode(), dl, Op.getValueType(),
2320 7 : DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0)));
2321 : }
2322 :
2323 : if (Op.getOperand(0).getValueType() != MVT::f128) {
2324 : // It's legal except when f128 is involved
2325 202 : return Op;
2326 : }
2327 :
2328 : RTLIB::Libcall LC;
2329 12 : if (Op.getOpcode() == ISD::FP_TO_SINT)
2330 5 : LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType());
2331 : else
2332 7 : LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType());
2333 :
2334 12 : SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2335 12 : return makeLibCall(DAG, LC, Op.getValueType(), Ops, false, SDLoc(Op)).first;
2336 : }
2337 :
2338 431 : static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2339 : // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2340 : // Any additional optimization in this function should be recorded
2341 : // in the cost tables.
2342 431 : EVT VT = Op.getValueType();
2343 : SDLoc dl(Op);
2344 431 : SDValue In = Op.getOperand(0);
2345 431 : EVT InVT = In.getValueType();
2346 :
2347 431 : if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2348 : MVT CastVT =
2349 : MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
2350 90 : InVT.getVectorNumElements());
2351 90 : In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
2352 90 : return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
2353 : }
2354 :
2355 341 : if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2356 : unsigned CastOpc =
2357 9 : Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2358 9 : EVT CastVT = VT.changeVectorElementTypeToInteger();
2359 9 : In = DAG.getNode(CastOpc, dl, CastVT, In);
2360 9 : return DAG.getNode(Op.getOpcode(), dl, VT, In);
2361 : }
2362 :
2363 332 : return Op;
2364 : }
2365 :
2366 731 : SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2367 : SelectionDAG &DAG) const {
2368 1462 : if (Op.getValueType().isVector())
2369 431 : return LowerVectorINT_TO_FP(Op, DAG);
2370 :
2371 : // f16 conversions are promoted to f32 when full fp16 is not supported.
2372 34 : if (Op.getValueType() == MVT::f16 &&
2373 34 : !Subtarget->hasFullFP16()) {
2374 : SDLoc dl(Op);
2375 : return DAG.getNode(
2376 : ISD::FP_ROUND, dl, MVT::f16,
2377 : DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)),
2378 16 : DAG.getIntPtrConstant(0, dl));
2379 : }
2380 :
2381 : // i128 conversions are libcalls.
2382 284 : if (Op.getOperand(0).getValueType() == MVT::i128)
2383 6 : return SDValue();
2384 :
2385 : // Other conversions are legal, unless it's to the completely software-based
2386 : // fp128.
2387 : if (Op.getValueType() != MVT::f128)
2388 272 : return Op;
2389 :
2390 : RTLIB::Libcall LC;
2391 6 : if (Op.getOpcode() == ISD::SINT_TO_FP)
2392 2 : LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2393 : else
2394 4 : LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2395 :
2396 6 : return LowerF128Call(Op, DAG, LC);
2397 : }
2398 :
2399 2 : SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2400 : SelectionDAG &DAG) const {
2401 : // For iOS, we want to call an alternative entry point: __sincos_stret,
2402 : // which returns the values in two S / D registers.
2403 : SDLoc dl(Op);
2404 2 : SDValue Arg = Op.getOperand(0);
2405 2 : EVT ArgVT = Arg.getValueType();
2406 2 : Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2407 :
2408 : ArgListTy Args;
2409 : ArgListEntry Entry;
2410 :
2411 2 : Entry.Node = Arg;
2412 2 : Entry.Ty = ArgTy;
2413 : Entry.IsSExt = false;
2414 : Entry.IsZExt = false;
2415 2 : Args.push_back(Entry);
2416 :
2417 : RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
2418 : : RTLIB::SINCOS_STRET_F32;
2419 : const char *LibcallName = getLibcallName(LC);
2420 : SDValue Callee =
2421 4 : DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
2422 :
2423 2 : StructType *RetTy = StructType::get(ArgTy, ArgTy);
2424 2 : TargetLowering::CallLoweringInfo CLI(DAG);
2425 : CLI.setDebugLoc(dl)
2426 2 : .setChain(DAG.getEntryNode())
2427 2 : .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
2428 :
2429 2 : std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2430 2 : return CallResult.first;
2431 : }
2432 :
2433 8 : static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2434 : if (Op.getValueType() != MVT::f16)
2435 0 : return SDValue();
2436 :
2437 : assert(Op.getOperand(0).getValueType() == MVT::i16);
2438 : SDLoc DL(Op);
2439 :
2440 8 : Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
2441 8 : Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
2442 : return SDValue(
2443 16 : DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op,
2444 : DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
2445 8 : 0);
2446 : }
2447 :
2448 0 : static EVT getExtensionTo64Bits(const EVT &OrigVT) {
2449 0 : if (OrigVT.getSizeInBits() >= 64)
2450 0 : return OrigVT;
2451 :
2452 : assert(OrigVT.isSimple() && "Expecting a simple value type");
2453 :
2454 : MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
2455 0 : switch (OrigSimpleTy) {
2456 0 : default: llvm_unreachable("Unexpected Vector Type");
2457 : case MVT::v2i8:
2458 : case MVT::v2i16:
2459 0 : return MVT::v2i32;
2460 : case MVT::v4i8:
2461 0 : return MVT::v4i16;
2462 : }
2463 : }
2464 :
2465 0 : static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2466 : const EVT &OrigTy,
2467 : const EVT &ExtTy,
2468 : unsigned ExtOpcode) {
2469 : // The vector originally had a size of OrigTy. It was then extended to ExtTy.
2470 : // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
2471 : // 64-bits we need to insert a new extension so that it will be 64-bits.
2472 : assert(ExtTy.is128BitVector() && "Unexpected extension size");
2473 0 : if (OrigTy.getSizeInBits() >= 64)
2474 0 : return N;
2475 :
2476 : // Must extend size to at least 64 bits to be used as an operand for VMULL.
2477 0 : EVT NewVT = getExtensionTo64Bits(OrigTy);
2478 :
2479 0 : return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
2480 : }
2481 :
2482 1718 : static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
2483 : bool isSigned) {
2484 1718 : EVT VT = N->getValueType(0);
2485 :
2486 1718 : if (N->getOpcode() != ISD::BUILD_VECTOR)
2487 : return false;
2488 :
2489 355 : for (const SDValue &Elt : N->op_values()) {
2490 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
2491 : unsigned EltSize = VT.getScalarSizeInBits();
2492 264 : unsigned HalfSize = EltSize / 2;
2493 264 : if (isSigned) {
2494 132 : if (!isIntN(HalfSize, C->getSExtValue()))
2495 : return false;
2496 : } else {
2497 132 : if (!isUIntN(HalfSize, C->getZExtValue()))
2498 : return false;
2499 : }
2500 : continue;
2501 : }
2502 : return false;
2503 : }
2504 :
2505 : return true;
2506 : }
2507 :
2508 51 : static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
2509 102 : if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
2510 : return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG,
2511 90 : N->getOperand(0)->getValueType(0),
2512 : N->getValueType(0),
2513 45 : N->getOpcode());
2514 :
2515 : assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR");
2516 12 : EVT VT = N->getValueType(0);
2517 : SDLoc dl(N);
2518 6 : unsigned EltSize = VT.getScalarSizeInBits() / 2;
2519 : unsigned NumElts = VT.getVectorNumElements();
2520 6 : MVT TruncVT = MVT::getIntegerVT(EltSize);
2521 : SmallVector<SDValue, 8> Ops;
2522 34 : for (unsigned i = 0; i != NumElts; ++i) {
2523 28 : ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
2524 28 : const APInt &CInt = C->getAPIntValue();
2525 : // Element types smaller than 32 bits are not legal, so use i32 elements.
2526 : // The values are implicitly truncated so sext vs. zext doesn't matter.
2527 28 : Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
2528 : }
2529 6 : return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
2530 : }
2531 :
2532 : static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
2533 827 : return N->getOpcode() == ISD::SIGN_EXTEND ||
2534 805 : isExtendedBUILD_VECTOR(N, DAG, true);
2535 : }
2536 :
2537 : static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
2538 1190 : return N->getOpcode() == ISD::ZERO_EXTEND ||
2539 783 : isExtendedBUILD_VECTOR(N, DAG, false);
2540 : }
2541 :
2542 24 : static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
2543 24 : unsigned Opcode = N->getOpcode();
2544 24 : if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2545 4 : SDNode *N0 = N->getOperand(0).getNode();
2546 4 : SDNode *N1 = N->getOperand(1).getNode();
2547 : return N0->hasOneUse() && N1->hasOneUse() &&
2548 : isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
2549 : }
2550 : return false;
2551 : }
2552 :
2553 27 : static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
2554 27 : unsigned Opcode = N->getOpcode();
2555 27 : if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2556 5 : SDNode *N0 = N->getOperand(0).getNode();
2557 5 : SDNode *N1 = N->getOperand(1).getNode();
2558 : return N0->hasOneUse() && N1->hasOneUse() &&
2559 : isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
2560 : }
2561 : return false;
2562 : }
2563 :
2564 1 : SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
2565 : SelectionDAG &DAG) const {
2566 : // The rounding mode is in bits 23:22 of the FPSCR.
2567 : // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
2568 : // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
2569 : // so that the shift + and get folded into a bitfield extract.
2570 : SDLoc dl(Op);
2571 :
2572 : SDValue FPCR_64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i64,
2573 : DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl,
2574 1 : MVT::i64));
2575 1 : SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
2576 : SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
2577 1 : DAG.getConstant(1U << 22, dl, MVT::i32));
2578 : SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
2579 1 : DAG.getConstant(22, dl, MVT::i32));
2580 : return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
2581 1 : DAG.getConstant(3, dl, MVT::i32));
2582 : }
2583 :
2584 414 : static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2585 : // Multiplications are only custom-lowered for 128-bit vectors so that
2586 : // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
2587 414 : EVT VT = Op.getValueType();
2588 : assert(VT.is128BitVector() && VT.isInteger() &&
2589 : "unexpected type for custom-lowering ISD::MUL");
2590 414 : SDNode *N0 = Op.getOperand(0).getNode();
2591 414 : SDNode *N1 = Op.getOperand(1).getNode();
2592 : unsigned NewOpc = 0;
2593 : bool isMLA = false;
2594 : bool isN0SExt = isSignExtended(N0, DAG);
2595 : bool isN1SExt = isSignExtended(N1, DAG);
2596 414 : if (isN0SExt && isN1SExt)
2597 : NewOpc = AArch64ISD::SMULL;
2598 : else {
2599 : bool isN0ZExt = isZeroExtended(N0, DAG);
2600 : bool isN1ZExt = isZeroExtended(N1, DAG);
2601 402 : if (isN0ZExt && isN1ZExt)
2602 : NewOpc = AArch64ISD::UMULL;
2603 390 : else if (isN1SExt || isN1ZExt) {
2604 : // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
2605 : // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
2606 27 : if (isN1SExt && isAddSubSExt(N0, DAG)) {
2607 : NewOpc = AArch64ISD::SMULL;
2608 : isMLA = true;
2609 27 : } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
2610 : NewOpc = AArch64ISD::UMULL;
2611 : isMLA = true;
2612 26 : } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
2613 : std::swap(N0, N1);
2614 : NewOpc = AArch64ISD::UMULL;
2615 : isMLA = true;
2616 : }
2617 : }
2618 :
2619 402 : if (!NewOpc) {
2620 : if (VT == MVT::v2i64)
2621 : // Fall through to expand this. It is not legal.
2622 2 : return SDValue();
2623 : else
2624 : // Other vector multiplications are legal.
2625 387 : return Op;
2626 : }
2627 : }
2628 :
2629 : // Legalize to a S/UMULL instruction
2630 : SDLoc DL(Op);
2631 25 : SDValue Op0;
2632 25 : SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
2633 25 : if (!isMLA) {
2634 24 : Op0 = skipExtensionForVectorMULL(N0, DAG);
2635 : assert(Op0.getValueType().is64BitVector() &&
2636 : Op1.getValueType().is64BitVector() &&
2637 : "unexpected types for extended operands to VMULL");
2638 24 : return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
2639 : }
2640 : // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
2641 : // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
2642 : // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
2643 1 : SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
2644 1 : SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
2645 2 : EVT Op1VT = Op1.getValueType();
2646 : return DAG.getNode(N0->getOpcode(), DL, VT,
2647 : DAG.getNode(NewOpc, DL, VT,
2648 : DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
2649 : DAG.getNode(NewOpc, DL, VT,
2650 1 : DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
2651 : }
2652 :
2653 : // Lower vector multiply high (ISD::MULHS and ISD::MULHU).
2654 0 : static SDValue LowerMULH(SDValue Op, SelectionDAG &DAG) {
2655 : // Multiplications are only custom-lowered for 128-bit vectors so that
2656 : // {S,U}MULL{2} can be detected. Otherwise v2i64 multiplications are not
2657 : // legal.
2658 0 : EVT VT = Op.getValueType();
2659 : assert(VT.is128BitVector() && VT.isInteger() &&
2660 : "unexpected type for custom-lowering ISD::MULH{U,S}");
2661 :
2662 0 : SDValue V0 = Op.getOperand(0);
2663 0 : SDValue V1 = Op.getOperand(1);
2664 :
2665 0 : SDLoc DL(Op);
2666 :
2667 0 : EVT ExtractVT = VT.getHalfNumVectorElementsVT(*DAG.getContext());
2668 :
2669 : // We turn (V0 mulhs/mulhu V1) to:
2670 : //
2671 : // (uzp2 (smull (extract_subvector (ExtractVT V128:V0, (i64 0)),
2672 : // (extract_subvector (ExtractVT V128:V1, (i64 0))))),
2673 : // (smull (extract_subvector (ExtractVT V128:V0, (i64 VMull2Idx)),
2674 : // (extract_subvector (ExtractVT V128:V2, (i64 VMull2Idx))))))
2675 : //
2676 : // Where ExtractVT is a subvector with half number of elements, and
2677 : // VMullIdx2 is the index of the middle element (the high part).
2678 : //
2679 : // The vector hight part extract and multiply will be matched against
2680 : // {S,U}MULL{v16i8_v8i16,v8i16_v4i32,v4i32_v2i64} which in turn will
2681 : // issue a {s}mull2 instruction.
2682 : //
2683 : // This basically multiply the lower subvector with '{s,u}mull', the high
2684 : // subvector with '{s,u}mull2', and shuffle both results high part in
2685 : // resulting vector.
2686 0 : unsigned Mull2VectorIdx = VT.getVectorNumElements () / 2;
2687 0 : SDValue VMullIdx = DAG.getConstant(0, DL, MVT::i64);
2688 0 : SDValue VMull2Idx = DAG.getConstant(Mull2VectorIdx, DL, MVT::i64);
2689 :
2690 : SDValue VMullV0 =
2691 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, V0, VMullIdx);
2692 : SDValue VMullV1 =
2693 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, V1, VMullIdx);
2694 :
2695 : SDValue VMull2V0 =
2696 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, V0, VMull2Idx);
2697 : SDValue VMull2V1 =
2698 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT, V1, VMull2Idx);
2699 :
2700 0 : unsigned MullOpc = Op.getOpcode() == ISD::MULHS ? AArch64ISD::SMULL
2701 : : AArch64ISD::UMULL;
2702 :
2703 0 : EVT MullVT = ExtractVT.widenIntegerVectorElementType(*DAG.getContext());
2704 0 : SDValue Mull = DAG.getNode(MullOpc, DL, MullVT, VMullV0, VMullV1);
2705 0 : SDValue Mull2 = DAG.getNode(MullOpc, DL, MullVT, VMull2V0, VMull2V1);
2706 :
2707 0 : Mull = DAG.getNode(ISD::BITCAST, DL, VT, Mull);
2708 0 : Mull2 = DAG.getNode(ISD::BITCAST, DL, VT, Mull2);
2709 :
2710 0 : return DAG.getNode(AArch64ISD::UZP2, DL, VT, Mull, Mull2);
2711 : }
2712 :
2713 5727 : SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2714 : SelectionDAG &DAG) const {
2715 11454 : unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2716 : SDLoc dl(Op);
2717 5727 : switch (IntNo) {
2718 5640 : default: return SDValue(); // Don't custom lower most intrinsics.
2719 6 : case Intrinsic::thread_pointer: {
2720 6 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
2721 6 : return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
2722 : }
2723 : case Intrinsic::aarch64_neon_abs:
2724 : return DAG.getNode(ISD::ABS, dl, Op.getValueType(),
2725 8 : Op.getOperand(1));
2726 : case Intrinsic::aarch64_neon_smax:
2727 : return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
2728 19 : Op.getOperand(1), Op.getOperand(2));
2729 : case Intrinsic::aarch64_neon_umax:
2730 : return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
2731 18 : Op.getOperand(1), Op.getOperand(2));
2732 : case Intrinsic::aarch64_neon_smin:
2733 : return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
2734 18 : Op.getOperand(1), Op.getOperand(2));
2735 : case Intrinsic::aarch64_neon_umin:
2736 : return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
2737 18 : Op.getOperand(1), Op.getOperand(2));
2738 : }
2739 : }
2740 :
2741 : // Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
2742 0 : static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
2743 : EVT VT, EVT MemVT,
2744 : SelectionDAG &DAG) {
2745 : assert(VT.isVector() && "VT should be a vector type");
2746 : assert(MemVT == MVT::v4i8 && VT == MVT::v4i16);
2747 :
2748 0 : SDValue Value = ST->getValue();
2749 :
2750 : // It first extend the promoted v4i16 to v8i16, truncate to v8i8, and extract
2751 : // the word lane which represent the v4i8 subvector. It optimizes the store
2752 : // to:
2753 : //
2754 : // xtn v0.8b, v0.8h
2755 : // str s0, [x0]
2756 :
2757 0 : SDValue Undef = DAG.getUNDEF(MVT::i16);
2758 : SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
2759 0 : {Undef, Undef, Undef, Undef});
2760 :
2761 : SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
2762 0 : Value, UndefVec);
2763 0 : SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
2764 :
2765 0 : Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
2766 : SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
2767 0 : Trunc, DAG.getConstant(0, DL, MVT::i64));
2768 :
2769 : return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
2770 0 : ST->getBasePtr(), ST->getMemOperand());
2771 : }
2772 :
2773 : // Custom lowering for any store, vector or scalar and/or default or with
2774 : // a truncate operations. Currently only custom lower truncate operation
2775 : // from vector v4i16 to v4i8.
2776 1 : SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
2777 : SelectionDAG &DAG) const {
2778 : SDLoc Dl(Op);
2779 : StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
2780 : assert (StoreNode && "Can only custom lower store nodes");
2781 :
2782 : SDValue Value = StoreNode->getValue();
2783 :
2784 : EVT VT = Value.getValueType();
2785 1 : EVT MemVT = StoreNode->getMemoryVT();
2786 :
2787 : assert (VT.isVector() && "Can only custom lower vector store types");
2788 :
2789 : unsigned AS = StoreNode->getAddressSpace();
2790 1 : unsigned Align = StoreNode->getAlignment();
2791 1 : if (Align < MemVT.getStoreSize() &&
2792 0 : !allowsMisalignedMemoryAccesses(MemVT, AS, Align, nullptr)) {
2793 0 : return scalarizeVectorStore(StoreNode, DAG);
2794 : }
2795 :
2796 1 : if (StoreNode->isTruncatingStore()) {
2797 2 : return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
2798 : }
2799 :
2800 0 : return SDValue();
2801 : }
2802 :
2803 29832 : SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
2804 : SelectionDAG &DAG) const {
2805 : LLVM_DEBUG(dbgs() << "Custom lowering: ");
2806 : LLVM_DEBUG(Op.dump());
2807 :
2808 29832 : switch (Op.getOpcode()) {
2809 0 : default:
2810 0 : llvm_unreachable("unimplemented operand");
2811 : return SDValue();
2812 8 : case ISD::BITCAST:
2813 8 : return LowerBITCAST(Op, DAG);
2814 3117 : case ISD::GlobalAddress:
2815 3117 : return LowerGlobalAddress(Op, DAG);
2816 121 : case ISD::GlobalTLSAddress:
2817 121 : return LowerGlobalTLSAddress(Op, DAG);
2818 816 : case ISD::SETCC:
2819 816 : return LowerSETCC(Op, DAG);
2820 976 : case ISD::BR_CC:
2821 976 : return LowerBR_CC(Op, DAG);
2822 367 : case ISD::SELECT:
2823 367 : return LowerSELECT(Op, DAG);
2824 244 : case ISD::SELECT_CC:
2825 244 : return LowerSELECT_CC(Op, DAG);
2826 28 : case ISD::JumpTable:
2827 28 : return LowerJumpTable(Op, DAG);
2828 182 : case ISD::ConstantPool:
2829 182 : return LowerConstantPool(Op, DAG);
2830 7 : case ISD::BlockAddress:
2831 7 : return LowerBlockAddress(Op, DAG);
2832 25 : case ISD::VASTART:
2833 25 : return LowerVASTART(Op, DAG);
2834 2 : case ISD::VACOPY:
2835 2 : return LowerVACOPY(Op, DAG);
2836 16 : case ISD::VAARG:
2837 16 : return LowerVAARG(Op, DAG);
2838 48 : case ISD::ADDC:
2839 : case ISD::ADDE:
2840 : case ISD::SUBC:
2841 : case ISD::SUBE:
2842 48 : return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
2843 31 : case ISD::SADDO:
2844 : case ISD::UADDO:
2845 : case ISD::SSUBO:
2846 : case ISD::USUBO:
2847 : case ISD::SMULO:
2848 : case ISD::UMULO:
2849 31 : return LowerXALUO(Op, DAG);
2850 78 : case ISD::FADD:
2851 78 : return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
2852 2 : case ISD::FSUB:
2853 2 : return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
2854 4 : case ISD::FMUL:
2855 4 : return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
2856 1 : case ISD::FDIV:
2857 1 : return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
2858 36 : case ISD::FP_ROUND:
2859 36 : return LowerFP_ROUND(Op, DAG);
2860 2 : case ISD::FP_EXTEND:
2861 2 : return LowerFP_EXTEND(Op, DAG);
2862 2 : case ISD::FRAMEADDR:
2863 2 : return LowerFRAMEADDR(Op, DAG);
2864 6 : case ISD::RETURNADDR:
2865 6 : return LowerRETURNADDR(Op, DAG);
2866 2130 : case ISD::INSERT_VECTOR_ELT:
2867 2130 : return LowerINSERT_VECTOR_ELT(Op, DAG);
2868 5267 : case ISD::EXTRACT_VECTOR_ELT:
2869 5267 : return LowerEXTRACT_VECTOR_ELT(Op, DAG);
2870 2161 : case ISD::BUILD_VECTOR:
2871 2161 : return LowerBUILD_VECTOR(Op, DAG);
2872 1680 : case ISD::VECTOR_SHUFFLE:
2873 1680 : return LowerVECTOR_SHUFFLE(Op, DAG);
2874 1736 : case ISD::EXTRACT_SUBVECTOR:
2875 1736 : return LowerEXTRACT_SUBVECTOR(Op, DAG);
2876 280 : case ISD::SRA:
2877 : case ISD::SRL:
2878 : case ISD::SHL:
2879 280 : return LowerVectorSRA_SRL_SHL(Op, DAG);
2880 1 : case ISD::SHL_PARTS:
2881 1 : return LowerShiftLeftParts(Op, DAG);
2882 2 : case ISD::SRL_PARTS:
2883 : case ISD::SRA_PARTS:
2884 2 : return LowerShiftRightParts(Op, DAG);
2885 23 : case ISD::CTPOP:
2886 23 : return LowerCTPOP(Op, DAG);
2887 29 : case ISD::FCOPYSIGN:
2888 29 : return LowerFCOPYSIGN(Op, DAG);
2889 1006 : case ISD::AND:
2890 1006 : return LowerVectorAND(Op, DAG);
2891 688 : case ISD::OR:
2892 688 : return LowerVectorOR(Op, DAG);
2893 951 : case ISD::XOR:
2894 951 : return LowerXOR(Op, DAG);
2895 24 : case ISD::PREFETCH:
2896 24 : return LowerPREFETCH(Op, DAG);
2897 731 : case ISD::SINT_TO_FP:
2898 : case ISD::UINT_TO_FP:
2899 731 : return LowerINT_TO_FP(Op, DAG);
2900 461 : case ISD::FP_TO_SINT:
2901 : case ISD::FP_TO_UINT:
2902 461 : return LowerFP_TO_INT(Op, DAG);
2903 2 : case ISD::FSINCOS:
2904 2 : return LowerFSINCOS(Op, DAG);
2905 1 : case ISD::FLT_ROUNDS_:
2906 1 : return LowerFLT_ROUNDS_(Op, DAG);
2907 414 : case ISD::MUL:
2908 414 : return LowerMUL(Op, DAG);
2909 19 : case ISD::MULHS:
2910 : case ISD::MULHU:
2911 19 : return LowerMULH(Op, DAG);
2912 5727 : case ISD::INTRINSIC_WO_CHAIN:
2913 5727 : return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2914 1 : case ISD::STORE:
2915 1 : return LowerSTORE(Op, DAG);
2916 23 : case ISD::VECREDUCE_ADD:
2917 : case ISD::VECREDUCE_SMAX:
2918 : case ISD::VECREDUCE_SMIN:
2919 : case ISD::VECREDUCE_UMAX:
2920 : case ISD::VECREDUCE_UMIN:
2921 : case ISD::VECREDUCE_FMAX:
2922 : case ISD::VECREDUCE_FMIN:
2923 23 : return LowerVECREDUCE(Op, DAG);
2924 176 : case ISD::ATOMIC_LOAD_SUB:
2925 176 : return LowerATOMIC_LOAD_SUB(Op, DAG);
2926 176 : case ISD::ATOMIC_LOAD_AND:
2927 176 : return LowerATOMIC_LOAD_AND(Op, DAG);
2928 4 : case ISD::DYNAMIC_STACKALLOC:
2929 4 : return LowerDYNAMIC_STACKALLOC(Op, DAG);
2930 : }
2931 : }
2932 :
2933 : //===----------------------------------------------------------------------===//
2934 : // Calling Convention Implementation
2935 : //===----------------------------------------------------------------------===//
2936 :
2937 : #include "AArch64GenCallingConv.inc"
2938 :
2939 : /// Selects the correct CCAssignFn for a given CallingConvention value.
2940 29608 : CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
2941 : bool IsVarArg) const {
2942 29608 : switch (CC) {
2943 0 : default:
2944 0 : report_fatal_error("Unsupported calling convention.");
2945 : case CallingConv::WebKit_JS:
2946 : return CC_AArch64_WebKit_JS;
2947 9 : case CallingConv::GHC:
2948 9 : return CC_AArch64_GHC;
2949 29468 : case CallingConv::C:
2950 : case CallingConv::Fast:
2951 : case CallingConv::PreserveMost:
2952 : case CallingConv::CXX_FAST_TLS:
2953 : case CallingConv::Swift:
2954 58936 : if (Subtarget->isTargetWindows() && IsVarArg)
2955 : return CC_AArch64_Win64_VarArg;
2956 : if (!Subtarget->isTargetDarwin())
2957 : return CC_AArch64_AAPCS;
2958 6421 : return IsVarArg ? CC_AArch64_DarwinPCS_VarArg : CC_AArch64_DarwinPCS;
2959 25 : case CallingConv::Win64:
2960 25 : return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
2961 0 : case CallingConv::AArch64_VectorCall:
2962 0 : return CC_AArch64_AAPCS;
2963 : }
2964 : }
2965 :
2966 : CCAssignFn *
2967 225 : AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
2968 : return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
2969 225 : : RetCC_AArch64_AAPCS;
2970 : }
2971 :
2972 13604 : SDValue AArch64TargetLowering::LowerFormalArguments(
2973 : SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
2974 : const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
2975 : SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
2976 13604 : MachineFunction &MF = DAG.getMachineFunction();
2977 13604 : MachineFrameInfo &MFI = MF.getFrameInfo();
2978 13604 : bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
2979 :
2980 : // Assign locations to all of the incoming arguments.
2981 : SmallVector<CCValAssign, 16> ArgLocs;
2982 : CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2983 27208 : *DAG.getContext());
2984 :
2985 : // At this point, Ins[].VT may already be promoted to i32. To correctly
2986 : // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
2987 : // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
2988 : // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
2989 : // we use a special version of AnalyzeFormalArguments to pass in ValVT and
2990 : // LocVT.
2991 13604 : unsigned NumArgs = Ins.size();
2992 13604 : Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
2993 : unsigned CurArgIdx = 0;
2994 38365 : for (unsigned i = 0; i != NumArgs; ++i) {
2995 24761 : MVT ValVT = Ins[i].VT;
2996 24761 : if (Ins[i].isOrigArg()) {
2997 24755 : std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
2998 : CurArgIdx = Ins[i].getOrigArgIndex();
2999 :
3000 : // Get type of the original argument.
3001 : EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
3002 24755 : /*AllowUnknown*/ true);
3003 24755 : MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
3004 : // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3005 24755 : if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3006 671 : ValVT = MVT::i8;
3007 24084 : else if (ActualMVT == MVT::i16)
3008 634 : ValVT = MVT::i16;
3009 : }
3010 24761 : CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3011 : bool Res =
3012 24761 : AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
3013 : assert(!Res && "Call operand has unhandled type");
3014 : (void)Res;
3015 : }
3016 : assert(ArgLocs.size() == Ins.size());
3017 : SmallVector<SDValue, 16> ArgValues;
3018 38365 : for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3019 24761 : CCValAssign &VA = ArgLocs[i];
3020 :
3021 24761 : if (Ins[i].Flags.isByVal()) {
3022 : // Byval is used for HFAs in the PCS, but the system should work in a
3023 : // non-compliant manner for larger structs.
3024 10 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
3025 10 : int Size = Ins[i].Flags.getByValSize();
3026 10 : unsigned NumRegs = (Size + 7) / 8;
3027 :
3028 : // FIXME: This works on big-endian for composite byvals, which are the common
3029 : // case. It should also work for fundamental types too.
3030 : unsigned FrameIdx =
3031 10 : MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
3032 10 : SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
3033 10 : InVals.push_back(FrameIdxN);
3034 :
3035 : continue;
3036 : }
3037 :
3038 24751 : if (VA.isRegLoc()) {
3039 : // Arguments stored in registers.
3040 : EVT RegVT = VA.getLocVT();
3041 :
3042 24309 : SDValue ArgValue;
3043 : const TargetRegisterClass *RC;
3044 :
3045 : if (RegVT == MVT::i32)
3046 : RC = &AArch64::GPR32RegClass;
3047 : else if (RegVT == MVT::i64)
3048 : RC = &AArch64::GPR64RegClass;
3049 : else if (RegVT == MVT::f16)
3050 : RC = &AArch64::FPR16RegClass;
3051 : else if (RegVT == MVT::f32)
3052 : RC = &AArch64::FPR32RegClass;
3053 10035 : else if (RegVT == MVT::f64 || RegVT.is64BitVector())
3054 : RC = &AArch64::FPR64RegClass;
3055 5901 : else if (RegVT == MVT::f128 || RegVT.is128BitVector())
3056 : RC = &AArch64::FPR128RegClass;
3057 : else
3058 0 : llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering");
3059 :
3060 : // Transform the arguments in physical registers into virtual ones.
3061 24309 : unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3062 24309 : ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3063 :
3064 : // If this is an 8, 16 or 32-bit value, it is really passed promoted
3065 : // to 64 bits. Insert an assert[sz]ext to capture this, then
3066 : // truncate to the right size.
3067 24309 : switch (VA.getLocInfo()) {
3068 0 : default:
3069 0 : llvm_unreachable("Unknown loc info!");
3070 : case CCValAssign::Full:
3071 : break;
3072 : case CCValAssign::BCvt:
3073 1895 : ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
3074 1895 : break;
3075 : case CCValAssign::AExt:
3076 : case CCValAssign::SExt:
3077 : case CCValAssign::ZExt:
3078 : // SelectionDAGBuilder will insert appropriate AssertZExt & AssertSExt
3079 : // nodes after our lowering.
3080 : assert(RegVT == Ins[i].VT && "incorrect register location selected");
3081 : break;
3082 : }
3083 :
3084 24309 : InVals.push_back(ArgValue);
3085 :
3086 : } else { // VA.isRegLoc()
3087 : assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem");
3088 442 : unsigned ArgOffset = VA.getLocMemOffset();
3089 442 : unsigned ArgSize = VA.getValVT().getSizeInBits() / 8;
3090 :
3091 : uint32_t BEAlign = 0;
3092 442 : if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
3093 6 : !Ins[i].Flags.isInConsecutiveRegs())
3094 4 : BEAlign = 8 - ArgSize;
3095 :
3096 442 : int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
3097 :
3098 : // Create load nodes to retrieve arguments from the stack.
3099 884 : SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3100 442 : SDValue ArgValue;
3101 :
3102 : // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
3103 : ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
3104 : MVT MemVT = VA.getValVT();
3105 :
3106 442 : switch (VA.getLocInfo()) {
3107 : default:
3108 : break;
3109 15 : case CCValAssign::BCvt:
3110 : MemVT = VA.getLocVT();
3111 15 : break;
3112 8 : case CCValAssign::SExt:
3113 : ExtType = ISD::SEXTLOAD;
3114 8 : break;
3115 5 : case CCValAssign::ZExt:
3116 : ExtType = ISD::ZEXTLOAD;
3117 5 : break;
3118 9 : case CCValAssign::AExt:
3119 : ExtType = ISD::EXTLOAD;
3120 9 : break;
3121 : }
3122 :
3123 442 : ArgValue = DAG.getExtLoad(
3124 : ExtType, DL, VA.getLocVT(), Chain, FIN,
3125 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
3126 442 : MemVT);
3127 :
3128 442 : InVals.push_back(ArgValue);
3129 : }
3130 : }
3131 :
3132 : // varargs
3133 13604 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3134 13604 : if (isVarArg) {
3135 36 : if (!Subtarget->isTargetDarwin() || IsWin64) {
3136 : // The AAPCS variadic function ABI is identical to the non-variadic
3137 : // one. As a result there may be more arguments in registers and we should
3138 : // save them for future reference.
3139 : // Win64 variadic functions also pass arguments in registers, but all float
3140 : // arguments are passed in integer registers.
3141 19 : saveVarArgRegisters(CCInfo, DAG, DL, Chain);
3142 : }
3143 :
3144 : // This will point to the next argument passed via stack.
3145 28 : unsigned StackOffset = CCInfo.getNextStackOffset();
3146 : // We currently pass all varargs at 8-byte alignment.
3147 28 : StackOffset = ((StackOffset + 7) & ~7);
3148 28 : FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
3149 : }
3150 :
3151 13604 : unsigned StackArgSize = CCInfo.getNextStackOffset();
3152 13604 : bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3153 13604 : if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
3154 : // This is a non-standard ABI so by fiat I say we're allowed to make full
3155 : // use of the stack area to be popped, which must be aligned to 16 bytes in
3156 : // any case:
3157 42 : StackArgSize = alignTo(StackArgSize, 16);
3158 :
3159 : // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
3160 : // a multiple of 16.
3161 : FuncInfo->setArgumentStackToRestore(StackArgSize);
3162 :
3163 : // This realignment carries over to the available bytes below. Our own
3164 : // callers will guarantee the space is free by giving an aligned value to
3165 : // CALLSEQ_START.
3166 : }
3167 : // Even if we're not expected to free up the space, it's useful to know how
3168 : // much is there while considering tail calls (because we can reuse it).
3169 : FuncInfo->setBytesInStackArgArea(StackArgSize);
3170 :
3171 27208 : if (Subtarget->hasCustomCallingConv())
3172 22 : Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
3173 :
3174 13604 : return Chain;
3175 : }
3176 :
3177 19 : void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
3178 : SelectionDAG &DAG,
3179 : const SDLoc &DL,
3180 : SDValue &Chain) const {
3181 19 : MachineFunction &MF = DAG.getMachineFunction();
3182 19 : MachineFrameInfo &MFI = MF.getFrameInfo();
3183 19 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3184 19 : auto PtrVT = getPointerTy(DAG.getDataLayout());
3185 19 : bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
3186 :
3187 : SmallVector<SDValue, 8> MemOps;
3188 :
3189 : static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
3190 : AArch64::X3, AArch64::X4, AArch64::X5,
3191 : AArch64::X6, AArch64::X7 };
3192 : static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
3193 : unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
3194 :
3195 19 : unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
3196 : int GPRIdx = 0;
3197 19 : if (GPRSaveSize != 0) {
3198 13 : if (IsWin64) {
3199 8 : GPRIdx = MFI.CreateFixedObject(GPRSaveSize, -(int)GPRSaveSize, false);
3200 8 : if (GPRSaveSize & 15)
3201 : // The extra size here, if triggered, will always be 8.
3202 7 : MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
3203 : } else
3204 5 : GPRIdx = MFI.CreateStackObject(GPRSaveSize, 8, false);
3205 :
3206 13 : SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
3207 :
3208 85 : for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
3209 72 : unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
3210 72 : SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3211 : SDValue Store = DAG.getStore(
3212 : Val.getValue(1), DL, Val, FIN,
3213 : IsWin64
3214 : ? MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
3215 : GPRIdx,
3216 39 : (i - FirstVariadicGPR) * 8)
3217 111 : : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8));
3218 72 : MemOps.push_back(Store);
3219 72 : FIN =
3220 72 : DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
3221 : }
3222 : }
3223 : FuncInfo->setVarArgsGPRIndex(GPRIdx);
3224 : FuncInfo->setVarArgsGPRSize(GPRSaveSize);
3225 :
3226 19 : if (Subtarget->hasFPARMv8() && !IsWin64) {
3227 : static const MCPhysReg FPRArgRegs[] = {
3228 : AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
3229 : AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
3230 : static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
3231 : unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
3232 :
3233 6 : unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
3234 : int FPRIdx = 0;
3235 6 : if (FPRSaveSize != 0) {
3236 5 : FPRIdx = MFI.CreateStackObject(FPRSaveSize, 16, false);
3237 :
3238 5 : SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
3239 :
3240 41 : for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
3241 36 : unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
3242 36 : SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
3243 :
3244 : SDValue Store = DAG.getStore(
3245 : Val.getValue(1), DL, Val, FIN,
3246 36 : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16));
3247 36 : MemOps.push_back(Store);
3248 36 : FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
3249 36 : DAG.getConstant(16, DL, PtrVT));
3250 : }
3251 : }
3252 : FuncInfo->setVarArgsFPRIndex(FPRIdx);
3253 : FuncInfo->setVarArgsFPRSize(FPRSaveSize);
3254 : }
3255 :
3256 19 : if (!MemOps.empty()) {
3257 14 : Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3258 : }
3259 19 : }
3260 :
3261 : /// LowerCallResult - Lower the result values of a call into the
3262 : /// appropriate copies out of appropriate physical registers.
3263 1773 : SDValue AArch64TargetLowering::LowerCallResult(
3264 : SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
3265 : const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3266 : SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
3267 : SDValue ThisVal) const {
3268 : CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
3269 1773 : ? RetCC_AArch64_WebKit_JS
3270 : : RetCC_AArch64_AAPCS;
3271 : // Assign locations to each value returned by this call.
3272 : SmallVector<CCValAssign, 16> RVLocs;
3273 : CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3274 1773 : *DAG.getContext());
3275 1773 : CCInfo.AnalyzeCallResult(Ins, RetCC);
3276 :
3277 : // Copy all of the result registers out of their specified physreg.
3278 3021 : for (unsigned i = 0; i != RVLocs.size(); ++i) {
3279 1248 : CCValAssign VA = RVLocs[i];
3280 :
3281 : // Pass 'this' value directly from the argument to return value, to avoid
3282 : // reg unit interference
3283 1248 : if (i == 0 && isThisReturn) {
3284 : assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
3285 : "unexpected return calling convention register assignment");
3286 7 : InVals.push_back(ThisVal);
3287 7 : continue;
3288 : }
3289 :
3290 : SDValue Val =
3291 1241 : DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
3292 1241 : Chain = Val.getValue(1);
3293 1241 : InFlag = Val.getValue(2);
3294 :
3295 1241 : switch (VA.getLocInfo()) {
3296 0 : default:
3297 0 : llvm_unreachable("Unknown loc info!");
3298 : case CCValAssign::Full:
3299 : break;
3300 : case CCValAssign::BCvt:
3301 136 : Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
3302 136 : break;
3303 : }
3304 :
3305 1241 : InVals.push_back(Val);
3306 : }
3307 :
3308 1773 : return Chain;
3309 : }
3310 :
3311 : /// Return true if the calling convention is one that we can guarantee TCO for.
3312 : static bool canGuaranteeTCO(CallingConv::ID CC) {
3313 13 : return CC == CallingConv::Fast;
3314 : }
3315 :
3316 : /// Return true if we might ever do TCO for calls with this calling convention.
3317 : static bool mayTailCallThisCC(CallingConv::ID CC) {
3318 : switch (CC) {
3319 : case CallingConv::C:
3320 : case CallingConv::PreserveMost:
3321 : case CallingConv::Swift:
3322 : return true;
3323 : default:
3324 : return canGuaranteeTCO(CC);
3325 : }
3326 : }
3327 :
3328 270 : bool AArch64TargetLowering::isEligibleForTailCallOptimization(
3329 : SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
3330 : const SmallVectorImpl<ISD::OutputArg> &Outs,
3331 : const SmallVectorImpl<SDValue> &OutVals,
3332 : const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
3333 13 : if (!mayTailCallThisCC(CalleeCC))
3334 : return false;
3335 :
3336 267 : MachineFunction &MF = DAG.getMachineFunction();
3337 267 : const Function &CallerF = MF.getFunction();
3338 : CallingConv::ID CallerCC = CallerF.getCallingConv();
3339 : bool CCMatch = CallerCC == CalleeCC;
3340 :
3341 : // Byval parameters hand the function a pointer directly into the stack area
3342 : // we want to reuse during a tail call. Working around this *is* possible (see
3343 : // X86) but less efficient and uglier in LowerCall.
3344 256 : for (Function::const_arg_iterator i = CallerF.arg_begin(),
3345 : e = CallerF.arg_end();
3346 523 : i != e; ++i)
3347 256 : if (i->hasByValAttr())
3348 : return false;
3349 :
3350 267 : if (getTargetMachine().Options.GuaranteedTailCallOpt)
3351 8 : return canGuaranteeTCO(CalleeCC) && CCMatch;
3352 :
3353 : // Externally-defined functions with weak linkage should not be
3354 : // tail-called on AArch64 when the OS does not support dynamic
3355 : // pre-emption of symbols, as the AAELF spec requires normal calls
3356 : // to undefined weak functions to be replaced with a NOP or jump to the
3357 : // next instruction. The behaviour of branch instructions in this
3358 : // situation (as used for tail calls) is implementation-defined, so we
3359 : // cannot rely on the linker replacing the tail call with a return.
3360 : if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3361 242 : const GlobalValue *GV = G->getGlobal();
3362 : const Triple &TT = getTargetMachine().getTargetTriple();
3363 242 : if (GV->hasExternalWeakLinkage() &&
3364 0 : (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
3365 : return false;
3366 : }
3367 :
3368 : // Now we search for cases where we can use a tail call without changing the
3369 : // ABI. Sibcall is used in some places (particularly gcc) to refer to this
3370 : // concept.
3371 :
3372 : // I want anyone implementing a new calling convention to think long and hard
3373 : // about this assert.
3374 : assert((!isVarArg || CalleeCC == CallingConv::C) &&
3375 : "Unexpected variadic calling convention");
3376 :
3377 259 : LLVMContext &C = *DAG.getContext();
3378 259 : if (isVarArg && !Outs.empty()) {
3379 : // At least two cases here: if caller is fastcc then we can't have any
3380 : // memory arguments (we'd be expected to clean up the stack afterwards). If
3381 : // caller is C then we could potentially use its argument area.
3382 :
3383 : // FIXME: for now we take the most conservative of these in both cases:
3384 : // disallow all variadic memory operands.
3385 : SmallVector<CCValAssign, 16> ArgLocs;
3386 2 : CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3387 :
3388 2 : CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
3389 10 : for (const CCValAssign &ArgLoc : ArgLocs)
3390 10 : if (!ArgLoc.isRegLoc())
3391 2 : return false;
3392 : }
3393 :
3394 : // Check that the call results are passed in the same way.
3395 257 : if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
3396 : CCAssignFnForCall(CalleeCC, isVarArg),
3397 : CCAssignFnForCall(CallerCC, isVarArg)))
3398 : return false;
3399 : // The callee has to preserve all registers the caller needs to preserve.
3400 257 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
3401 257 : const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3402 257 : if (!CCMatch) {
3403 7 : const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3404 14 : if (Subtarget->hasCustomCallingConv()) {
3405 0 : TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
3406 0 : TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
3407 : }
3408 7 : if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3409 4 : return false;
3410 : }
3411 :
3412 : // Nothing more to check if the callee is taking no arguments
3413 253 : if (Outs.empty())
3414 : return true;
3415 :
3416 : SmallVector<CCValAssign, 16> ArgLocs;
3417 408 : CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3418 :
3419 204 : CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
3420 :
3421 204 : const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3422 :
3423 : // If the stack arguments for this call do not fit into our own save area then
3424 : // the call cannot be made tail.
3425 204 : if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
3426 : return false;
3427 :
3428 176 : const MachineRegisterInfo &MRI = MF.getRegInfo();
3429 176 : if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
3430 4 : return false;
3431 :
3432 : return true;
3433 : }
3434 :
3435 11 : SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
3436 : SelectionDAG &DAG,
3437 : MachineFrameInfo &MFI,
3438 : int ClobberedFI) const {
3439 : SmallVector<SDValue, 8> ArgChains;
3440 : int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
3441 11 : int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
3442 :
3443 : // Include the original chain at the beginning of the list. When this is
3444 : // used by target LowerCall hooks, this helps legalize find the
3445 : // CALLSEQ_BEGIN node.
3446 11 : ArgChains.push_back(Chain);
3447 :
3448 : // Add a chain value for each stack argument corresponding
3449 11 : for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
3450 : UE = DAG.getEntryNode().getNode()->use_end();
3451 120 : U != UE; ++U)
3452 : if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3453 : if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3454 18 : if (FI->getIndex() < 0) {
3455 : int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
3456 : int64_t InLastByte = InFirstByte;
3457 18 : InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
3458 :
3459 18 : if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
3460 9 : (FirstByte <= InFirstByte && InFirstByte <= LastByte))
3461 9 : ArgChains.push_back(SDValue(L, 1));
3462 : }
3463 :
3464 : // Build a tokenfactor for all the chains.
3465 22 : return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
3466 : }
3467 :
3468 15377 : bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
3469 : bool TailCallOpt) const {
3470 15377 : return CallCC == CallingConv::Fast && TailCallOpt;
3471 : }
3472 :
3473 : /// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
3474 : /// and add input and output parameter nodes.
3475 : SDValue
3476 2001 : AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
3477 : SmallVectorImpl<SDValue> &InVals) const {
3478 2001 : SelectionDAG &DAG = CLI.DAG;
3479 2001 : SDLoc &DL = CLI.DL;
3480 : SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
3481 : SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
3482 : SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
3483 2001 : SDValue Chain = CLI.Chain;
3484 2001 : SDValue Callee = CLI.Callee;
3485 : bool &IsTailCall = CLI.IsTailCall;
3486 2001 : CallingConv::ID CallConv = CLI.CallConv;
3487 2001 : bool IsVarArg = CLI.IsVarArg;
3488 :
3489 2001 : MachineFunction &MF = DAG.getMachineFunction();
3490 : bool IsThisReturn = false;
3491 :
3492 2001 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3493 2001 : bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3494 : bool IsSibCall = false;
3495 :
3496 2001 : if (IsTailCall) {
3497 : // Check if it's really possible to do a tail call.
3498 270 : IsTailCall = isEligibleForTailCallOptimization(
3499 : Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
3500 270 : if (!IsTailCall && CLI.CS && CLI.CS.isMustTailCall())
3501 0 : report_fatal_error("failed to perform tail call elimination on a call "
3502 : "site marked musttail");
3503 :
3504 : // A sibling call is one where we're under the usual C ABI and not planning
3505 : // to change that but can still do a tail call:
3506 270 : if (!TailCallOpt && IsTailCall)
3507 : IsSibCall = true;
3508 :
3509 : if (IsTailCall)
3510 : ++NumTailCalls;
3511 : }
3512 :
3513 : // Analyze operands of the call, assigning locations to each operand.
3514 : SmallVector<CCValAssign, 16> ArgLocs;
3515 : CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
3516 4002 : *DAG.getContext());
3517 :
3518 2001 : if (IsVarArg) {
3519 : // Handle fixed and variable vector arguments differently.
3520 : // Variable vector arguments always go into memory.
3521 26 : unsigned NumArgs = Outs.size();
3522 :
3523 123 : for (unsigned i = 0; i != NumArgs; ++i) {
3524 97 : MVT ArgVT = Outs[i].VT;
3525 97 : ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3526 97 : CCAssignFn *AssignFn = CCAssignFnForCall(CallConv,
3527 97 : /*IsVarArg=*/ !Outs[i].IsFixed);
3528 97 : bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
3529 : assert(!Res && "Call operand has unhandled type");
3530 : (void)Res;
3531 : }
3532 : } else {
3533 : // At this point, Outs[].VT may already be promoted to i32. To correctly
3534 : // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
3535 : // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
3536 : // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
3537 : // we use a special version of AnalyzeCallOperands to pass in ValVT and
3538 : // LocVT.
3539 1975 : unsigned NumArgs = Outs.size();
3540 5588 : for (unsigned i = 0; i != NumArgs; ++i) {
3541 3613 : MVT ValVT = Outs[i].VT;
3542 : // Get type of the original argument.
3543 : EVT ActualVT = getValueType(DAG.getDataLayout(),
3544 3613 : CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
3545 7226 : /*AllowUnknown*/ true);
3546 3613 : MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
3547 3613 : ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3548 : // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3549 3613 : if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3550 : ValVT = MVT::i8;
3551 3542 : else if (ActualMVT == MVT::i16)
3552 : ValVT = MVT::i16;
3553 :
3554 3613 : CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3555 3613 : bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
3556 : assert(!Res && "Call operand has unhandled type");
3557 : (void)Res;
3558 : }
3559 : }
3560 :
3561 : // Get a count of how many bytes are to be pushed on the stack.
3562 2001 : unsigned NumBytes = CCInfo.getNextStackOffset();
3563 :
3564 2001 : if (IsSibCall) {
3565 : // Since we're not changing the ABI to make this a tail call, the memory
3566 : // operands are already available in the caller's incoming argument space.
3567 : NumBytes = 0;
3568 : }
3569 :
3570 : // FPDiff is the byte offset of the call's argument area from the callee's.
3571 : // Stores to callee stack arguments will be placed in FixedStackSlots offset
3572 : // by this amount for a tail call. In a sibling call it must be 0 because the
3573 : // caller will deallocate the entire stack and the callee still expects its
3574 : // arguments to begin at SP+0. Completely unused for non-tail calls.
3575 : int FPDiff = 0;
3576 :
3577 2001 : if (IsTailCall && !IsSibCall) {
3578 7 : unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
3579 :
3580 : // Since callee will pop argument stack as a tail call, we must keep the
3581 : // popped size 16-byte aligned.
3582 7 : NumBytes = alignTo(NumBytes, 16);
3583 :
3584 : // FPDiff will be negative if this tail call requires more space than we
3585 : // would automatically have in our incoming argument space. Positive if we
3586 : // can actually shrink the stack.
3587 7 : FPDiff = NumReusableBytes - NumBytes;
3588 :
3589 : // The stack pointer must be 16-byte aligned at all times it's used for a
3590 : // memory operation, which in practice means at *all* times and in
3591 : // particular across call boundaries. Therefore our own arguments started at
3592 : // a 16-byte aligned SP and the delta applied for the tail call should
3593 : // satisfy the same constraint.
3594 : assert(FPDiff % 16 == 0 && "unaligned stack on tail call");
3595 : }
3596 :
3597 : // Adjust the stack pointer for the new arguments...
3598 : // These operations are automatically eliminated by the prolog/epilog pass
3599 2001 : if (!IsSibCall)
3600 1780 : Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
3601 :
3602 : SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
3603 4002 : getPointerTy(DAG.getDataLayout()));
3604 :
3605 : SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3606 : SmallVector<SDValue, 8> MemOpChains;
3607 2001 : auto PtrVT = getPointerTy(DAG.getDataLayout());
3608 :
3609 : // Walk the register/memloc assignments, inserting copies/loads.
3610 5711 : for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
3611 : ++i, ++realArgIdx) {
3612 3710 : CCValAssign &VA = ArgLocs[i];
3613 3710 : SDValue Arg = OutVals[realArgIdx];
3614 3710 : ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
3615 :
3616 : // Promote the value if needed.
3617 3710 : switch (VA.getLocInfo()) {
3618 0 : default:
3619 0 : llvm_unreachable("Unknown loc info!");
3620 : case CCValAssign::Full:
3621 : break;
3622 : case CCValAssign::SExt:
3623 25 : Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
3624 25 : break;
3625 : case CCValAssign::ZExt:
3626 13 : Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3627 13 : break;
3628 : case CCValAssign::AExt:
3629 : if (Outs[realArgIdx].ArgVT == MVT::i1) {
3630 : // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
3631 1 : Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
3632 1 : Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
3633 : }
3634 89 : Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3635 89 : break;
3636 : case CCValAssign::BCvt:
3637 230 : Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
3638 230 : break;
3639 : case CCValAssign::FPExt:
3640 0 : Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
3641 0 : break;
3642 : }
3643 :
3644 3710 : if (VA.isRegLoc()) {
3645 2945 : if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
3646 10 : Outs[0].VT == MVT::i64) {
3647 : assert(VA.getLocVT() == MVT::i64 &&
3648 : "unexpected calling convention register assignment");
3649 : assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&
3650 : "unexpected use of 'returned'");
3651 : IsThisReturn = true;
3652 : }
3653 5890 : RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
3654 : } else {
3655 : assert(VA.isMemLoc());
3656 :
3657 765 : SDValue DstAddr;
3658 : MachinePointerInfo DstInfo;
3659 :
3660 : // FIXME: This works on big-endian for composite byvals, which are the
3661 : // common case. It should also work for fundamental types too.
3662 : uint32_t BEAlign = 0;
3663 765 : unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
3664 760 : : VA.getValVT().getSizeInBits();
3665 765 : OpSize = (OpSize + 7) / 8;
3666 765 : if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
3667 10 : !Flags.isInConsecutiveRegs()) {
3668 8 : if (OpSize < 8)
3669 6 : BEAlign = 8 - OpSize;
3670 : }
3671 765 : unsigned LocMemOffset = VA.getLocMemOffset();
3672 765 : int32_t Offset = LocMemOffset + BEAlign;
3673 765 : SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3674 765 : PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
3675 :
3676 765 : if (IsTailCall) {
3677 11 : Offset = Offset + FPDiff;
3678 11 : int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
3679 :
3680 11 : DstAddr = DAG.getFrameIndex(FI, PtrVT);
3681 : DstInfo =
3682 11 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
3683 :
3684 : // Make sure any stack arguments overlapping with where we're storing
3685 : // are loaded before this eventual operation. Otherwise they'll be
3686 : // clobbered.
3687 11 : Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
3688 : } else {
3689 754 : SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3690 :
3691 754 : DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
3692 : DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(),
3693 754 : LocMemOffset);
3694 : }
3695 :
3696 765 : if (Outs[i].Flags.isByVal()) {
3697 : SDValue SizeNode =
3698 5 : DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
3699 : SDValue Cpy = DAG.getMemcpy(
3700 : Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
3701 : /*isVol = */ false, /*AlwaysInline = */ false,
3702 : /*isTailCall = */ false,
3703 5 : DstInfo, MachinePointerInfo());
3704 :
3705 5 : MemOpChains.push_back(Cpy);
3706 : } else {
3707 : // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
3708 : // promoted to a legal register type i32, we should truncate Arg back to
3709 : // i1/i8/i16.
3710 760 : if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
3711 : VA.getValVT() == MVT::i16)
3712 23 : Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
3713 :
3714 760 : SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
3715 760 : MemOpChains.push_back(Store);
3716 : }
3717 : }
3718 : }
3719 :
3720 2001 : if (!MemOpChains.empty())
3721 100 : Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
3722 :
3723 : // Build a sequence of copy-to-reg nodes chained together with token chain
3724 : // and flag operands which copy the outgoing args into the appropriate regs.
3725 2001 : SDValue InFlag;
3726 4946 : for (auto &RegToPass : RegsToPass) {
3727 2945 : Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
3728 2945 : RegToPass.second, InFlag);
3729 2945 : InFlag = Chain.getValue(1);
3730 : }
3731 :
3732 : // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
3733 : // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
3734 : // node so that legalize doesn't hack it.
3735 : if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3736 1362 : auto GV = G->getGlobal();
3737 1362 : if (Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine()) ==
3738 : AArch64II::MO_GOT) {
3739 2 : Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT);
3740 2 : Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
3741 2720 : } else if (Subtarget->isTargetCOFF() && GV->hasDLLImportStorageClass()) {
3742 : assert(Subtarget->isTargetWindows() &&
3743 : "Windows is the only supported COFF target");
3744 3 : Callee = getGOT(G, DAG, AArch64II::MO_DLLIMPORT);
3745 : } else {
3746 1357 : const GlobalValue *GV = G->getGlobal();
3747 1357 : Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
3748 : }
3749 : } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
3750 588 : if (getTargetMachine().getCodeModel() == CodeModel::Large &&
3751 3 : Subtarget->isTargetMachO()) {
3752 0 : const char *Sym = S->getSymbol();
3753 0 : Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
3754 0 : Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
3755 : } else {
3756 588 : const char *Sym = S->getSymbol();
3757 588 : Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
3758 : }
3759 : }
3760 :
3761 : // We don't usually want to end the call-sequence here because we would tidy
3762 : // the frame up *after* the call, however in the ABI-changing tail-call case
3763 : // we've carefully laid out the parameters so that when sp is reset they'll be
3764 : // in the correct location.
3765 2001 : if (IsTailCall && !IsSibCall) {
3766 7 : Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
3767 7 : DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
3768 7 : InFlag = Chain.getValue(1);
3769 : }
3770 :
3771 : std::vector<SDValue> Ops;
3772 2001 : Ops.push_back(Chain);
3773 2001 : Ops.push_back(Callee);
3774 :
3775 2001 : if (IsTailCall) {
3776 : // Each tail call may have to adjust the stack by a different amount, so
3777 : // this information must travel along with the operation for eventual
3778 : // consumption by emitEpilogue.
3779 228 : Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
3780 : }
3781 :
3782 : // Add argument registers to the end of the list so that they are known live
3783 : // into the call.
3784 4946 : for (auto &RegToPass : RegsToPass)
3785 2945 : Ops.push_back(DAG.getRegister(RegToPass.first,
3786 5890 : RegToPass.second.getValueType()));
3787 :
3788 : // Add a register mask operand representing the call-preserved registers.
3789 : const uint32_t *Mask;
3790 2001 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
3791 2001 : if (IsThisReturn) {
3792 : // For 'this' returns, use the X0-preserving mask if applicable
3793 10 : Mask = TRI->getThisReturnPreservedMask(MF, CallConv);
3794 10 : if (!Mask) {
3795 : IsThisReturn = false;
3796 0 : Mask = TRI->getCallPreservedMask(MF, CallConv);
3797 : }
3798 : } else
3799 1991 : Mask = TRI->getCallPreservedMask(MF, CallConv);
3800 :
3801 4002 : if (Subtarget->hasCustomCallingConv())
3802 10 : TRI->UpdateCustomCallPreservedMask(MF, &Mask);
3803 :
3804 2001 : if (TRI->isAnyArgRegReserved(MF))
3805 2 : TRI->emitReservedArgRegCallError(MF);
3806 :
3807 : assert(Mask && "Missing call preserved mask for calling convention");
3808 2001 : Ops.push_back(DAG.getRegisterMask(Mask));
3809 :
3810 2001 : if (InFlag.getNode())
3811 1665 : Ops.push_back(InFlag);
3812 :
3813 2001 : SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
3814 :
3815 : // If we're doing a tall call, use a TC_RETURN here rather than an
3816 : // actual call instruction.
3817 2001 : if (IsTailCall) {
3818 228 : MF.getFrameInfo().setHasTailCall();
3819 228 : return DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
3820 : }
3821 :
3822 : // Returns a chain and a flag for retval copy to use.
3823 1773 : Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
3824 1773 : InFlag = Chain.getValue(1);
3825 :
3826 : uint64_t CalleePopBytes =
3827 1773 : DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
3828 :
3829 1773 : Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
3830 : DAG.getIntPtrConstant(CalleePopBytes, DL, true),
3831 1773 : InFlag, DL);
3832 1773 : if (!Ins.empty())
3833 1150 : InFlag = Chain.getValue(1);
3834 :
3835 : // Handle result values, copying them out of physregs into vregs that we
3836 : // return.
3837 : return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
3838 : InVals, IsThisReturn,
3839 1773 : IsThisReturn ? OutVals[0] : SDValue());
3840 : }
3841 :
3842 16938 : bool AArch64TargetLowering::CanLowerReturn(
3843 : CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
3844 : const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
3845 : CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
3846 16938 : ? RetCC_AArch64_WebKit_JS
3847 : : RetCC_AArch64_AAPCS;
3848 : SmallVector<CCValAssign, 16> RVLocs;
3849 33876 : CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
3850 16938 : return CCInfo.CheckReturn(Outs, RetCC);
3851 : }
3852 :
3853 : SDValue
3854 13564 : AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
3855 : bool isVarArg,
3856 : const SmallVectorImpl<ISD::OutputArg> &Outs,
3857 : const SmallVectorImpl<SDValue> &OutVals,
3858 : const SDLoc &DL, SelectionDAG &DAG) const {
3859 : CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
3860 13564 : ? RetCC_AArch64_WebKit_JS
3861 : : RetCC_AArch64_AAPCS;
3862 : SmallVector<CCValAssign, 16> RVLocs;
3863 : CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3864 27128 : *DAG.getContext());
3865 13564 : CCInfo.AnalyzeReturn(Outs, RetCC);
3866 :
3867 : // Copy the result values into the output registers.
3868 13564 : SDValue Flag;
3869 : SmallVector<SDValue, 4> RetOps(1, Chain);
3870 25626 : for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
3871 : ++i, ++realRVLocIdx) {
3872 : CCValAssign &VA = RVLocs[i];
3873 : assert(VA.isRegLoc() && "Can only return in registers!");
3874 12062 : SDValue Arg = OutVals[realRVLocIdx];
3875 :
3876 12062 : switch (VA.getLocInfo()) {
3877 0 : default:
3878 0 : llvm_unreachable("Unknown loc info!");
3879 : case CCValAssign::Full:
3880 : if (Outs[i].ArgVT == MVT::i1) {
3881 : // AAPCS requires i1 to be zero-extended to i8 by the producer of the
3882 : // value. This is strictly redundant on Darwin (which uses "zeroext
3883 : // i1"), but will be optimised out before ISel.
3884 209 : Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
3885 209 : Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3886 : }
3887 : break;
3888 : case CCValAssign::BCvt:
3889 1206 : Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
3890 1206 : break;
3891 : }
3892 :
3893 12062 : Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Flag);
3894 12062 : Flag = Chain.getValue(1);
3895 12062 : RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3896 : }
3897 13564 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
3898 : const MCPhysReg *I =
3899 13564 : TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3900 13564 : if (I) {
3901 342 : for (; *I; ++I) {
3902 336 : if (AArch64::GPR64RegClass.contains(*I))
3903 144 : RetOps.push_back(DAG.getRegister(*I, MVT::i64));
3904 192 : else if (AArch64::FPR64RegClass.contains(*I))
3905 192 : RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
3906 : else
3907 0 : llvm_unreachable("Unexpected register class in CSRsViaCopy!");
3908 : }
3909 : }
3910 :
3911 13564 : RetOps[0] = Chain; // Update chain.
3912 :
3913 : // Add the flag if we have it.
3914 13564 : if (Flag.getNode())
3915 10920 : RetOps.push_back(Flag);
3916 :
3917 13564 : return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
3918 : }
3919 :
3920 : //===----------------------------------------------------------------------===//
3921 : // Other Lowering Code
3922 : //===----------------------------------------------------------------------===//
3923 :
3924 5978 : SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
3925 : SelectionDAG &DAG,
3926 : unsigned Flag) const {
3927 5978 : return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty,
3928 5988 : N->getOffset(), Flag);
3929 : }
3930 :
3931 57 : SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
3932 : SelectionDAG &DAG,
3933 : unsigned Flag) const {
3934 57 : return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
3935 : }
3936 :
3937 365 : SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
3938 : SelectionDAG &DAG,
3939 : unsigned Flag) const {
3940 : return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
3941 730 : N->getOffset(), Flag);
3942 : }
3943 :
3944 17 : SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
3945 : SelectionDAG &DAG,
3946 : unsigned Flag) const {
3947 17 : return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
3948 : }
3949 :
3950 : // (loadGOT sym)
3951 : template <class NodeTy>
3952 218 : SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
3953 : unsigned Flags) const {
3954 : LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n");
3955 : SDLoc DL(N);
3956 218 : EVT Ty = getPointerTy(DAG.getDataLayout());
3957 218 : SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
3958 : // FIXME: Once remat is capable of dealing with instructions with register
3959 : // operands, expand this into two nodes instead of using a wrapper node.
3960 218 : return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
3961 : }
3962 0 :
3963 : // (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
3964 : template <class NodeTy>
3965 : SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
3966 0 : unsigned Flags) const {
3967 0 : LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n");
3968 : SDLoc DL(N);
3969 : EVT Ty = getPointerTy(DAG.getDataLayout());
3970 0 : const unsigned char MO_NC = AArch64II::MO_NC;
3971 : return DAG.getNode(
3972 218 : AArch64ISD::WrapperLarge, DL, Ty,
3973 : getTargetNode(N, Ty, DAG, AArch64II::MO_G3 | Flags),
3974 : getTargetNode(N, Ty, DAG, AArch64II::MO_G2 | MO_NC | Flags),
3975 : getTargetNode(N, Ty, DAG, AArch64II::MO_G1 | MO_NC | Flags),
3976 218 : getTargetNode(N, Ty, DAG, AArch64II::MO_G0 | MO_NC | Flags));
3977 218 : }
3978 :
3979 : // (addlow (adrp %hi(sym)) %lo(sym))
3980 218 : template <class NodeTy>
3981 : SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
3982 : unsigned Flags) const {
3983 : LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddr\n");
3984 : SDLoc DL(N);
3985 29 : EVT Ty = getPointerTy(DAG.getDataLayout());
3986 : SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
3987 : SDValue Lo = getTargetNode(N, Ty, DAG,
3988 : AArch64II::MO_PAGEOFF | AArch64II::MO_NC | Flags);
3989 29 : SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
3990 : return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
3991 : }
3992 :
3993 : // (adr sym)
3994 : template <class NodeTy>
3995 : SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
3996 29 : unsigned Flags) const {
3997 : LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrTiny\n");
3998 2 : SDLoc DL(N);
3999 : EVT Ty = getPointerTy(DAG.getDataLayout());
4000 : SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
4001 : return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
4002 2 : }
4003 :
4004 : SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4005 : SelectionDAG &DAG) const {
4006 : GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
4007 : const GlobalValue *GV = GN->getGlobal();
4008 : unsigned char OpFlags =
4009 2 : Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
4010 :
4011 4 : if (OpFlags != AArch64II::MO_NO_FLAG)
4012 : assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
4013 : "unexpected offset in global node");
4014 :
4015 4 : // This also catches the large code model case for Darwin, and tiny code
4016 : // model with got relocations.
4017 : if ((OpFlags & AArch64II::MO_GOT) != 0) {
4018 : return getGOT(GN, DAG, OpFlags);
4019 : }
4020 :
4021 : SDValue Result;
4022 4 : if (getTargetMachine().getCodeModel() == CodeModel::Large) {
4023 : Result = getAddrLarge(GN, DAG, OpFlags);
4024 1 : } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4025 : Result = getAddrTiny(GN, DAG, OpFlags);
4026 : } else {
4027 : Result = getAddr(GN, DAG, OpFlags);
4028 1 : }
4029 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
4030 : SDLoc DL(GN);
4031 : if (OpFlags & (AArch64II::MO_DLLIMPORT | AArch64II::MO_COFFSTUB))
4032 : Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4033 : MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4034 : return Result;
4035 1 : }
4036 :
4037 22 : /// Convert a TLS address reference into the correct sequence of loads
4038 : /// and calls to compute the variable's address (for Darwin, currently) and
4039 : /// return an SDValue containing the final node.
4040 :
4041 22 : /// Darwin only has one TLS scheme which must be capable of dealing with the
4042 : /// fully general situation, in the worst case. This means:
4043 : /// + "extern __thread" declaration.
4044 : /// + Defined in a possibly unknown dynamic library.
4045 : ///
4046 : /// The general system is that each __thread variable has a [3 x i64] descriptor
4047 : /// which contains information used by the runtime to calculate the address. The
4048 22 : /// only part of this the compiler needs to know about is the first xword, which
4049 : /// contains a function pointer that must be called with the address of the
4050 : /// entire descriptor in "x0".
4051 : ///
4052 : /// Since this descriptor may be in a different unit, in general even the
4053 2993 : /// descriptor must be accessed via an indirect load. The "ideal" code sequence
4054 : /// is:
4055 : /// adrp x0, _var@TLVPPAGE
4056 : /// ldr x0, [x0, _var@TLVPPAGEOFF] ; x0 now contains address of descriptor
4057 2993 : /// ldr x1, [x0] ; x1 contains 1st entry of descriptor,
4058 2993 : /// ; the function pointer
4059 2993 : /// blr x1 ; Uses descriptor address in x0
4060 : /// ; Address of _var is now in x0.
4061 2993 : ///
4062 2993 : /// If the address of _var's descriptor *is* known to the linker, then it can
4063 : /// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
4064 4 : /// a slight efficiency gain.
4065 : SDValue
4066 : AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
4067 : SelectionDAG &DAG) const {
4068 4 : assert(Subtarget->isTargetDarwin() &&
4069 4 : "This function expects a Darwin target");
4070 4 :
4071 : SDLoc DL(Op);
4072 4 : MVT PtrVT = getPointerTy(DAG.getDataLayout());
4073 4 : const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
4074 :
4075 171 : SDValue TLVPAddr =
4076 : DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4077 : SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
4078 :
4079 171 : // The first entry in the descriptor is a function pointer that we must call
4080 171 : // to obtain the address of the variable.
4081 171 : SDValue Chain = DAG.getEntryNode();
4082 : SDValue FuncTLVGet = DAG.getLoad(
4083 171 : MVT::i64, DL, Chain, DescAddr,
4084 171 : MachinePointerInfo::getGOT(DAG.getMachineFunction()),
4085 : /* Alignment = */ 8,
4086 26 : MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant |
4087 : MachineMemOperand::MODereferenceable);
4088 : Chain = FuncTLVGet.getValue(1);
4089 :
4090 26 : MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
4091 26 : MFI.setAdjustsStack(true);
4092 26 :
4093 : // TLS calls preserve all registers except those that absolutely must be
4094 26 : // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
4095 26 : // silly).
4096 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4097 2792 : const uint32_t *Mask = TRI->getTLSCallPreservedMask();
4098 : if (Subtarget->hasCustomCallingConv())
4099 : TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
4100 :
4101 2792 : // Finally, we can make the call. This is just a degenerate version of a
4102 2792 : // normal AArch64 call node: x0 takes the address of the descriptor, and
4103 2792 : // returns the address of the variable in this thread.
4104 : Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
4105 2792 : Chain =
4106 2792 : DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
4107 : Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
4108 : DAG.getRegisterMask(Mask), Chain.getValue(1));
4109 : return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
4110 : }
4111 97 :
4112 : /// When accessing thread-local variables under either the general-dynamic or
4113 : /// local-dynamic system, we make a "TLS-descriptor" call. The variable will
4114 : /// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
4115 97 : /// is a function pointer to carry out the resolution.
4116 97 : ///
4117 97 : /// The sequence is:
4118 : /// adrp x0, :tlsdesc:var
4119 1 : /// ldr x1, [x0, #:tlsdesc_lo12:var]
4120 : /// add x0, x0, #:tlsdesc_lo12:var
4121 : /// .tlsdesccall var
4122 : /// blr x1
4123 1 : /// (TPIDR_EL0 offset now in x0)
4124 1 : ///
4125 1 : /// The above sequence must be produced unscheduled, to enable the linker to
4126 : /// optimize/relax this sequence.
4127 7 : /// Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the
4128 : /// above sequence, and expanded really late in the compilation flow, to ensure
4129 : /// the sequence is produced as per above.
4130 : SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
4131 7 : const SDLoc &DL,
4132 7 : SelectionDAG &DAG) const {
4133 7 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
4134 :
4135 1 : SDValue Chain = DAG.getEntryNode();
4136 : SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4137 :
4138 : Chain =
4139 1 : DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
4140 1 : SDValue Glue = Chain.getValue(1);
4141 1 :
4142 : return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
4143 88 : }
4144 :
4145 : SDValue
4146 : AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
4147 88 : SelectionDAG &DAG) const {
4148 88 : assert(Subtarget->isTargetELF() && "This function expects an ELF target");
4149 88 : if (getTargetMachine().getCodeModel() == CodeModel::Large)
4150 : report_fatal_error("ELF TLS only supported in small memory model");
4151 : // Different choices can be made for the maximum size of the TLS area for a
4152 3117 : // module. For the small address model, the default TLS size is 16MiB and the
4153 : // maximum TLS size is 4GiB.
4154 : // FIXME: add -mtls-size command line option and make it control the 16MiB
4155 3117 : // vs. 4GiB code sequence generation.
4156 : // FIXME: add tiny codemodel support. We currently generate the same code as
4157 3117 : // small, which may be larger than needed.
4158 : const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4159 :
4160 : TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
4161 :
4162 : if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
4163 : if (Model == TLSModel::LocalDynamic)
4164 : Model = TLSModel::GeneralDynamic;
4165 3117 : }
4166 215 :
4167 : SDValue TPOff;
4168 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
4169 : SDLoc DL(Op);
4170 2902 : const GlobalValue *GV = GA->getGlobal();
4171 22 :
4172 2880 : SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
4173 88 :
4174 : if (Model == TLSModel::LocalExec) {
4175 2792 : SDValue HiVar = DAG.getTargetGlobalAddress(
4176 : GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4177 2902 : SDValue LoVar = DAG.getTargetGlobalAddress(
4178 : GV, DL, PtrVT, 0,
4179 2902 : AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4180 0 :
4181 0 : SDValue TPWithOff_lo =
4182 2902 : SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
4183 : HiVar,
4184 : DAG.getTargetConstant(0, DL, MVT::i32)),
4185 : 0);
4186 : SDValue TPWithOff =
4187 : SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPWithOff_lo,
4188 : LoVar,
4189 : DAG.getTargetConstant(0, DL, MVT::i32)),
4190 : 0);
4191 : return TPWithOff;
4192 : } else if (Model == TLSModel::InitialExec) {
4193 : TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4194 : TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
4195 : } else if (Model == TLSModel::LocalDynamic) {
4196 : // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
4197 : // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
4198 : // the beginning of the module's TLS region, followed by a DTPREL offset
4199 : // calculation.
4200 :
4201 : // These accesses will need deduplicating if there's more than one.
4202 : AArch64FunctionInfo *MFI =
4203 : DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
4204 : MFI->incNumLocalDynamicTLSAccesses();
4205 :
4206 : // The call needs a relocation too for linker relaxation. It doesn't make
4207 : // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4208 : // the address.
4209 : SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
4210 : AArch64II::MO_TLS);
4211 :
4212 : // Now we can calculate the offset from TPIDR_EL0 to this module's
4213 : // thread-local area.
4214 27 : TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4215 :
4216 : // Now use :dtprel_whatever: operations to calculate this variable's offset
4217 : // in its thread-storage area.
4218 : SDValue HiVar = DAG.getTargetGlobalAddress(
4219 : GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4220 27 : SDValue LoVar = DAG.getTargetGlobalAddress(
4221 27 : GV, DL, MVT::i64, 0,
4222 : AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4223 :
4224 27 : TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
4225 27 : DAG.getTargetConstant(0, DL, MVT::i32)),
4226 : 0);
4227 : TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
4228 : DAG.getTargetConstant(0, DL, MVT::i32)),
4229 27 : 0);
4230 : } else if (Model == TLSModel::GeneralDynamic) {
4231 : // The call needs a relocation too for linker relaxation. It doesn't make
4232 : // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4233 : // the address.
4234 : SDValue SymAddr =
4235 27 : DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4236 27 :
4237 : // Finally we can make a call to calculate the offset from tpidr_el0.
4238 27 : TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4239 : } else
4240 : llvm_unreachable("Unsupported ELF TLS access model");
4241 :
4242 : return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
4243 : }
4244 27 :
4245 27 : SDValue
4246 54 : AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
4247 0 : SelectionDAG &DAG) const {
4248 : assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering");
4249 :
4250 : SDValue Chain = DAG.getEntryNode();
4251 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
4252 27 : SDLoc DL(Op);
4253 27 :
4254 27 : SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
4255 :
4256 27 : // Load the ThreadLocalStoragePointer from the TEB
4257 27 : // A pointer to the TLS array is located at offset 0x58 from the TEB.
4258 : SDValue TLSArray =
4259 : DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
4260 : TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
4261 : Chain = TLSArray.getValue(1);
4262 :
4263 : // Load the TLS index from the C runtime;
4264 : // This does the same as getAddr(), but without having a GlobalAddressSDNode.
4265 : // This also does the same as LOADgot, but using a generic i32 load,
4266 : // while LOADgot only loads i64.
4267 : SDValue TLSIndexHi =
4268 : DAG.getTargetExternalSymbol("_tls_index", PtrVT, AArch64II::MO_PAGE);
4269 : SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
4270 : "_tls_index", PtrVT, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4271 : SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
4272 : SDValue TLSIndex =
4273 : DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
4274 : TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
4275 : Chain = TLSIndex.getValue(1);
4276 :
4277 : // The pointer to the thread's TLS data area is at the TLS Index scaled by 8
4278 32 : // offset into the TLSArray.
4279 : TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
4280 : SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
4281 32 : DAG.getConstant(3, DL, PtrVT));
4282 : SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
4283 32 : DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
4284 32 : MachinePointerInfo());
4285 : Chain = TLS.getValue(1);
4286 32 :
4287 64 : const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4288 32 : const GlobalValue *GV = GA->getGlobal();
4289 : SDValue TGAHi = DAG.getTargetGlobalAddress(
4290 32 : GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4291 : SDValue TGALo = DAG.getTargetGlobalAddress(
4292 : GV, DL, PtrVT, 0,
4293 : AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4294 51 :
4295 : // Add the offset from the start of the .tls section (section base).
4296 : SDValue Addr =
4297 51 : SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
4298 2 : DAG.getTargetConstant(0, DL, MVT::i32)),
4299 : 0);
4300 : Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
4301 : return Addr;
4302 : }
4303 :
4304 : SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
4305 : SelectionDAG &DAG) const {
4306 : const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4307 : if (DAG.getTarget().useEmulatedTLS())
4308 49 : return LowerToTLSEmulatedModel(GA, DAG);
4309 :
4310 49 : if (Subtarget->isTargetDarwin())
4311 31 : return LowerDarwinGlobalTLSAddress(Op, DAG);
4312 : if (Subtarget->isTargetELF())
4313 : return LowerELFGlobalTLSAddress(Op, DAG);
4314 : if (Subtarget->isTargetWindows())
4315 49 : return LowerWindowsGlobalTLSAddress(Op, DAG);
4316 49 :
4317 : llvm_unreachable("Unexpected platform trying to use TLS");
4318 49 : }
4319 :
4320 49 : SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4321 : SDValue Chain = Op.getOperand(0);
4322 49 : ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
4323 : SDValue LHS = Op.getOperand(2);
4324 9 : SDValue RHS = Op.getOperand(3);
4325 : SDValue Dest = Op.getOperand(4);
4326 : SDLoc dl(Op);
4327 9 :
4328 : // Handle f128 first, since lowering it will result in comparing the return
4329 : // value of a libcall against zero, which is just what the rest of LowerBR_CC
4330 9 : // is expecting to deal with.
4331 : if (LHS.getValueType() == MVT::f128) {
4332 : softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
4333 :
4334 : // If softenSetCCOperands returned a scalar, we need to compare the result
4335 9 : // against zero to select between true and false values.
4336 : if (!RHS.getNode()) {
4337 : RHS = DAG.getConstant(0, dl, LHS.getValueType());
4338 : CC = ISD::SETNE;
4339 9 : }
4340 40 : }
4341 8 :
4342 8 : // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
4343 32 : // instruction.
4344 : if (isOverflowIntrOpRes(LHS) && isOneConstant(RHS) &&
4345 : (CC == ISD::SETEQ || CC == ISD::SETNE)) {
4346 : // Only lower legal XALUO ops.
4347 : if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
4348 : return SDValue();
4349 :
4350 : // The actual operation with overflow check.
4351 12 : AArch64CC::CondCode OFCC;
4352 : SDValue Value, Overflow;
4353 : std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
4354 :
4355 : if (CC == ISD::SETNE)
4356 : OFCC = getInvertedCondCode(OFCC);
4357 : SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
4358 12 :
4359 : return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4360 : Overflow);
4361 : }
4362 12 :
4363 : if (LHS.getValueType().isInteger()) {
4364 : assert((LHS.getValueType() == RHS.getValueType()) &&
4365 : (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
4366 :
4367 12 : // If the RHS of the comparison is zero, we can potentially fold this
4368 : // to a specialized branch.
4369 : const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
4370 12 : if (RHSC && RHSC->getZExtValue() == 0) {
4371 : if (CC == ISD::SETEQ) {
4372 12 : // See if we can use a TBZ to fold in an AND as well.
4373 : // TBZ has a smaller branch displacement than CBZ. If the offset is
4374 : // out of bounds, a late MI-layer pass rewrites branches.
4375 12 : // 403.gcc is an example that hits this case.
4376 : if (LHS.getOpcode() == ISD::AND &&
4377 : isa<ConstantSDNode>(LHS.getOperand(1)) &&
4378 20 : isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4379 : SDValue Test = LHS.getOperand(0);
4380 : uint64_t Mask = LHS.getConstantOperandVal(1);
4381 : return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
4382 : DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4383 20 : Dest);
4384 : }
4385 :
4386 20 : return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
4387 : } else if (CC == ISD::SETNE) {
4388 0 : // See if we can use a TBZ to fold in an AND as well.
4389 : // TBZ has a smaller branch displacement than CBZ. If the offset is
4390 40 : // out of bounds, a late MI-layer pass rewrites branches.
4391 : // 403.gcc is an example that hits this case.
4392 : if (LHS.getOpcode() == ISD::AND &&
4393 : isa<ConstantSDNode>(LHS.getOperand(1)) &&
4394 5 : isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4395 : SDValue Test = LHS.getOperand(0);
4396 : uint64_t Mask = LHS.getConstantOperandVal(1);
4397 : return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
4398 5 : DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4399 5 : Dest);
4400 : }
4401 :
4402 5 : return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
4403 : } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
4404 : // Don't combine AND since emitComparison converts the AND to an ANDS
4405 : // (a.k.a. TST) and the test in the test bit and branch instruction
4406 : // becomes redundant. This would also increase register pressure.
4407 5 : uint64_t Mask = LHS.getValueSizeInBits() - 1;
4408 5 : return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
4409 5 : DAG.getConstant(Mask, dl, MVT::i64), Dest);
4410 : }
4411 : }
4412 : if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
4413 : LHS.getOpcode() != ISD::AND) {
4414 : // Don't combine AND since emitComparison converts the AND to an ANDS
4415 : // (a.k.a. TST) and the test in the test bit and branch instruction
4416 5 : // becomes redundant. This would also increase register pressure.
4417 : uint64_t Mask = LHS.getValueSizeInBits() - 1;
4418 5 : return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
4419 5 : DAG.getConstant(Mask, dl, MVT::i64), Dest);
4420 : }
4421 5 :
4422 5 : SDValue CCVal;
4423 5 : SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4424 : return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4425 : Cmp);
4426 : }
4427 5 :
4428 : assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
4429 5 : LHS.getValueType() == MVT::f64);
4430 :
4431 : // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
4432 5 : // clean. Some of them require two branches to implement.
4433 : SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4434 : AArch64CC::CondCode CC1, CC2;
4435 : changeFPCCToAArch64CC(CC, CC1, CC2);
4436 5 : SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4437 : SDValue BR1 =
4438 5 : DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
4439 : if (CC2 != AArch64CC::AL) {
4440 : SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4441 5 : return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
4442 : Cmp);
4443 : }
4444 :
4445 5 : return BR1;
4446 : }
4447 :
4448 5 : SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
4449 5 : SelectionDAG &DAG) const {
4450 : EVT VT = Op.getValueType();
4451 : SDLoc DL(Op);
4452 121 :
4453 : SDValue In1 = Op.getOperand(0);
4454 : SDValue In2 = Op.getOperand(1);
4455 121 : EVT SrcVT = In2.getValueType();
4456 38 :
4457 : if (SrcVT.bitsLT(VT))
4458 83 : In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
4459 27 : else if (SrcVT.bitsGT(VT))
4460 56 : In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
4461 51 :
4462 5 : EVT VecVT;
4463 5 : uint64_t EltMask;
4464 : SDValue VecVal1, VecVal2;
4465 0 :
4466 : auto setVecVal = [&] (int Idx) {
4467 : if (!VT.isVector()) {
4468 976 : VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4469 976 : DAG.getUNDEF(VecVT), In1);
4470 976 : VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4471 976 : DAG.getUNDEF(VecVT), In2);
4472 976 : } else {
4473 976 : VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
4474 : VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
4475 : }
4476 : };
4477 :
4478 : if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
4479 976 : VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32);
4480 2 : EltMask = 0x80000000ULL;
4481 : setVecVal(AArch64::ssub);
4482 : } else if (VT == MVT::f64 || VT == MVT::v2f64) {
4483 : VecVT = MVT::v2i64;
4484 1 :
4485 0 : // We want to materialize a mask with the high bit set, but the AdvSIMD
4486 0 : // immediate moves cannot materialize that in a single instruction for
4487 : // 64-bit elements. Instead, materialize zero and then negate it.
4488 : EltMask = 0;
4489 :
4490 : setVecVal(AArch64::dsub);
4491 : } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) {
4492 990 : VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16);
4493 14 : EltMask = 0x8000ULL;
4494 : setVecVal(AArch64::hsub);
4495 14 : } else {
4496 0 : llvm_unreachable("Invalid type for copysign!");
4497 : }
4498 :
4499 : SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
4500 :
4501 14 : // If we couldn't materialize the mask above, then the mask vector will be
4502 : // the zero vector, and we need to negate it here.
4503 14 : if (VT == MVT::f64 || VT == MVT::v2f64) {
4504 28 : BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
4505 14 : BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
4506 : BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
4507 : }
4508 14 :
4509 : SDValue Sel =
4510 : DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
4511 1924 :
4512 : if (VT == MVT::f16)
4513 : return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
4514 : if (VT == MVT::f32)
4515 : return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
4516 : else if (VT == MVT::f64)
4517 : return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
4518 1480 : else
4519 512 : return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
4520 : }
4521 :
4522 : SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
4523 : if (DAG.getMachineFunction().getFunction().hasFnAttribute(
4524 : Attribute::NoImplicitFloat))
4525 199 : return SDValue();
4526 :
4527 78 : if (!Subtarget->hasNEON())
4528 : return SDValue();
4529 :
4530 : // While there is no integer popcount instruction, it can
4531 78 : // be more efficiently lowered to the following sequence that uses
4532 : // AdvSIMD registers/instructions as long as the copies to/from
4533 : // the AdvSIMD registers are cheap.
4534 121 : // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
4535 313 : // CNT V0.8B, V0.8B // 8xbyte pop-counts
4536 : // ADDV B0, V0.8B // sum 8xbyte pop-counts
4537 : // UMOV X0, V0.B[0] // copy byte result back to integer reg
4538 : SDValue Val = Op.getOperand(0);
4539 : SDLoc DL(Op);
4540 : EVT VT = Op.getValueType();
4541 256 :
4542 : if (VT == MVT::i32 || VT == MVT::i64) {
4543 30 : if (VT == MVT::i32)
4544 : Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
4545 : Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
4546 :
4547 30 : SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
4548 : SDValue UaddLV = DAG.getNode(
4549 : ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
4550 226 : DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
4551 57 :
4552 : if (VT == MVT::i64)
4553 : UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
4554 : return UaddLV;
4555 29 : }
4556 :
4557 29 : assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
4558 : VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
4559 : "Unexpected type for custom ctpop lowering");
4560 690 :
4561 : EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4562 : Val = DAG.getBitcast(VT8Bit, Val);
4563 : Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
4564 :
4565 20 : // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
4566 : unsigned EltSize = 8;
4567 20 : unsigned NumElts = VT.is64BitVector() ? 8 : 16;
4568 : while (EltSize != VT.getScalarSizeInBits()) {
4569 : EltSize *= 2;
4570 414 : NumElts /= 2;
4571 414 : MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
4572 : Val = DAG.getNode(
4573 414 : ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
4574 : DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
4575 : }
4576 :
4577 : return Val;
4578 : }
4579 :
4580 : SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
4581 44 :
4582 : if (Op.getValueType().isVector())
4583 44 : return LowerVSETCC(Op, DAG);
4584 44 :
4585 : SDValue LHS = Op.getOperand(0);
4586 44 : SDValue RHS = Op.getOperand(1);
4587 44 : ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
4588 5 : SDLoc dl(Op);
4589 :
4590 5 : // We chose ZeroOrOneBooleanContents, so use zero and one.
4591 : EVT VT = Op.getValueType();
4592 : SDValue TVal = DAG.getConstant(1, dl, VT);
4593 39 : SDValue FVal = DAG.getConstant(0, dl, VT);
4594 :
4595 : // Handle f128 first, since one possible outcome is a normal integer
4596 29 : // comparison which gets picked up by the next if statement.
4597 : if (LHS.getValueType() == MVT::f128) {
4598 29 : softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
4599 :
4600 : // If softenSetCCOperands returned a scalar, use it.
4601 29 : if (!RHS.getNode()) {
4602 29 : assert(LHS.getValueType() == Op.getValueType() &&
4603 29 : "Unexpected setcc expansion!");
4604 : return LHS;
4605 29 : }
4606 7 : }
4607 22 :
4608 10 : if (LHS.getValueType().isInteger()) {
4609 : SDValue CCVal;
4610 29 : SDValue Cmp =
4611 : getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl);
4612 29 :
4613 : // Note that we inverted the condition above, so we reverse the order of
4614 : // the true and false operands here. This will allow the setcc to be
4615 : // matched to a single CSINC instruction.
4616 : return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
4617 : }
4618 :
4619 : // Now we know we're dealing with FP values.
4620 : assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
4621 : LHS.getValueType() == MVT::f64);
4622 :
4623 : // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead
4624 29 : // and do the comparison.
4625 : SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4626 :
4627 15 : AArch64CC::CondCode CC1, CC2;
4628 : changeFPCCToAArch64CC(CC, CC1, CC2);
4629 15 : if (CC2 == AArch64CC::AL) {
4630 : changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2);
4631 10 : SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4632 :
4633 : // Note that we inverted the condition above, so we reverse the order of
4634 : // the true and false operands here. This will allow the setcc to be
4635 : // matched to a single CSINC instruction.
4636 : return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
4637 : } else {
4638 10 : // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
4639 : // totally clean. Some of them require two CSELs to implement. As is in
4640 4 : // this case, we emit the first CSEL and then emit a second using the output
4641 : // of the first as the RHS. We're effectively OR'ing the two CC's together.
4642 4 :
4643 : // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
4644 0 : SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4645 : SDValue CS1 =
4646 : DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
4647 29 :
4648 : SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4649 : return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
4650 : }
4651 : }
4652 10 :
4653 10 : SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
4654 10 : SDValue RHS, SDValue TVal,
4655 : SDValue FVal, const SDLoc &dl,
4656 : SelectionDAG &DAG) const {
4657 : // Handle f128 first, because it will result in a comparison of some RTLIB
4658 29 : // call result against zero.
4659 : if (LHS.getValueType() == MVT::f128) {
4660 : softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl);
4661 4 :
4662 : // If softenSetCCOperands returned a scalar, we need to compare the result
4663 11 : // against zero to select between true and false values.
4664 : if (!RHS.getNode()) {
4665 4 : RHS = DAG.getConstant(0, dl, LHS.getValueType());
4666 : CC = ISD::SETNE;
4667 10 : }
4668 : }
4669 :
4670 23 : // Also handle f16, for which we need to do a f32 comparison.
4671 23 : if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
4672 : LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
4673 4 : RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
4674 : }
4675 19 :
4676 3 : // Next, handle integers.
4677 : if (LHS.getValueType().isInteger()) {
4678 : assert((LHS.getValueType() == RHS.getValueType()) &&
4679 : (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64));
4680 :
4681 : unsigned Opcode = AArch64ISD::CSEL;
4682 :
4683 : // If both the TVal and the FVal are constants, see if we can swap them in
4684 : // order to for a CSINV or CSINC out of them.
4685 : ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
4686 16 : ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
4687 :
4688 16 : if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
4689 : std::swap(TVal, FVal);
4690 : std::swap(CTVal, CFVal);
4691 : CC = ISD::getSetCCInverse(CC, true);
4692 3 : } else if (CTVal && CFVal && CTVal->isOne() && CFVal->isNullValue()) {
4693 5 : std::swap(TVal, FVal);
4694 : std::swap(CTVal, CFVal);
4695 5 : CC = ISD::getSetCCInverse(CC, true);
4696 : } else if (TVal.getOpcode() == ISD::XOR) {
4697 : // If TVal is a NOT we want to swap TVal and FVal so that we can match
4698 5 : // with a CSINV rather than a CSEL.
4699 : if (isAllOnesConstant(TVal.getOperand(1))) {
4700 : std::swap(TVal, FVal);
4701 2 : std::swap(CTVal, CFVal);
4702 5 : CC = ISD::getSetCCInverse(CC, true);
4703 : }
4704 : } else if (TVal.getOpcode() == ISD::SUB) {
4705 : // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
4706 : // that we can match with a CSNEG rather than a CSEL.
4707 : if (isNullConstant(TVal.getOperand(0))) {
4708 : std::swap(TVal, FVal);
4709 11 : std::swap(CTVal, CFVal);
4710 11 : CC = ISD::getSetCCInverse(CC, true);
4711 11 : }
4712 : } else if (CTVal && CFVal) {
4713 : const int64_t TrueVal = CTVal->getSExtValue();
4714 : const int64_t FalseVal = CFVal->getSExtValue();
4715 11 : bool Swap = false;
4716 37 :
4717 26 : // If both TVal and FVal are constants, see if FVal is the
4718 26 : // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
4719 26 : // instead of a CSEL in that case.
4720 26 : if (TrueVal == ~FalseVal) {
4721 : Opcode = AArch64ISD::CSINV;
4722 26 : } else if (TrueVal == -FalseVal) {
4723 : Opcode = AArch64ISD::CSNEG;
4724 : } else if (TVal.getValueType() == MVT::i32) {
4725 11 : // If our operands are only 32-bit wide, make sure we use 32-bit
4726 : // arithmetic for the check whether we can use CSINC. This ensures that
4727 : // the addition in the check will wrap around properly in case there is
4728 816 : // an overflow (which would not be the case if we do the check with
4729 : // 64-bit arithmetic).
4730 1632 : const uint32_t TrueVal32 = CTVal->getZExtValue();
4731 564 : const uint32_t FalseVal32 = CFVal->getZExtValue();
4732 :
4733 252 : if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
4734 252 : Opcode = AArch64ISD::CSINC;
4735 252 :
4736 : if (TrueVal32 > FalseVal32) {
4737 : Swap = true;
4738 : }
4739 252 : }
4740 252 : // 64-bit check whether we can use CSINC.
4741 252 : } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
4742 : Opcode = AArch64ISD::CSINC;
4743 :
4744 : if (TrueVal > FalseVal) {
4745 252 : Swap = true;
4746 10 : }
4747 : }
4748 :
4749 5 : // Swap TVal and FVal if necessary.
4750 : if (Swap) {
4751 : std::swap(TVal, FVal);
4752 1 : std::swap(CTVal, CFVal);
4753 : CC = ISD::getSetCCInverse(CC, true);
4754 : }
4755 :
4756 502 : if (Opcode != AArch64ISD::CSEL) {
4757 210 : // Drop FVal since we can get its value by simply inverting/negating
4758 : // TVal.
4759 210 : FVal = TVal;
4760 : }
4761 : }
4762 :
4763 : // Avoid materializing a constant when possible by reusing a known value in
4764 210 : // a register. However, don't perform this optimization if the known value
4765 : // is one, zero or negative one in the case of a CSEL. We can always
4766 : // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the
4767 : // FVal, respectively.
4768 : ConstantSDNode *RHSVal = dyn_cast<ConstantSDNode>(RHS);
4769 : if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() &&
4770 : !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) {
4771 : AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
4772 : // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to
4773 41 : // "a != C ? x : a" to avoid materializing C.
4774 : if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ)
4775 : TVal = LHS;
4776 41 : else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE)
4777 41 : FVal = LHS;
4778 37 : } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) {
4779 37 : assert (CTVal && CFVal && "Expected constant operands for CSNEG.");
4780 : // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to
4781 : // avoid materializing C.
4782 : AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
4783 : if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
4784 37 : Opcode = AArch64ISD::CSINV;
4785 : TVal = LHS;
4786 : FVal = DAG.getConstant(0, dl, FVal.getValueType());
4787 : }
4788 : }
4789 :
4790 : SDValue CCVal;
4791 : SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4792 4 : EVT VT = TVal.getValueType();
4793 : return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
4794 4 : }
4795 :
4796 4 : // Now we know we're dealing with FP values.
4797 4 : assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||
4798 : LHS.getValueType() == MVT::f64);
4799 : assert(LHS.getValueType() == RHS.getValueType());
4800 : EVT VT = TVal.getValueType();
4801 592 : SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4802 :
4803 : // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
4804 : // clean. Some of them require two CSELs to implement.
4805 : AArch64CC::CondCode CC1, CC2;
4806 : changeFPCCToAArch64CC(CC, CC1, CC2);
4807 592 :
4808 0 : if (DAG.getTarget().Options.UnsafeFPMath) {
4809 : // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and
4810 : // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0.
4811 : ConstantFPSDNode *RHSVal = dyn_cast<ConstantFPSDNode>(RHS);
4812 0 : if (RHSVal && RHSVal->isZero()) {
4813 0 : ConstantFPSDNode *CFVal = dyn_cast<ConstantFPSDNode>(FVal);
4814 0 : ConstantFPSDNode *CTVal = dyn_cast<ConstantFPSDNode>(TVal);
4815 :
4816 : if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
4817 : CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType())
4818 : TVal = LHS;
4819 592 : else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) &&
4820 114 : CFVal && CFVal->isZero() &&
4821 114 : FVal.getValueType() == LHS.getValueType())
4822 : FVal = LHS;
4823 : }
4824 : }
4825 1184 :
4826 : // Emit first, and possibly only, CSEL.
4827 : SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4828 : SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
4829 :
4830 : // If we need a second CSEL, emit it, using the output of the first as the
4831 : // RHS. We're effectively OR'ing the two CC's together.
4832 : if (CC2 != AArch64CC::AL) {
4833 : SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4834 : return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
4835 : }
4836 118 :
4837 : // Otherwise, return the output of the first CSEL.
4838 : return CS1;
4839 8 : }
4840 284 :
4841 : SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
4842 : SelectionDAG &DAG) const {
4843 0 : ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
4844 240 : SDValue LHS = Op.getOperand(0);
4845 : SDValue RHS = Op.getOperand(1);
4846 : SDValue TVal = Op.getOperand(2);
4847 5 : SDValue FVal = Op.getOperand(3);
4848 : SDLoc DL(Op);
4849 : return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
4850 5 : }
4851 :
4852 235 : SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
4853 : SelectionDAG &DAG) const {
4854 : SDValue CCVal = Op->getOperand(0);
4855 6 : SDValue TVal = Op->getOperand(1);
4856 : SDValue FVal = Op->getOperand(2);
4857 : SDLoc DL(Op);
4858 6 :
4859 : // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
4860 229 : // instruction.
4861 36 : if (isOverflowIntrOpRes(CCVal)) {
4862 36 : // Only lower legal XALUO ops.
4863 : if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
4864 : return SDValue();
4865 :
4866 : AArch64CC::CondCode OFCC;
4867 : SDValue Value, Overflow;
4868 36 : std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
4869 : SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
4870 34 :
4871 : return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
4872 : CCVal, Overflow);
4873 : }
4874 :
4875 : // Lower it the same way as we would lower a SELECT_CC node.
4876 : ISD::CondCode CC;
4877 : SDValue LHS, RHS;
4878 16 : if (CCVal.getOpcode() == ISD::SETCC) {
4879 16 : LHS = CCVal.getOperand(0);
4880 : RHS = CCVal.getOperand(1);
4881 16 : CC = cast<CondCodeSDNode>(CCVal->getOperand(2))->get();
4882 : } else {
4883 : LHS = CCVal;
4884 6 : RHS = DAG.getConstant(0, DL, CCVal.getValueType());
4885 : CC = ISD::SETNE;
4886 : }
4887 : return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
4888 : }
4889 15 :
4890 : SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
4891 : SelectionDAG &DAG) const {
4892 4 : // Jump table entries as PC relative offsets. No additional tweaking
4893 : // is necessary here. Just get the address of the jump table.
4894 : JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
4895 :
4896 : if (getTargetMachine().getCodeModel() == CodeModel::Large &&
4897 : !Subtarget->isTargetMachO()) {
4898 : return getAddrLarge(JT, DAG);
4899 : } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4900 : return getAddrTiny(JT, DAG);
4901 5 : }
4902 : return getAddr(JT, DAG);
4903 : }
4904 36 :
4905 : SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
4906 : SelectionDAG &DAG) const {
4907 15 : ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
4908 :
4909 : if (getTargetMachine().getCodeModel() == CodeModel::Large) {
4910 : // Use the GOT for the large code model on iOS.
4911 : if (Subtarget->isTargetMachO()) {
4912 : return getGOT(CP, DAG);
4913 : }
4914 : return getAddrLarge(CP, DAG);
4915 : } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4916 : return getAddrTiny(CP, DAG);
4917 406 : } else {
4918 286 : return getAddr(CP, DAG);
4919 33 : }
4920 : }
4921 :
4922 33 : SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
4923 4 : SelectionDAG &DAG) const {
4924 29 : BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Op);
4925 2 : if (getTargetMachine().getCodeModel() == CodeModel::Large &&
4926 218 : !Subtarget->isTargetMachO()) {
4927 : return getAddrLarge(BA, DAG);
4928 : } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4929 : return getAddrTiny(BA, DAG);
4930 1 : }
4931 1 : return getAddr(BA, DAG);
4932 : }
4933 1 :
4934 1 : SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
4935 : SelectionDAG &DAG) const {
4936 : AArch64FunctionInfo *FuncInfo =
4937 : DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
4938 248 :
4939 248 : SDLoc DL(Op);
4940 248 : SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
4941 248 : getPointerTy(DAG.getDataLayout()));
4942 : const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4943 : return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
4944 : MachinePointerInfo(SV));
4945 : }
4946 :
4947 : SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
4948 344 : SelectionDAG &DAG) const {
4949 344 : AArch64FunctionInfo *FuncInfo =
4950 : DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
4951 :
4952 : SDLoc DL(Op);
4953 : SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
4954 344 : ? FuncInfo->getVarArgsGPRIndex()
4955 : : FuncInfo->getVarArgsStackIndex(),
4956 344 : getPointerTy(DAG.getDataLayout()));
4957 : const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4958 : return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
4959 : MachinePointerInfo(SV));
4960 26 : }
4961 :
4962 : SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
4963 : SelectionDAG &DAG) const {
4964 13 : // The layout of the va_list struct is specified in the AArch64 Procedure Call
4965 23 : // Standard, section B.3.
4966 : MachineFunction &MF = DAG.getMachineFunction();
4967 9 : AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4968 17 : auto PtrVT = getPointerTy(DAG.getDataLayout());
4969 4 : SDLoc DL(Op);
4970 4 :
4971 : SDValue Chain = Op.getOperand(0);
4972 : SDValue VAList = Op.getOperand(1);
4973 : const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4974 : SmallVector<SDValue, 4> MemOps;
4975 344 :
4976 344 : // void *__stack at offset 0
4977 : SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
4978 : MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
4979 : MachinePointerInfo(SV), /* Alignment = */ 8));
4980 344 :
4981 50 : // void *__gr_top at offset 8
4982 50 : int GPRSize = FuncInfo->getVarArgsGPRSize();
4983 : if (GPRSize > 0) {
4984 : SDValue GRTop, GRTopAddr;
4985 :
4986 294 : GRTopAddr =
4987 : DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT));
4988 :
4989 244 : GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
4990 : GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
4991 244 : DAG.getConstant(GPRSize, DL, PtrVT));
4992 244 :
4993 244 : MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
4994 244 : MachinePointerInfo(SV, 8),
4995 244 : /* Alignment = */ 8));
4996 : }
4997 244 :
4998 : // void *__vr_top at offset 16
4999 : int FPRSize = FuncInfo->getVarArgsFPRSize();
5000 367 : if (FPRSize > 0) {
5001 : SDValue VRTop, VRTopAddr;
5002 367 : VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5003 367 : DAG.getConstant(16, DL, PtrVT));
5004 367 :
5005 : VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
5006 : VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
5007 : DAG.getConstant(FPRSize, DL, PtrVT));
5008 :
5009 : MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
5010 : MachinePointerInfo(SV, 16),
5011 19 : /* Alignment = */ 8));
5012 0 : }
5013 :
5014 : // int __gr_offs at offset 24
5015 : SDValue GROffsAddr =
5016 19 : DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT));
5017 19 : MemOps.push_back(DAG.getStore(
5018 : Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32), GROffsAddr,
5019 : MachinePointerInfo(SV, 24), /* Alignment = */ 4));
5020 19 :
5021 : // int __vr_offs at offset 28
5022 : SDValue VROffsAddr =
5023 : DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT));
5024 : MemOps.push_back(DAG.getStore(
5025 348 : Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32), VROffsAddr,
5026 348 : MachinePointerInfo(SV, 28), /* Alignment = */ 4));
5027 280 :
5028 280 : return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
5029 280 : }
5030 :
5031 68 : SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
5032 68 : SelectionDAG &DAG) const {
5033 : MachineFunction &MF = DAG.getMachineFunction();
5034 :
5035 348 : if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
5036 : return LowerWin64_VASTART(Op, DAG);
5037 : else if (Subtarget->isTargetDarwin())
5038 28 : return LowerDarwin_VASTART(Op, DAG);
5039 : else
5040 : return LowerAAPCS_VASTART(Op, DAG);
5041 : }
5042 :
5043 : SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
5044 28 : SelectionDAG &DAG) const {
5045 1 : // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
5046 1 : // pointer.
5047 27 : SDLoc DL(Op);
5048 1 : unsigned VaListSize =
5049 : Subtarget->isTargetDarwin() || Subtarget->isTargetWindows() ? 8 : 32;
5050 26 : const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
5051 : const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
5052 :
5053 182 : return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1),
5054 : Op.getOperand(2),
5055 : DAG.getConstant(VaListSize, DL, MVT::i32),
5056 : 8, false, false, false, MachinePointerInfo(DestSV),
5057 182 : MachinePointerInfo(SrcSV));
5058 : }
5059 8 :
5060 0 : SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
5061 : assert(Subtarget->isTargetDarwin() &&
5062 4 : "automatic va_arg instruction only works on Darwin");
5063 178 :
5064 7 : const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5065 : EVT VT = Op.getValueType();
5066 171 : SDLoc DL(Op);
5067 : SDValue Chain = Op.getOperand(0);
5068 : SDValue Addr = Op.getOperand(1);
5069 : unsigned Align = Op.getConstantOperandVal(3);
5070 7 : auto PtrVT = getPointerTy(DAG.getDataLayout());
5071 :
5072 : SDValue VAList = DAG.getLoad(PtrVT, DL, Chain, Addr, MachinePointerInfo(V));
5073 7 : Chain = VAList.getValue(1);
5074 2 :
5075 2 : if (Align > 8) {
5076 5 : assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2");
5077 1 : VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5078 : DAG.getConstant(Align - 1, DL, PtrVT));
5079 4 : VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
5080 : DAG.getConstant(-(int64_t)Align, DL, PtrVT));
5081 : }
5082 8 :
5083 : Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
5084 : uint64_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
5085 8 :
5086 : // Scalar integer and FP values smaller than 64 bits are implicitly extended
5087 : // up to 64 bits. At the very least, we have to increase the striding of the
5088 : // vaargs list to match this, and for FP values we need to introduce
5089 16 : // FP_ROUND nodes as well.
5090 8 : if (VT.isInteger() && !VT.isVector())
5091 : ArgSize = 8;
5092 8 : bool NeedFPTrunc = false;
5093 : if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
5094 : ArgSize = 8;
5095 12 : NeedFPTrunc = true;
5096 : }
5097 :
5098 12 : // Increment the pointer, VAList, to the next vaarg
5099 : SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5100 : DAG.getConstant(ArgSize, DL, PtrVT));
5101 12 : // Store the incremented VAList to the legalized pointer
5102 8 : SDValue APStore =
5103 4 : DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
5104 36 :
5105 12 : // Load the actual argument out of the pointer VAList
5106 : if (NeedFPTrunc) {
5107 12 : // Load the value as an f64.
5108 : SDValue WideFP =
5109 : DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
5110 5 : // Round the value down to an f32.
5111 : SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
5112 : DAG.getIntPtrConstant(1, DL));
5113 : SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
5114 5 : // Merge the rounded value with the chain output of the load.
5115 5 : return DAG.getMergeValues(Ops, DL);
5116 5 : }
5117 :
5118 : return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
5119 5 : }
5120 5 :
5121 5 : SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
5122 : SelectionDAG &DAG) const {
5123 : MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5124 : MFI.setFrameAddressIsTaken(true);
5125 5 :
5126 5 : EVT VT = Op.getValueType();
5127 5 : SDLoc DL(Op);
5128 : unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5129 : SDValue FrameAddr =
5130 5 : DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
5131 5 : while (Depth--)
5132 : FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
5133 : MachinePointerInfo());
5134 3 : return FrameAddr;
5135 3 : }
5136 :
5137 3 : // FIXME? Maybe this could be a TableGen attribute on some registers and
5138 3 : // this table could be generated automatically from RegInfo.
5139 3 : unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT,
5140 : SelectionDAG &DAG) const {
5141 3 : unsigned Reg = StringSwitch<unsigned>(RegName)
5142 : .Case("sp", AArch64::SP)
5143 6 : .Case("x1", AArch64::X1)
5144 : .Case("w1", AArch64::W1)
5145 : .Case("x2", AArch64::X2)
5146 : .Case("w2", AArch64::W2)
5147 5 : .Case("x3", AArch64::X3)
5148 5 : .Case("w3", AArch64::W3)
5149 : .Case("x4", AArch64::X4)
5150 4 : .Case("w4", AArch64::W4)
5151 4 : .Case("x5", AArch64::X5)
5152 : .Case("w5", AArch64::W5)
5153 4 : .Case("x6", AArch64::X6)
5154 4 : .Case("w6", AArch64::W6)
5155 4 : .Case("x7", AArch64::X7)
5156 : .Case("w7", AArch64::W7)
5157 4 : .Case("x18", AArch64::X18)
5158 : .Case("w18", AArch64::W18)
5159 8 : .Case("x20", AArch64::X20)
5160 : .Case("w20", AArch64::W20)
5161 : .Default(0);
5162 : if (((Reg == AArch64::X1 || Reg == AArch64::W1) &&
5163 : !Subtarget->isXRegisterReserved(1)) ||
5164 5 : ((Reg == AArch64::X2 || Reg == AArch64::W2) &&
5165 5 : !Subtarget->isXRegisterReserved(2)) ||
5166 5 : ((Reg == AArch64::X3 || Reg == AArch64::W3) &&
5167 10 : !Subtarget->isXRegisterReserved(3)) ||
5168 : ((Reg == AArch64::X4 || Reg == AArch64::W4) &&
5169 : !Subtarget->isXRegisterReserved(4)) ||
5170 : ((Reg == AArch64::X5 || Reg == AArch64::W5) &&
5171 5 : !Subtarget->isXRegisterReserved(5)) ||
5172 5 : ((Reg == AArch64::X6 || Reg == AArch64::W6) &&
5173 5 : !Subtarget->isXRegisterReserved(6)) ||
5174 10 : ((Reg == AArch64::X7 || Reg == AArch64::W7) &&
5175 : !Subtarget->isXRegisterReserved(7)) ||
5176 5 : ((Reg == AArch64::X18 || Reg == AArch64::W18) &&
5177 : !Subtarget->isXRegisterReserved(18)) ||
5178 : ((Reg == AArch64::X20 || Reg == AArch64::W20) &&
5179 25 : !Subtarget->isXRegisterReserved(20)))
5180 : Reg = 0;
5181 25 : if (Reg)
5182 : return Reg;
5183 25 : report_fatal_error(Twine("Invalid register name \""
5184 12 : + StringRef(RegName) + "\"."));
5185 : }
5186 8 :
5187 : SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
5188 5 : SelectionDAG &DAG) const {
5189 : MachineFunction &MF = DAG.getMachineFunction();
5190 : MachineFrameInfo &MFI = MF.getFrameInfo();
5191 2 : MFI.setReturnAddressIsTaken(true);
5192 :
5193 : EVT VT = Op.getValueType();
5194 : SDLoc DL(Op);
5195 : unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5196 : if (Depth) {
5197 4 : SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5198 2 : SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5199 2 : return DAG.getLoad(VT, DL, DAG.getEntryNode(),
5200 : DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
5201 : MachinePointerInfo());
5202 : }
5203 :
5204 : // Return LR, which contains the return address. Mark it an implicit live-in.
5205 2 : unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
5206 : return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
5207 : }
5208 16 :
5209 : /// LowerShiftRightParts - Lower SRA_PARTS, which returns two
5210 : /// i64 values and take a 2 x i64 value to shift plus a shift amount.
5211 : SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
5212 16 : SelectionDAG &DAG) const {
5213 16 : assert(Op.getNumOperands() == 3 && "Not a double-shift!");
5214 : EVT VT = Op.getValueType();
5215 16 : unsigned VTBits = VT.getSizeInBits();
5216 16 : SDLoc dl(Op);
5217 16 : SDValue ShOpLo = Op.getOperand(0);
5218 16 : SDValue ShOpHi = Op.getOperand(1);
5219 : SDValue ShAmt = Op.getOperand(2);
5220 16 : unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
5221 16 :
5222 : assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS);
5223 16 :
5224 : SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5225 1 : DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5226 1 : SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
5227 1 :
5228 1 : // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which
5229 : // is "undef". We wanted 0, so CSEL it directly.
5230 : SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5231 16 : ISD::SETEQ, dl, DAG);
5232 16 : SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5233 : HiBitsForLo =
5234 : DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5235 : HiBitsForLo, CCVal, Cmp);
5236 :
5237 : SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5238 31 : DAG.getConstant(VTBits, dl, MVT::i64));
5239 :
5240 : SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
5241 17 : SDValue LoForNormalShift =
5242 : DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo);
5243 :
5244 : Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5245 : dl, DAG);
5246 : CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5247 : SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
5248 16 : SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5249 : LoForNormalShift, CCVal, Cmp);
5250 :
5251 16 : // AArch64 shifts larger than the register width are wrapped rather than
5252 : // clamped, so we can't just emit "hi >> x".
5253 : SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
5254 16 : SDValue HiForBigShift =
5255 : Opc == ISD::SRA
5256 : ? DAG.getNode(Opc, dl, VT, ShOpHi,
5257 1 : DAG.getConstant(VTBits - 1, dl, MVT::i64))
5258 : : DAG.getConstant(0, dl, VT);
5259 : SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5260 1 : HiForNormalShift, CCVal, Cmp);
5261 1 :
5262 : SDValue Ops[2] = { Lo, Hi };
5263 1 : return DAG.getMergeValues(Ops, dl);
5264 : }
5265 :
5266 15 : /// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
5267 : /// i64 values and take a 2 x i64 value to shift plus a shift amount.
5268 : SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
5269 4 : SelectionDAG &DAG) const {
5270 : assert(Op.getNumOperands() == 3 && "Not a double-shift!");
5271 4 : EVT VT = Op.getValueType();
5272 : unsigned VTBits = VT.getSizeInBits();
5273 : SDLoc dl(Op);
5274 4 : SDValue ShOpLo = Op.getOperand(0);
5275 : SDValue ShOpHi = Op.getOperand(1);
5276 8 : SDValue ShAmt = Op.getOperand(2);
5277 :
5278 4 : assert(Op.getOpcode() == ISD::SHL_PARTS);
5279 10 : SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5280 6 : DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5281 6 : SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
5282 4 :
5283 : // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which
5284 : // is "undef". We wanted 0, so CSEL it directly.
5285 : SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5286 : ISD::SETEQ, dl, DAG);
5287 10 : SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5288 : LoBitsForHi =
5289 10 : DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5290 10 : LoBitsForHi, CCVal, Cmp);
5291 10 :
5292 10 : SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5293 10 : DAG.getConstant(VTBits, dl, MVT::i64));
5294 10 : SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
5295 10 : SDValue HiForNormalShift =
5296 10 : DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi);
5297 10 :
5298 10 : SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
5299 10 :
5300 10 : Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5301 10 : dl, DAG);
5302 10 : CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5303 10 : SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5304 10 : HiForNormalShift, CCVal, Cmp);
5305 10 :
5306 10 : // AArch64 shifts of larger than register sizes are wrapped rather than
5307 10 : // clamped, so we can't just emit "lo << a" if a is too big.
5308 10 : SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
5309 : SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5310 0 : SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5311 0 : LoForNormalShift, CCVal, Cmp);
5312 10 :
5313 0 : SDValue Ops[2] = { Lo, Hi };
5314 10 : return DAG.getMergeValues(Ops, dl);
5315 0 : }
5316 10 :
5317 0 : bool AArch64TargetLowering::isOffsetFoldingLegal(
5318 10 : const GlobalAddressSDNode *GA) const {
5319 2 : // Offsets are folded in the DAG combine rather than here so that we can
5320 8 : // intelligently choose an offset based on the uses.
5321 0 : return false;
5322 8 : }
5323 0 :
5324 8 : bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const {
5325 12 : // We can materialize #0.0 as fmov $Rd, XZR for 64-bit and 32-bit cases.
5326 8 : // FIXME: We should be able to handle f128 as well with a clever lowering.
5327 0 : if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32 ||
5328 : (VT == MVT::f16 && Subtarget->hasFullFP16()))) {
5329 8 : LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString() << " imm value: 0\n");
5330 6 : return true;
5331 4 : }
5332 :
5333 : bool IsLegal = false;
5334 : SmallString<128> ImmStrVal;
5335 6 : Imm.toString(ImmStrVal);
5336 :
5337 6 : if (VT == MVT::f64)
5338 6 : IsLegal = AArch64_AM::getFP64Imm(Imm) != -1;
5339 : else if (VT == MVT::f32)
5340 : IsLegal = AArch64_AM::getFP32Imm(Imm) != -1;
5341 6 : else if (VT == MVT::f16 && Subtarget->hasFullFP16())
5342 : IsLegal = AArch64_AM::getFP16Imm(Imm) != -1;
5343 6 :
5344 6 : if (IsLegal) {
5345 2 : LLVM_DEBUG(dbgs() << "Legal " << VT.getEVTString()
5346 4 : << " imm value: " << ImmStrVal << "\n");
5347 : return true;
5348 : }
5349 2 :
5350 : LLVM_DEBUG(dbgs() << "Illegal " << VT.getEVTString()
5351 : << " imm value: " << ImmStrVal << "\n");
5352 : return false;
5353 4 : }
5354 4 :
5355 : //===----------------------------------------------------------------------===//
5356 : // AArch64 Optimization Hooks
5357 : //===----------------------------------------------------------------------===//
5358 :
5359 2 : static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
5360 : SDValue Operand, SelectionDAG &DAG,
5361 : int &ExtraSteps) {
5362 2 : EVT VT = Operand.getValueType();
5363 2 : if (ST->hasNEON() &&
5364 : (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
5365 2 : VT == MVT::f32 || VT == MVT::v1f32 ||
5366 2 : VT == MVT::v2f32 || VT == MVT::v4f32)) {
5367 2 : if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
5368 2 : // For the reciprocal estimates, convergence is quadratic, so the number
5369 : // of digits is doubled after each iteration. In ARMv8, the accuracy of
5370 : // the initial estimate is 2^-8. Thus the number of extra steps to refine
5371 : // the result for float (23 mantissa bits) is 2 and for double (52
5372 : // mantissa bits) is 3.
5373 2 : ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
5374 2 :
5375 : return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
5376 : }
5377 :
5378 : return SDValue();
5379 2 : }
5380 2 :
5381 2 : SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
5382 2 : SelectionDAG &DAG, int Enabled,
5383 2 : int &ExtraSteps,
5384 : bool &UseOneConst,
5385 : bool Reciprocal) const {
5386 2 : if (Enabled == ReciprocalEstimate::Enabled ||
5387 : (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt()))
5388 2 : if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
5389 : DAG, ExtraSteps)) {
5390 2 : SDLoc DL(Operand);
5391 : EVT VT = Operand.getValueType();
5392 2 :
5393 2 : SDNodeFlags Flags;
5394 2 : Flags.setAllowReassociation(true);
5395 2 :
5396 : // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
5397 2 : // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
5398 : for (int i = ExtraSteps; i > 0; --i) {
5399 : SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
5400 : Flags);
5401 2 : Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
5402 : Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5403 : }
5404 : if (!Reciprocal) {
5405 1 : EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
5406 2 : VT);
5407 : SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5408 2 : SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
5409 :
5410 2 : Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
5411 2 : // Correct the result if the operand is 0.0.
5412 : Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL,
5413 : VT, Eq, Operand, Estimate);
5414 : }
5415 :
5416 1 : ExtraSteps = 0;
5417 : return Estimate;
5418 : }
5419 1 :
5420 1 : return SDValue();
5421 : }
5422 1 :
5423 1 : SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
5424 1 : SelectionDAG &DAG, int Enabled,
5425 : int &ExtraSteps) const {
5426 : if (Enabled == ReciprocalEstimate::Enabled)
5427 : if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
5428 1 : DAG, ExtraSteps)) {
5429 1 : SDLoc DL(Operand);
5430 : EVT VT = Operand.getValueType();
5431 :
5432 : SDNodeFlags Flags;
5433 : Flags.setAllowReassociation(true);
5434 1 :
5435 1 : // Newton reciprocal iteration: E * (2 - X * E)
5436 1 : // AArch64 reciprocal iteration instruction: (2 - M * N)
5437 1 : for (int i = ExtraSteps; i > 0; --i) {
5438 1 : SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
5439 : Estimate, Flags);
5440 : Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5441 1 : }
5442 1 :
5443 : ExtraSteps = 0;
5444 1 : return Estimate;
5445 : }
5446 1 :
5447 : return SDValue();
5448 1 : }
5449 1 :
5450 1 : //===----------------------------------------------------------------------===//
5451 : // AArch64 Inline Assembly Support
5452 1 : //===----------------------------------------------------------------------===//
5453 :
5454 : // Table of Constraints
5455 : // TODO: This is the current set of constraints supported by ARM for the
5456 1 : // compiler, not all of them may make sense.
5457 1 : //
5458 : // r - A general register
5459 1 : // w - An FP/SIMD register of some size in the range v0-v31
5460 : // x - An FP/SIMD register of some size in the range v0-v15
5461 1 : // I - Constant that can be used with an ADD instruction
5462 1 : // J - Constant that can be used with a SUB instruction
5463 : // K - Constant that can be used with a 32-bit logical instruction
5464 : // L - Constant that can be used with a 64-bit logical instruction
5465 4075 : // M - Constant that can be used as a 32-bit MOV immediate
5466 : // N - Constant that can be used as a 64-bit MOV immediate
5467 : // Q - A memory reference with base register and no offset
5468 : // S - A symbolic address
5469 4075 : // Y - Floating point constant zero
5470 : // Z - Integer constant zero
5471 : //
5472 746 : // Note that general register operands will be output using their 64-bit x
5473 : // register name, whatever the size of the variable, unless the asm operand
5474 : // is prefixed by the %w modifier. Floating-point and SIMD register operands
5475 746 : // will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
5476 28 : // %q modifier.
5477 : const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5478 297 : // At this point, we have to lower this constraint to something else, so we
5479 : // lower it to an "r" or "w". However, by doing this we will force the result
5480 : // to be in register, while the X constraint is much more permissive.
5481 : //
5482 : // Although we are correct (we are free to emit anything, without
5483 449 : // constraints), we might break use cases that would expect us to be more
5484 : // efficient and emit something else.
5485 : if (!Subtarget->hasFPARMv8())
5486 152 : return "r";
5487 :
5488 243 : if (ConstraintVT.isFloatingPoint())
5489 44 : return "w";
5490 26 :
5491 : if (ConstraintVT.isVector() &&
5492 421 : (ConstraintVT.getSizeInBits() == 64 ||
5493 : ConstraintVT.getSizeInBits() == 128))
5494 : return "w";
5495 345 :
5496 : return "r";
5497 : }
5498 :
5499 : /// getConstraintType - Given a constraint letter, return the type of
5500 : /// constraint it is for this target.
5501 : AArch64TargetLowering::ConstraintType
5502 : AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
5503 : if (Constraint.size() == 1) {
5504 : switch (Constraint[0]) {
5505 : default:
5506 : break;
5507 37 : case 'z':
5508 : return C_Other;
5509 : case 'x':
5510 37 : case 'w':
5511 37 : return C_RegisterClass;
5512 : // An address with a single base register. Due to the way we
5513 : // currently handle addresses it is the same as 'r'.
5514 : case 'Q':
5515 29 : return C_Memory;
5516 : case 'S': // A symbolic address
5517 : return C_Other;
5518 : }
5519 : }
5520 : return TargetLowering::getConstraintType(Constraint);
5521 58 : }
5522 :
5523 58 : /// Examine constraint type and operand type and determine a weight value.
5524 : /// This object must already have been set up with the operand type
5525 : /// and the current alternative constraint selected.
5526 8 : TargetLowering::ConstraintWeight
5527 : AArch64TargetLowering::getSingleConstraintMatchWeight(
5528 : AsmOperandInfo &info, const char *constraint) const {
5529 69 : ConstraintWeight weight = CW_Invalid;
5530 : Value *CallOperandVal = info.CallOperandVal;
5531 : // If we don't have a value, we can't do a match,
5532 : // but allow it at the lowest weight.
5533 : if (!CallOperandVal)
5534 69 : return CW_Default;
5535 69 : Type *type = CallOperandVal->getType();
5536 52 : // Look at the constraint type.
5537 26 : switch (*constraint) {
5538 : default:
5539 20 : weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
5540 : break;
5541 : case 'x':
5542 : case 'w':
5543 : if (type->isFloatingPointTy() || type->isVectorTy())
5544 : weight = CW_Register;
5545 : break;
5546 69 : case 'z':
5547 : weight = CW_Constant;
5548 49 : break;
5549 49 : }
5550 49 : return weight;
5551 : }
5552 20 :
5553 11 : std::pair<unsigned, const TargetRegisterClass *>
5554 11 : AArch64TargetLowering::getRegForInlineAsmConstraint(
5555 11 : const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
5556 11 : if (Constraint.size() == 1) {
5557 : switch (Constraint[0]) {
5558 11 : case 'r':
5559 : if (VT.getSizeInBits() == 64)
5560 11 : return std::make_pair(0U, &AArch64::GPR64commonRegClass);
5561 15 : return std::make_pair(0U, &AArch64::GPR32commonRegClass);
5562 : case 'w':
5563 : if (!Subtarget->hasFPARMv8())
5564 20 : break;
5565 20 : if (VT.getSizeInBits() == 16)
5566 : return std::make_pair(0U, &AArch64::FPR16RegClass);
5567 : if (VT.getSizeInBits() == 32)
5568 49 : return std::make_pair(0U, &AArch64::FPR32RegClass);
5569 : if (VT.getSizeInBits() == 64)
5570 : return std::make_pair(0U, &AArch64::FPR64RegClass);
5571 56 : if (VT.getSizeInBits() == 128)
5572 : return std::make_pair(0U, &AArch64::FPR128RegClass);
5573 : break;
5574 56 : // The instructions that this constraint is designed for can
5575 22 : // only take 128-bit registers so just use that regclass.
5576 11 : case 'x':
5577 : if (!Subtarget->hasFPARMv8())
5578 9 : break;
5579 : if (VT.getSizeInBits() == 128)
5580 : return std::make_pair(0U, &AArch64::FPR128_loRegClass);
5581 : break;
5582 : }
5583 : }
5584 : if (StringRef("{cc}").equals_lower(Constraint))
5585 31 : return std::make_pair(unsigned(AArch64::NZCV), &AArch64::CCRRegClass);
5586 :
5587 22 : // Use the default implementation in TargetLowering to convert the register
5588 22 : // constraint into a member of a register class.
5589 : std::pair<unsigned, const TargetRegisterClass *> Res;
5590 : Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
5591 9 :
5592 9 : // Not found as a standard register?
5593 : if (!Res.second) {
5594 : unsigned Size = Constraint.size();
5595 47 : if ((Size == 4 || Size == 5) && Constraint[0] == '{' &&
5596 : tolower(Constraint[1]) == 'v' && Constraint[Size - 1] == '}') {
5597 : int RegNo;
5598 : bool Failed = Constraint.slice(2, Size - 1).getAsInteger(10, RegNo);
5599 : if (!Failed && RegNo >= 0 && RegNo <= 31) {
5600 : // v0 - v31 are aliases of q0 - q31 or d0 - d31 depending on size.
5601 : // By default we'll emit v0-v31 for this unless there's a modifier where
5602 : // we'll emit the correct register as well.
5603 : if (VT != MVT::Other && VT.getSizeInBits() == 64) {
5604 : Res.first = AArch64::FPR64RegClass.getRegister(RegNo);
5605 : Res.second = &AArch64::FPR64RegClass;
5606 : } else {
5607 : Res.first = AArch64::FPR128RegClass.getRegister(RegNo);
5608 : Res.second = &AArch64::FPR128RegClass;
5609 : }
5610 : }
5611 : }
5612 : }
5613 :
5614 : if (Res.second && !Subtarget->hasFPARMv8() &&
5615 : !AArch64::GPR32allRegClass.hasSubClassEq(Res.second) &&
5616 : !AArch64::GPR64allRegClass.hasSubClassEq(Res.second))
5617 : return std::make_pair(0U, nullptr);
5618 :
5619 : return Res;
5620 : }
5621 :
5622 : /// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
5623 : /// vector. If it is invalid, don't add anything to Ops.
5624 : void AArch64TargetLowering::LowerAsmOperandForConstraint(
5625 22 : SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
5626 : SelectionDAG &DAG) const {
5627 : SDValue Result;
5628 :
5629 : // Currently only support length 1 constraints.
5630 : if (Constraint.length() != 1)
5631 : return;
5632 :
5633 22 : char ConstraintLetter = Constraint[0];
5634 : switch (ConstraintLetter) {
5635 : default:
5636 19 : break;
5637 :
5638 : // This set of constraints deal with valid constants for various instructions.
5639 16 : // Validate and return a target constant for them if we can.
5640 4 : case 'z': {
5641 : // 'z' maps to xzr or wzr so it needs an input of 0.
5642 4 : if (!isNullConstant(Op))
5643 : return;
5644 :
5645 : if (Op.getValueType() == MVT::i64)
5646 : Result = DAG.getRegister(AArch64::XZR, MVT::i64);
5647 : else
5648 : Result = DAG.getRegister(AArch64::WZR, MVT::i32);
5649 : break;
5650 15330 : }
5651 15330 : case 'S': {
5652 1156 : // An absolute symbolic address or label reference.
5653 : if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
5654 : Result = DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
5655 : GA->getValueType(0));
5656 : } else if (const BlockAddressSDNode *BA =
5657 72 : dyn_cast<BlockAddressSDNode>(Op)) {
5658 : Result =
5659 72 : DAG.getTargetBlockAddress(BA->getBlockAddress(), BA->getValueType(0));
5660 : } else if (const ExternalSymbolSDNode *ES =
5661 : dyn_cast<ExternalSymbolSDNode>(Op)) {
5662 6 : Result =
5663 6 : DAG.getTargetExternalSymbol(ES->getSymbol(), ES->getValueType(0));
5664 : } else
5665 : return;
5666 : break;
5667 : }
5668 15208 :
5669 : case 'I':
5670 : case 'J':
5671 : case 'K':
5672 : case 'L':
5673 : case 'M':
5674 : case 'N':
5675 0 : ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
5676 : if (!C)
5677 : return;
5678 0 :
5679 : // Grab the value and do some validation.
5680 : uint64_t CVal = C->getZExtValue();
5681 0 : switch (ConstraintLetter) {
5682 : // The I constraint applies only to simple ADD or SUB immediate operands:
5683 0 : // i.e. 0 to 4095 with optional shift by 12
5684 : // The J constraint applies only to ADD or SUB immediates that would be
5685 0 : // valid when negated, i.e. if [an add pattern] were to be output as a SUB
5686 0 : // instruction [or vice versa], in other words -1 to -4095 with optional
5687 0 : // left shift by 12.
5688 0 : case 'I':
5689 : if (isUInt<12>(CVal) || isShiftedUInt<12, 12>(CVal))
5690 : break;
5691 0 : return;
5692 : case 'J': {
5693 : uint64_t NVal = -C->getSExtValue();
5694 : if (isUInt<12>(NVal) || isShiftedUInt<12, 12>(NVal)) {
5695 : CVal = C->getSExtValue();
5696 : break;
5697 : }
5698 : return;
5699 : }
5700 : // The K and L constraints apply *only* to logical immediates, including
5701 : // what used to be the MOVI alias for ORR (though the MOVI alias has now
5702 5891 : // been removed and MOV should be used). So these constraints have to
5703 : // distinguish between bit patterns that are valid 32-bit or 64-bit
5704 5891 : // "bitmask immediates": for example 0xaaaaaaaa is a valid bimm32 (K), but
5705 164 : // not a valid bimm64 (L) where 0xaaaaaaaaaaaaaaaa would be valid, and vice
5706 64 : // versa.
5707 64 : case 'K':
5708 23 : if (AArch64_AM::isLogicalImmediate(CVal, 32))
5709 41 : break;
5710 17 : return;
5711 17 : case 'L':
5712 : if (AArch64_AM::isLogicalImmediate(CVal, 64))
5713 16 : break;
5714 3 : return;
5715 13 : // The M and N constraints are a superset of K and L respectively, for use
5716 3 : // with the MOV (immediate) alias. As well as the logical immediates they
5717 10 : // also match 32 or 64-bit immediates that can be loaded either using a
5718 8 : // *single* MOVZ or MOVN , such as 32-bit 0x12340000, 0x00001234, 0xffffedca
5719 2 : // (M) or 64-bit 0x1234000000000000 (N) etc.
5720 2 : // As a note some of this code is liberally stolen from the asm parser.
5721 : case 'M': {
5722 : if (!isUInt<32>(CVal))
5723 : return;
5724 1 : if (AArch64_AM::isLogicalImmediate(CVal, 32))
5725 1 : break;
5726 : if ((CVal & 0xFFFF) == CVal)
5727 1 : break;
5728 1 : if ((CVal & 0xFFFF0000ULL) == CVal)
5729 : break;
5730 : uint64_t NCVal = ~(uint32_t)CVal;
5731 : if ((NCVal & 0xFFFFULL) == NCVal)
5732 5810 : break;
5733 0 : if ((NCVal & 0xFFFF0000ULL) == NCVal)
5734 : break;
5735 : return;
5736 : }
5737 : case 'N': {
5738 5810 : if (AArch64_AM::isLogicalImmediate(CVal, 64))
5739 : break;
5740 : if ((CVal & 0xFFFFULL) == CVal)
5741 5810 : break;
5742 905 : if ((CVal & 0xFFFF0000ULL) == CVal)
5743 1724 : break;
5744 1767 : if ((CVal & 0xFFFF00000000ULL) == CVal)
5745 : break;
5746 836 : if ((CVal & 0xFFFF000000000000ULL) == CVal)
5747 836 : break;
5748 : uint64_t NCVal = ~CVal;
5749 : if ((NCVal & 0xFFFFULL) == NCVal)
5750 : break;
5751 836 : if ((NCVal & 0xFFFF0000ULL) == NCVal)
5752 1 : break;
5753 : if ((NCVal & 0xFFFF00000000ULL) == NCVal)
5754 : break;
5755 835 : if ((NCVal & 0xFFFF000000000000ULL) == NCVal)
5756 : break;
5757 : return;
5758 : }
5759 : default:
5760 : return;
5761 : }
5762 5741 :
5763 5815 : // All assembler immediates are 64-bit integers.
5764 5 : Result = DAG.getTargetConstant(CVal, SDLoc(Op), MVT::i64);
5765 5 : break;
5766 : }
5767 5805 :
5768 : if (Result.getNode()) {
5769 : Ops.push_back(Result);
5770 : return;
5771 : }
5772 55 :
5773 : return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
5774 : }
5775 55 :
5776 : //===----------------------------------------------------------------------===//
5777 : // AArch64 Advanced SIMD Support
5778 55 : //===----------------------------------------------------------------------===//
5779 :
5780 : /// WidenVector - Given a value in the V64 register class, produce the
5781 55 : /// equivalent value in the V128 register class.
5782 55 : static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG) {
5783 : EVT VT = V64Reg.getValueType();
5784 : unsigned NarrowSize = VT.getVectorNumElements();
5785 : MVT EltTy = VT.getVectorElementType().getSimpleVT();
5786 : MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
5787 : SDLoc DL(V64Reg);
5788 9 :
5789 : return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideTy, DAG.getUNDEF(WideTy),
5790 9 : V64Reg, DAG.getConstant(0, DL, MVT::i32));
5791 : }
5792 :
5793 : /// getExtFactor - Determine the adjustment factor for the position when
5794 0 : /// generating an "extract from vector registers" instruction.
5795 : static unsigned getExtFactor(SDValue &V) {
5796 7 : EVT EltType = V.getValueType().getVectorElementType();
5797 : return EltType.getSizeInBits() / 8;
5798 : }
5799 :
5800 : /// NarrowVector - Given a value in the V128 register class, produce the
5801 : /// equivalent value in the V64 register class.
5802 2 : static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
5803 2 : EVT VT = V128Reg.getValueType();
5804 : unsigned WideSize = VT.getVectorNumElements();
5805 : MVT EltTy = VT.getVectorElementType().getSimpleVT();
5806 1 : MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
5807 2 : SDLoc DL(V128Reg);
5808 :
5809 : return DAG.getTargetExtractSubreg(AArch64::dsub, DL, NarrowTy, V128Reg);
5810 0 : }
5811 0 :
5812 : // Gather data to see if the operation can be modelled as a
5813 : // shuffle in combination with VEXTs.
5814 : SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op,
5815 : SelectionDAG &DAG) const {
5816 : assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
5817 : LLVM_DEBUG(dbgs() << "AArch64TargetLowering::ReconstructShuffle\n");
5818 : SDLoc dl(Op);
5819 : EVT VT = Op.getValueType();
5820 : unsigned NumElts = VT.getVectorNumElements();
5821 :
5822 : struct ShuffleSourceInfo {
5823 : SDValue Vec;
5824 : unsigned MinElt;
5825 : unsigned MaxElt;
5826 :
5827 : // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
5828 28 : // be compatible with the shuffle we intend to construct. As a result
5829 : // ShuffleVec will be some sliding window into the original Vec.
5830 : SDValue ShuffleVec;
5831 :
5832 : // Code should guarantee that element i in Vec starts at element "WindowBase
5833 : // + i * WindowScale in ShuffleVec".
5834 : int WindowBase;
5835 : int WindowScale;
5836 :
5837 4 : ShuffleSourceInfo(SDValue Vec)
5838 : : Vec(Vec), MinElt(std::numeric_limits<unsigned>::max()), MaxElt(0),
5839 : ShuffleVec(Vec), WindowBase(0), WindowScale(1) {}
5840 5 :
5841 5 : bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
5842 5 : };
5843 4 :
5844 4 : // First gather all vectors used as an immediate source for this BUILD_VECTOR
5845 : // node.
5846 : SmallVector<ShuffleSourceInfo, 2> Sources;
5847 : for (unsigned i = 0; i < NumElts; ++i) {
5848 : SDValue V = Op.getOperand(i);
5849 : if (V.isUndef())
5850 : continue;
5851 : else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5852 : !isa<ConstantSDNode>(V.getOperand(1))) {
5853 : LLVM_DEBUG(
5854 : dbgs() << "Reshuffle failed: "
5855 : "a shuffle can only come from building a vector from "
5856 4 : "various elements of other vectors, provided their "
5857 : "indices are constant\n");
5858 : return SDValue();
5859 : }
5860 3 :
5861 : // Add this element source to the list if it's not already there.
5862 : SDValue SourceVec = V.getOperand(0);
5863 : auto Source = find(Sources, SourceVec);
5864 : if (Source == Sources.end())
5865 : Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
5866 :
5867 : // Update the minimum and maximum lane number seen.
5868 : unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
5869 : Source->MinElt = std::min(Source->MinElt, EltNo);
5870 6 : Source->MaxElt = std::max(Source->MaxElt, EltNo);
5871 : }
5872 6 :
5873 : if (Sources.size() > 2) {
5874 4 : LLVM_DEBUG(
5875 : dbgs() << "Reshuffle failed: currently only do something sane when at "
5876 3 : "most two source vectors are involved\n");
5877 : return SDValue();
5878 2 : }
5879 2 :
5880 : // Find out the smallest element size among result and two sources, and use
5881 1 : // it as element size to build the shuffle_vector.
5882 : EVT SmallestEltTy = VT.getVectorElementType();
5883 : for (auto &Source : Sources) {
5884 : EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
5885 : if (SrcEltTy.bitsLT(SmallestEltTy)) {
5886 6 : SmallestEltTy = SrcEltTy;
5887 : }
5888 5 : }
5889 : unsigned ResMultiplier =
5890 3 : VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
5891 : NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
5892 3 : EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
5893 :
5894 3 : // If the source vector is too wide or too narrow, we may nevertheless be able
5895 : // to construct a compatible shuffle either by concatenating it with UNDEF or
5896 2 : // extracting a suitable range of elements.
5897 2 : for (auto &Src : Sources) {
5898 : EVT SrcVT = Src.ShuffleVec.getValueType();
5899 1 :
5900 : if (SrcVT.getSizeInBits() == VT.getSizeInBits())
5901 1 : continue;
5902 :
5903 1 : // This stage of the search produces a source with the same element type as
5904 : // the original, but with a total width matching the BUILD_VECTOR output.
5905 : EVT EltVT = SrcVT.getVectorElementType();
5906 : unsigned NumSrcElts = VT.getSizeInBits() / EltVT.getSizeInBits();
5907 : EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
5908 :
5909 12 : if (SrcVT.getSizeInBits() < VT.getSizeInBits()) {
5910 : assert(2 * SrcVT.getSizeInBits() == VT.getSizeInBits());
5911 : // We can pad out the smaller vector for free, so if it's part of a
5912 18 : // shuffle...
5913 18 : Src.ShuffleVec =
5914 : DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
5915 : DAG.getUNDEF(Src.ShuffleVec.getValueType()));
5916 43 : continue;
5917 28 : }
5918 28 :
5919 : assert(SrcVT.getSizeInBits() == 2 * VT.getSizeInBits());
5920 :
5921 15 : if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
5922 : LLVM_DEBUG(
5923 : dbgs() << "Reshuffle failed: span too large for a VEXT to cope\n");
5924 : return SDValue();
5925 : }
5926 :
5927 : if (Src.MinElt >= NumSrcElts) {
5928 : // The extraction can just take the second half
5929 : Src.ShuffleVec =
5930 1498 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5931 1498 : DAG.getConstant(NumSrcElts, dl, MVT::i64));
5932 : Src.WindowBase = -NumSrcElts;
5933 1498 : } else if (Src.MaxElt < NumSrcElts) {
5934 1498 : // The extraction can just take the first half
5935 : Src.ShuffleVec =
5936 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5937 : DAG.getConstant(0, dl, MVT::i64));
5938 1498 : } else {
5939 : // An actual VEXT is needed
5940 : SDValue VEXTSrc1 =
5941 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5942 : DAG.getConstant(0, dl, MVT::i64));
5943 98 : SDValue VEXTSrc2 =
5944 196 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
5945 98 : DAG.getConstant(NumSrcElts, dl, MVT::i64));
5946 : unsigned Imm = Src.MinElt * getExtFactor(VEXTSrc1);
5947 :
5948 : Src.ShuffleVec = DAG.getNode(AArch64ISD::EXT, dl, DestVT, VEXTSrc1,
5949 : VEXTSrc2,
5950 250 : DAG.getConstant(Imm, dl, MVT::i32));
5951 250 : Src.WindowBase = -Src.MinElt;
5952 : }
5953 250 : }
5954 250 :
5955 : // Another possible incompatibility occurs from the vector element types. We
5956 : // can fix this by bitcasting the source vectors to the same type we intend
5957 250 : // for the shuffle.
5958 : for (auto &Src : Sources) {
5959 : EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
5960 : if (SrcEltTy == SmallestEltTy)
5961 : continue;
5962 140 : assert(ShuffleVT.getVectorElementType() == SmallestEltTy);
5963 : Src.ShuffleVec = DAG.getNode(ISD::BITCAST, dl, ShuffleVT, Src.ShuffleVec);
5964 : Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
5965 : Src.WindowBase *= Src.WindowScale;
5966 : }
5967 140 :
5968 : // Final sanity check before we try to actually produce a shuffle.
5969 : LLVM_DEBUG(for (auto Src
5970 : : Sources)
5971 : assert(Src.ShuffleVec.getValueType() == ShuffleVT););
5972 :
5973 : // The stars all align, our next step is to produce the mask for the shuffle.
5974 : SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
5975 : int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
5976 : for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
5977 : SDValue Entry = Op.getOperand(i);
5978 : if (Entry.isUndef())
5979 : continue;
5980 :
5981 : auto Src = find(Sources, Entry.getOperand(0));
5982 : int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
5983 :
5984 : // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
5985 : // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
5986 4 : // segment.
5987 4 : EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
5988 : int BitsDefined =
5989 0 : std::min(OrigEltTy.getSizeInBits(), VT.getScalarSizeInBits());
5990 : int LanesDefined = BitsDefined / BitsPerShuffleLane;
5991 :
5992 : // This source is expected to fill ResMultiplier lanes of the final shuffle,
5993 : // starting at the appropriate offset.
5994 140 : int *LaneMask = &Mask[i * ResMultiplier];
5995 152 :
5996 149 : int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
5997 149 : ExtractBase += NumElts * (Src - Sources.begin());
5998 0 : for (int j = 0; j < LanesDefined; ++j)
5999 149 : LaneMask[j] = ExtractBase + j;
6000 : }
6001 :
6002 : // Final check before we try to produce nonsense...
6003 : if (!isShuffleMaskLegal(Mask, ShuffleVT)) {
6004 : LLVM_DEBUG(dbgs() << "Reshuffle failed: illegal shuffle mask\n");
6005 : return SDValue();
6006 137 : }
6007 :
6008 : SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
6009 : for (unsigned i = 0; i < Sources.size(); ++i)
6010 12 : ShuffleOps[i] = Sources[i].ShuffleVec;
6011 12 :
6012 12 : SDValue Shuffle = DAG.getVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
6013 4 : ShuffleOps[1], Mask);
6014 : SDValue V = DAG.getNode(ISD::BITCAST, dl, VT, Shuffle);
6015 :
6016 12 : LLVM_DEBUG(dbgs() << "Reshuffle, creating node: "; Shuffle.dump();
6017 12 : dbgs() << "Reshuffle, creating node: "; V.dump(););
6018 19 :
6019 : return V;
6020 : }
6021 3 :
6022 : // check if an EXT instruction can handle the shuffle mask when the
6023 : // vector sources of the shuffle are the same.
6024 : static bool isSingletonEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
6025 0 : unsigned NumElts = VT.getVectorNumElements();
6026 :
6027 : // Assume that the first shuffle index is not UNDEF. Fail if it is.
6028 : if (M[0] < 0)
6029 : return false;
6030 3 :
6031 7 : Imm = M[0];
6032 8 :
6033 4 : // If this is a VEXT shuffle, the immediate value is the index of the first
6034 2 : // element. The other shuffle indices must be the successive elements after
6035 : // the first one.
6036 : unsigned ExpectedElt = Imm;
6037 : for (unsigned i = 1; i < NumElts; ++i) {
6038 3 : // Increment the expected index. If it wraps around, just follow it
6039 3 : // back to index zero and keep going.
6040 3 : ++ExpectedElt;
6041 : if (ExpectedElt == NumElts)
6042 : ExpectedElt = 0;
6043 :
6044 : if (M[i] < 0)
6045 7 : continue; // ignore UNDEF indices
6046 4 : if (ExpectedElt != static_cast<unsigned>(M[i]))
6047 : return false;
6048 4 : }
6049 2 :
6050 : return true;
6051 : }
6052 :
6053 2 : // check if an EXT instruction can handle the shuffle mask when the
6054 2 : // vector sources of the shuffle are different.
6055 2 : static bool isEXTMask(ArrayRef<int> M, EVT VT, bool &ReverseEXT,
6056 : unsigned &Imm) {
6057 2 : // Look for the first non-undef element.
6058 : const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
6059 :
6060 : // Benefit form APInt to handle overflow when calculating expected element.
6061 0 : unsigned NumElts = VT.getVectorNumElements();
6062 0 : unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
6063 0 : APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
6064 0 : // The following shuffle indices must be the successive elements after the
6065 : // first real element.
6066 : const int *FirstWrongElt = std::find_if(FirstRealElt + 1, M.end(),
6067 : [&](int Elt) {return Elt != ExpectedElt++ && Elt != -1;});
6068 : if (FirstWrongElt != M.end())
6069 2 : return false;
6070 :
6071 : // The index of an EXT is the first element if it is not UNDEF.
6072 0 : // Watch out for the beginning UNDEFs. The EXT index should be the expected
6073 : // value of the first element. E.g.
6074 : // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
6075 2 : // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
6076 : // ExpectedElt is the last mask index plus 1.
6077 1 : Imm = ExpectedElt.getZExtValue();
6078 1 :
6079 1 : // There are two difference cases requiring to reverse input vectors.
6080 1 : // For example, for vector <4 x i32> we have the following cases,
6081 1 : // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
6082 : // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
6083 1 : // For both cases, we finally use mask <5, 6, 7, 0>, which requires
6084 1 : // to reverse two input vectors.
6085 1 : if (Imm < NumElts)
6086 : ReverseEXT = true;
6087 : else
6088 : Imm -= NumElts;
6089 :
6090 0 : return true;
6091 : }
6092 :
6093 0 : /// isREVMask - Check if a vector shuffle corresponds to a REV
6094 0 : /// instruction with the specified blocksize. (The order of the elements
6095 : /// within each block of the vector is reversed.)
6096 0 : static bool isREVMask(ArrayRef<int> M, EVT VT, unsigned BlockSize) {
6097 : assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
6098 0 : "Only possible block sizes for REV are: 16, 32, 64");
6099 0 :
6100 : unsigned EltSz = VT.getScalarSizeInBits();
6101 : if (EltSz == 64)
6102 : return false;
6103 :
6104 : unsigned NumElts = VT.getVectorNumElements();
6105 : unsigned BlockElts = M[0] + 1;
6106 7 : // If the first shuffle index is UNDEF, be optimistic.
6107 8 : if (M[0] < 0)
6108 4 : BlockElts = BlockSize / EltSz;
6109 3 :
6110 : if (BlockSize <= EltSz || BlockSize != BlockElts * EltSz)
6111 1 : return false;
6112 1 :
6113 1 : for (unsigned i = 0; i < NumElts; ++i) {
6114 : if (M[i] < 0)
6115 : continue; // ignore UNDEF indices
6116 : if ((unsigned)M[i] != (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
6117 : return false;
6118 : }
6119 :
6120 : return true;
6121 : }
6122 6 :
6123 3 : static bool isZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6124 27 : unsigned NumElts = VT.getVectorNumElements();
6125 12 : WhichResult = (M[0] == 0 ? 0 : 1);
6126 12 : unsigned Idx = WhichResult * NumElts / 2;
6127 0 : for (unsigned i = 0; i != NumElts; i += 2) {
6128 : if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
6129 12 : (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx + NumElts))
6130 12 : return false;
6131 : Idx += 1;
6132 : }
6133 :
6134 : return true;
6135 24 : }
6136 :
6137 12 : static bool isUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6138 12 : unsigned NumElts = VT.getVectorNumElements();
6139 : WhichResult = (M[0] == 0 ? 0 : 1);
6140 : for (unsigned i = 0; i != NumElts; ++i) {
6141 : if (M[i] < 0)
6142 12 : continue; // ignore UNDEF indices
6143 : if ((unsigned)M[i] != 2 * i + WhichResult)
6144 12 : return false;
6145 24 : }
6146 24 :
6147 12 : return true;
6148 : }
6149 :
6150 : static bool isTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6151 6 : unsigned NumElts = VT.getVectorNumElements();
6152 : WhichResult = (M[0] == 0 ? 0 : 1);
6153 0 : for (unsigned i = 0; i < NumElts; i += 2) {
6154 : if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
6155 : (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + NumElts + WhichResult))
6156 3 : return false;
6157 7 : }
6158 4 : return true;
6159 : }
6160 :
6161 3 : /// isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of
6162 3 : /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
6163 : /// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
6164 : static bool isZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6165 : unsigned NumElts = VT.getVectorNumElements();
6166 : WhichResult = (M[0] == 0 ? 0 : 1);
6167 3 : unsigned Idx = WhichResult * NumElts / 2;
6168 : for (unsigned i = 0; i != NumElts; i += 2) {
6169 : if ((M[i] >= 0 && (unsigned)M[i] != Idx) ||
6170 : (M[i + 1] >= 0 && (unsigned)M[i + 1] != Idx))
6171 : return false;
6172 0 : Idx += 1;
6173 : }
6174 :
6175 : return true;
6176 0 : }
6177 0 :
6178 : /// isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of
6179 0 : /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
6180 : /// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
6181 : static bool isUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6182 : unsigned Half = VT.getVectorNumElements() / 2;
6183 : WhichResult = (M[0] == 0 ? 0 : 1);
6184 : for (unsigned j = 0; j != 2; ++j) {
6185 0 : unsigned Idx = WhichResult;
6186 : for (unsigned i = 0; i != Half; ++i) {
6187 : int MIdx = M[i + j * Half];
6188 0 : if (MIdx >= 0 && (unsigned)MIdx != Idx)
6189 0 : return false;
6190 : Idx += 2;
6191 : }
6192 0 : }
6193 0 :
6194 0 : return true;
6195 0 : }
6196 :
6197 : /// isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of
6198 : /// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
6199 : /// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
6200 : static bool isTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
6201 : unsigned NumElts = VT.getVectorNumElements();
6202 : WhichResult = (M[0] == 0 ? 0 : 1);
6203 637 : for (unsigned i = 0; i < NumElts; i += 2) {
6204 : if ((M[i] >= 0 && (unsigned)M[i] != i + WhichResult) ||
6205 : (M[i + 1] >= 0 && (unsigned)M[i + 1] != i + WhichResult))
6206 0 : return false;
6207 : }
6208 : return true;
6209 : }
6210 637 :
6211 637 : static bool isINSMask(ArrayRef<int> M, int NumInputElements,
6212 : bool &DstIsLeft, int &Anomaly) {
6213 : if (M.size() != static_cast<size_t>(NumInputElements))
6214 637 : return false;
6215 :
6216 637 : int NumLHSMatch = 0, NumRHSMatch = 0;
6217 : int LastLHSMismatch = -1, LastRHSMismatch = -1;
6218 :
6219 : for (int i = 0; i < NumInputElements; ++i) {
6220 : if (M[i] == -1) {
6221 : ++NumLHSMatch;
6222 : ++NumRHSMatch;
6223 : continue;
6224 : }
6225 68 :
6226 : if (M[i] == i)
6227 : ++NumLHSMatch;
6228 : else
6229 : LastLHSMismatch = i;
6230 :
6231 : if (M[i] == i + NumInputElements)
6232 : ++NumRHSMatch;
6233 68 : else
6234 23 : LastRHSMismatch = i;
6235 : }
6236 45 :
6237 : if (NumLHSMatch == NumInputElements - 1) {
6238 : DstIsLeft = true;
6239 : Anomaly = LastLHSMismatch;
6240 : return true;
6241 : } else if (NumRHSMatch == NumInputElements - 1) {
6242 : DstIsLeft = false;
6243 : Anomaly = LastRHSMismatch;
6244 0 : return true;
6245 : }
6246 :
6247 : return false;
6248 : }
6249 0 :
6250 0 : static bool isConcatMask(ArrayRef<int> Mask, EVT VT, bool SplitLHS) {
6251 : if (VT.getSizeInBits() != 128)
6252 : return false;
6253 0 :
6254 : unsigned NumElts = VT.getVectorNumElements();
6255 0 :
6256 0 : for (int I = 0, E = NumElts / 2; I != E; I++) {
6257 : if (Mask[I] != I)
6258 0 : return false;
6259 0 : }
6260 :
6261 0 : int Offset = NumElts / 2;
6262 0 : for (int I = NumElts / 2, E = NumElts; I != E; I++) {
6263 0 : if (Mask[I] != I + SplitLHS * Offset)
6264 0 : return false;
6265 0 : }
6266 :
6267 : return true;
6268 : }
6269 :
6270 : static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) {
6271 0 : SDLoc DL(Op);
6272 : EVT VT = Op.getValueType();
6273 0 : SDValue V0 = Op.getOperand(0);
6274 0 : SDValue V1 = Op.getOperand(1);
6275 0 : ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
6276 0 :
6277 0 : if (VT.getVectorElementType() != V0.getValueType().getVectorElementType() ||
6278 0 : VT.getVectorElementType() != V1.getValueType().getVectorElementType())
6279 0 : return SDValue();
6280 :
6281 : bool SplitV0 = V0.getValueSizeInBits() == 128;
6282 :
6283 : if (!isConcatMask(Mask, VT, SplitV0))
6284 : return SDValue();
6285 0 :
6286 : EVT CastVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
6287 0 : VT.getVectorNumElements() / 2);
6288 0 : if (SplitV0) {
6289 0 : V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0,
6290 0 : DAG.getConstant(0, DL, MVT::i64));
6291 0 : }
6292 0 : if (V1.getValueSizeInBits() == 128) {
6293 : V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1,
6294 : DAG.getConstant(0, DL, MVT::i64));
6295 : }
6296 : return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, V0, V1);
6297 : }
6298 0 :
6299 : /// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
6300 0 : /// the specified operations to build the shuffle.
6301 0 : static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
6302 0 : SDValue RHS, SelectionDAG &DAG,
6303 0 : const SDLoc &dl) {
6304 0 : unsigned OpNum = (PFEntry >> 26) & 0x0F;
6305 : unsigned LHSID = (PFEntry >> 13) & ((1 << 13) - 1);
6306 : unsigned RHSID = (PFEntry >> 0) & ((1 << 13) - 1);
6307 :
6308 : enum {
6309 : OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
6310 : OP_VREV,
6311 : OP_VDUP0,
6312 0 : OP_VDUP1,
6313 : OP_VDUP2,
6314 0 : OP_VDUP3,
6315 0 : OP_VEXT1,
6316 0 : OP_VEXT2,
6317 0 : OP_VEXT3,
6318 0 : OP_VUZPL, // VUZP, left result
6319 0 : OP_VUZPR, // VUZP, right result
6320 0 : OP_VZIPL, // VZIP, left result
6321 : OP_VZIPR, // VZIP, right result
6322 : OP_VTRNL, // VTRN, left result
6323 : OP_VTRNR // VTRN, right result
6324 : };
6325 :
6326 : if (OpNum == OP_COPY) {
6327 : if (LHSID == (1 * 9 + 2) * 9 + 3)
6328 : return LHS;
6329 0 : assert(LHSID == ((4 * 9 + 5) * 9 + 6) * 9 + 7 && "Illegal OP_COPY!");
6330 0 : return RHS;
6331 0 : }
6332 0 :
6333 : SDValue OpLHS, OpRHS;
6334 0 : OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
6335 0 : OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
6336 0 : EVT VT = OpLHS.getValueType();
6337 0 :
6338 0 : switch (OpNum) {
6339 : default:
6340 : llvm_unreachable("Unknown shuffle opcode!");
6341 : case OP_VREV:
6342 : // VREV divides the vector in half and swaps within the half.
6343 : if (VT.getVectorElementType() == MVT::i32 ||
6344 : VT.getVectorElementType() == MVT::f32)
6345 : return DAG.getNode(AArch64ISD::REV64, dl, VT, OpLHS);
6346 : // vrev <4 x i16> -> REV32
6347 : if (VT.getVectorElementType() == MVT::i16 ||
6348 0 : VT.getVectorElementType() == MVT::f16)
6349 : return DAG.getNode(AArch64ISD::REV32, dl, VT, OpLHS);
6350 0 : // vrev <4 x i8> -> REV16
6351 0 : assert(VT.getVectorElementType() == MVT::i8);
6352 0 : return DAG.getNode(AArch64ISD::REV16, dl, VT, OpLHS);
6353 0 : case OP_VDUP0:
6354 0 : case OP_VDUP1:
6355 : case OP_VDUP2:
6356 : case OP_VDUP3: {
6357 : EVT EltTy = VT.getVectorElementType();
6358 : unsigned Opcode;
6359 44 : if (EltTy == MVT::i8)
6360 : Opcode = AArch64ISD::DUPLANE8;
6361 44 : else if (EltTy == MVT::i16 || EltTy == MVT::f16)
6362 : Opcode = AArch64ISD::DUPLANE16;
6363 : else if (EltTy == MVT::i32 || EltTy == MVT::f32)
6364 : Opcode = AArch64ISD::DUPLANE32;
6365 : else if (EltTy == MVT::i64 || EltTy == MVT::f64)
6366 : Opcode = AArch64ISD::DUPLANE64;
6367 372 : else
6368 656 : llvm_unreachable("Invalid vector element type?");
6369 1 :
6370 1 : if (VT.getSizeInBits() == 64)
6371 1 : OpLHS = WidenVector(OpLHS, DAG);
6372 : SDValue Lane = DAG.getConstant(OpNum - OP_VDUP0, dl, MVT::i64);
6373 : return DAG.getNode(Opcode, dl, VT, OpLHS, Lane);
6374 327 : }
6375 101 : case OP_VEXT1:
6376 : case OP_VEXT2:
6377 : case OP_VEXT3: {
6378 : unsigned Imm = (OpNum - OP_VEXT1 + 1) * getExtFactor(OpLHS);
6379 327 : return DAG.getNode(AArch64ISD::EXT, dl, VT, OpLHS, OpRHS,
6380 75 : DAG.getConstant(Imm, dl, MVT::i32));
6381 : }
6382 : case OP_VUZPL:
6383 : return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), OpLHS,
6384 : OpRHS);
6385 44 : case OP_VUZPR:
6386 10 : return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), OpLHS,
6387 10 : OpRHS);
6388 10 : case OP_VZIPL:
6389 34 : return DAG.getNode(AArch64ISD::ZIP1, dl, DAG.getVTList(VT, VT), OpLHS,
6390 2 : OpRHS);
6391 2 : case OP_VZIPR:
6392 2 : return DAG.getNode(AArch64ISD::ZIP2, dl, DAG.getVTList(VT, VT), OpLHS,
6393 : OpRHS);
6394 : case OP_VTRNL:
6395 : return DAG.getNode(AArch64ISD::TRN1, dl, DAG.getVTList(VT, VT), OpLHS,
6396 : OpRHS);
6397 : case OP_VTRNR:
6398 52 : return DAG.getNode(AArch64ISD::TRN2, dl, DAG.getVTList(VT, VT), OpLHS,
6399 52 : OpRHS);
6400 : }
6401 : }
6402 :
6403 : static SDValue GenerateTBL(SDValue Op, ArrayRef<int> ShuffleMask,
6404 111 : SelectionDAG &DAG) {
6405 194 : // Check to see if we can use the TBL instruction.
6406 : SDValue V1 = Op.getOperand(0);
6407 : SDValue V2 = Op.getOperand(1);
6408 : SDLoc DL(Op);
6409 :
6410 72 : EVT EltVT = Op.getValueType().getVectorElementType();
6411 122 : unsigned BytesPerElt = EltVT.getSizeInBits() / 8;
6412 :
6413 : SmallVector<SDValue, 8> TBLMask;
6414 : for (int Val : ShuffleMask) {
6415 : for (unsigned Byte = 0; Byte < BytesPerElt; ++Byte) {
6416 : unsigned Offset = Byte + Val * BytesPerElt;
6417 : TBLMask.push_back(DAG.getConstant(Offset, DL, MVT::i32));
6418 0 : }
6419 0 : }
6420 0 :
6421 0 : MVT IndexVT = MVT::v8i8;
6422 0 : unsigned IndexLen = 8;
6423 0 : if (Op.getValueSizeInBits() == 128) {
6424 : IndexVT = MVT::v16i8;
6425 0 : IndexLen = 16;
6426 0 : }
6427 0 :
6428 : SDValue V1Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V1);
6429 0 : SDValue V2Cst = DAG.getNode(ISD::BITCAST, DL, IndexVT, V2);
6430 :
6431 0 : SDValue Shuffle;
6432 0 : if (V2.getNode()->isUndef()) {
6433 : if (IndexLen == 8)
6434 0 : V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V1Cst);
6435 0 : Shuffle = DAG.getNode(
6436 0 : ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
6437 0 : DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
6438 0 : DAG.getBuildVector(IndexVT, DL,
6439 : makeArrayRef(TBLMask.data(), IndexLen)));
6440 0 : } else {
6441 0 : if (IndexLen == 8) {
6442 0 : V1Cst = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V1Cst, V2Cst);
6443 : Shuffle = DAG.getNode(
6444 0 : ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
6445 : DAG.getConstant(Intrinsic::aarch64_neon_tbl1, DL, MVT::i32), V1Cst,
6446 : DAG.getBuildVector(IndexVT, DL,
6447 : makeArrayRef(TBLMask.data(), IndexLen)));
6448 : } else {
6449 74 : // FIXME: We cannot, for the moment, emit a TBL2 instruction because we
6450 : // cannot currently represent the register constraints on the input
6451 : // table registers.
6452 74 : // Shuffle = DAG.getNode(AArch64ISD::TBL2, DL, IndexVT, V1Cst, V2Cst,
6453 74 : // DAG.getBuildVector(IndexVT, DL, &TBLMask[0],
6454 74 : // IndexLen));
6455 : Shuffle = DAG.getNode(
6456 : ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
6457 : DAG.getConstant(Intrinsic::aarch64_neon_tbl2, DL, MVT::i32), V1Cst,
6458 : V2Cst, DAG.getBuildVector(IndexVT, DL,
6459 : makeArrayRef(TBLMask.data(), IndexLen)));
6460 : }
6461 : }
6462 : return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Shuffle);
6463 : }
6464 :
6465 : static unsigned getDUPLANEOp(EVT EltType) {
6466 : if (EltType == MVT::i8)
6467 : return AArch64ISD::DUPLANE8;
6468 : if (EltType == MVT::i16 || EltType == MVT::f16)
6469 : return AArch64ISD::DUPLANE16;
6470 : if (EltType == MVT::i32 || EltType == MVT::f32)
6471 : return AArch64ISD::DUPLANE32;
6472 : if (EltType == MVT::i64 || EltType == MVT::f64)
6473 : return AArch64ISD::DUPLANE64;
6474 74 :
6475 43 : llvm_unreachable("Invalid vector element type?");
6476 28 : }
6477 :
6478 15 : SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
6479 : SelectionDAG &DAG) const {
6480 : SDLoc dl(Op);
6481 31 : EVT VT = Op.getValueType();
6482 31 :
6483 31 : ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
6484 31 :
6485 : // Convert shuffles that are directly supported on NEON to target-specific
6486 31 : // DAG nodes, instead of keeping them as shuffles and matching them again
6487 0 : // during code selection. This is more efficient and avoids the possibility
6488 0 : // of inconsistencies between legalization and selection.
6489 : ArrayRef<int> ShuffleMask = SVN->getMask();
6490 :
6491 4 : SDValue V1 = Op.getOperand(0);
6492 6 : SDValue V2 = Op.getOperand(1);
6493 3 :
6494 : if (SVN->isSplat()) {
6495 1 : int Lane = SVN->getSplatIndex();
6496 1 : // If this is undef splat, generate it via "just" vdup, if possible.
6497 1 : if (Lane == -1)
6498 : Lane = 0;
6499 :
6500 0 : if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR)
6501 8 : return DAG.getNode(AArch64ISD::DUP, dl, V1.getValueType(),
6502 : V1.getOperand(0));
6503 : // Test if V1 is a BUILD_VECTOR and the lane being referenced is a non-
6504 : // constant. If so, we can just reference the lane's definition directly.
6505 8 : if (V1.getOpcode() == ISD::BUILD_VECTOR &&
6506 : !isa<ConstantSDNode>(V1.getOperand(Lane)))
6507 : return DAG.getNode(AArch64ISD::DUP, dl, VT, V1.getOperand(Lane));
6508 :
6509 : // Otherwise, duplicate from the lane of the input vector.
6510 : unsigned Opcode = getDUPLANEOp(V1.getValueType().getVectorElementType());
6511 :
6512 : // SelectionDAGBuilder may have "helpfully" already extracted or conatenated
6513 : // to make a vector of the same size as this SHUFFLE. We can ignore the
6514 : // extract entirely, and canonicalise the concat using WidenVector.
6515 : if (V1.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6516 0 : Lane += cast<ConstantSDNode>(V1.getOperand(1))->getZExtValue();
6517 : V1 = V1.getOperand(0);
6518 8 : } else if (V1.getOpcode() == ISD::CONCAT_VECTORS) {
6519 4 : unsigned Idx = Lane >= (int)VT.getVectorNumElements() / 2;
6520 8 : Lane -= Idx * VT.getVectorNumElements() / 2;
6521 8 : V1 = WidenVector(V1.getOperand(Idx), DAG);
6522 : } else if (VT.getSizeInBits() == 64)
6523 14 : V1 = WidenVector(V1, DAG);
6524 :
6525 : return DAG.getNode(Opcode, dl, VT, V1, DAG.getConstant(Lane, dl, MVT::i64));
6526 14 : }
6527 :
6528 14 : if (isREVMask(ShuffleMask, VT, 64))
6529 : return DAG.getNode(AArch64ISD::REV64, dl, V1.getValueType(), V1, V2);
6530 0 : if (isREVMask(ShuffleMask, VT, 32))
6531 : return DAG.getNode(AArch64ISD::REV32, dl, V1.getValueType(), V1, V2);
6532 0 : if (isREVMask(ShuffleMask, VT, 16))
6533 0 : return DAG.getNode(AArch64ISD::REV16, dl, V1.getValueType(), V1, V2);
6534 :
6535 0 : bool ReverseEXT = false;
6536 2 : unsigned Imm;
6537 : if (isEXTMask(ShuffleMask, VT, ReverseEXT, Imm)) {
6538 2 : if (ReverseEXT)
6539 0 : std::swap(V1, V2);
6540 : Imm *= getExtFactor(V1);
6541 0 : return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V2,
6542 0 : DAG.getConstant(Imm, dl, MVT::i32));
6543 : } else if (V2->isUndef() && isSingletonEXTMask(ShuffleMask, VT, Imm)) {
6544 0 : Imm *= getExtFactor(V1);
6545 3 : return DAG.getNode(AArch64ISD::EXT, dl, V1.getValueType(), V1, V1,
6546 : DAG.getConstant(Imm, dl, MVT::i32));
6547 3 : }
6548 :
6549 : unsigned WhichResult;
6550 : if (isZIPMask(ShuffleMask, VT, WhichResult)) {
6551 7 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
6552 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
6553 : }
6554 7 : if (isUZPMask(ShuffleMask, VT, WhichResult)) {
6555 7 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
6556 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
6557 : }
6558 14 : if (isTRNMask(ShuffleMask, VT, WhichResult)) {
6559 7 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
6560 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V2);
6561 : }
6562 71 :
6563 144 : if (isZIP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
6564 80 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::ZIP1 : AArch64ISD::ZIP2;
6565 80 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
6566 : }
6567 : if (isUZP_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
6568 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::UZP1 : AArch64ISD::UZP2;
6569 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
6570 : }
6571 7 : if (isTRN_v_undef_Mask(ShuffleMask, VT, WhichResult)) {
6572 : unsigned Opc = (WhichResult == 0) ? AArch64ISD::TRN1 : AArch64ISD::TRN2;
6573 : return DAG.getNode(Opc, dl, V1.getValueType(), V1, V1);
6574 : }
6575 :
6576 7 : if (SDValue Concat = tryFormConcatFromShuffle(Op, DAG))
6577 7 : return Concat;
6578 :
6579 7 : bool DstIsLeft;
6580 7 : int Anomaly;
6581 1 : int NumInputElements = V1.getValueType().getVectorNumElements();
6582 0 : if (isINSMask(ShuffleMask, NumInputElements, DstIsLeft, Anomaly)) {
6583 1 : SDValue DstVec = DstIsLeft ? V1 : V2;
6584 : SDValue DstLaneV = DAG.getConstant(Anomaly, dl, MVT::i64);
6585 :
6586 : SDValue SrcVec = V1;
6587 2 : int SrcLane = ShuffleMask[Anomaly];
6588 : if (SrcLane >= NumInputElements) {
6589 6 : SrcVec = V2;
6590 4 : SrcLane -= VT.getVectorNumElements();
6591 4 : }
6592 : SDValue SrcLaneV = DAG.getConstant(SrcLane, dl, MVT::i64);
6593 :
6594 : EVT ScalarVT = VT.getVectorElementType();
6595 8 :
6596 : if (ScalarVT.getSizeInBits() < 32 && ScalarVT.isInteger())
6597 : ScalarVT = MVT::i32;
6598 :
6599 : return DAG.getNode(
6600 : ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
6601 : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, SrcVec, SrcLaneV),
6602 : DstLaneV);
6603 2 : }
6604 :
6605 : // If the shuffle is not directly supported and it has 4 elements, use
6606 : // the PerfectShuffle-generated table to synthesize it from other shuffles.
6607 4 : unsigned NumElts = VT.getVectorNumElements();
6608 : if (NumElts == 4) {
6609 : unsigned PFIndexes[4];
6610 14 : for (unsigned i = 0; i != 4; ++i) {
6611 : if (ShuffleMask[i] < 0)
6612 : PFIndexes[i] = 8;
6613 0 : else
6614 0 : PFIndexes[i] = ShuffleMask[i];
6615 0 : }
6616 0 :
6617 0 : // Compute the index in the perfect shuffle table.
6618 0 : unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
6619 0 : PFIndexes[2] * 9 + PFIndexes[3];
6620 0 : unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
6621 0 : unsigned Cost = (PFEntry >> 30);
6622 :
6623 0 : if (Cost <= 4)
6624 : return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
6625 : }
6626 1680 :
6627 : return GenerateTBL(Op, ShuffleMask, DAG);
6628 : }
6629 1680 :
6630 : static bool resolveBuildVector(BuildVectorSDNode *BVN, APInt &CnstBits,
6631 : APInt &UndefBits) {
6632 : EVT VT = BVN->getValueType(0);
6633 : APInt SplatBits, SplatUndef;
6634 : unsigned SplatBitSize;
6635 : bool HasAnyUndefs;
6636 : if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
6637 1680 : unsigned NumSplats = VT.getSizeInBits() / SplatBitSize;
6638 :
6639 1680 : for (unsigned i = 0; i < NumSplats; ++i) {
6640 1680 : CnstBits <<= SplatBitSize;
6641 : UndefBits <<= SplatBitSize;
6642 1680 : CnstBits |= SplatBits.zextOrTrunc(VT.getSizeInBits());
6643 1045 : UndefBits |= (SplatBits ^ SplatUndef).zextOrTrunc(VT.getSizeInBits());
6644 : }
6645 1045 :
6646 : return true;
6647 : }
6648 1045 :
6649 : return false;
6650 0 : }
6651 :
6652 : // Try 64-bit splatted SIMD immediate.
6653 2090 : static SDValue tryAdvSIMDModImm64(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
6654 0 : const APInt &Bits) {
6655 0 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6656 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6657 : EVT VT = Op.getValueType();
6658 2090 : MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v2i64 : MVT::f64;
6659 :
6660 : if (AArch64_AM::isAdvSIMDModImmType10(Value)) {
6661 : Value = AArch64_AM::encodeAdvSIMDModImmType10(Value);
6662 :
6663 2090 : SDLoc dl(Op);
6664 265 : SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
6665 265 : DAG.getConstant(Value, dl, MVT::i32));
6666 780 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6667 123 : }
6668 123 : }
6669 123 :
6670 657 : return SDValue();
6671 447 : }
6672 :
6673 1045 : // Try 32-bit splatted SIMD immediate.
6674 : static SDValue tryAdvSIMDModImm32(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
6675 : const APInt &Bits,
6676 635 : const SDValue *LHS = nullptr) {
6677 36 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6678 617 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6679 32 : EVT VT = Op.getValueType();
6680 601 : MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
6681 16 : bool isAdvSIMDModImm = false;
6682 : uint64_t Shift;
6683 593 :
6684 : if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType1(Value))) {
6685 593 : Value = AArch64_AM::encodeAdvSIMDModImmType1(Value);
6686 64 : Shift = 0;
6687 : }
6688 64 : else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType2(Value))) {
6689 : Value = AArch64_AM::encodeAdvSIMDModImmType2(Value);
6690 64 : Shift = 8;
6691 529 : }
6692 20 : else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType3(Value))) {
6693 : Value = AArch64_AM::encodeAdvSIMDModImmType3(Value);
6694 20 : Shift = 16;
6695 : }
6696 : else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType4(Value))) {
6697 : Value = AArch64_AM::encodeAdvSIMDModImmType4(Value);
6698 509 : Shift = 24;
6699 180 : }
6700 360 :
6701 : if (isAdvSIMDModImm) {
6702 329 : SDLoc dl(Op);
6703 112 : SDValue Mov;
6704 224 :
6705 : if (LHS)
6706 217 : Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
6707 88 : DAG.getConstant(Value, dl, MVT::i32),
6708 176 : DAG.getConstant(Shift, dl, MVT::i32));
6709 : else
6710 : Mov = DAG.getNode(NewOp, dl, MovTy,
6711 129 : DAG.getConstant(Value, dl, MVT::i32),
6712 30 : DAG.getConstant(Shift, dl, MVT::i32));
6713 60 :
6714 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6715 99 : }
6716 30 : }
6717 60 :
6718 : return SDValue();
6719 69 : }
6720 30 :
6721 60 : // Try 16-bit splatted SIMD immediate.
6722 : static SDValue tryAdvSIMDModImm16(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
6723 : const APInt &Bits,
6724 39 : const SDValue *LHS = nullptr) {
6725 10 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6726 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6727 : EVT VT = Op.getValueType();
6728 : MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v8i16 : MVT::v4i16;
6729 58 : bool isAdvSIMDModImm = false;
6730 29 : uint64_t Shift;
6731 12 :
6732 10 : if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType5(Value))) {
6733 : Value = AArch64_AM::encodeAdvSIMDModImmType5(Value);
6734 10 : Shift = 0;
6735 10 : }
6736 10 : else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType6(Value))) {
6737 8 : Value = AArch64_AM::encodeAdvSIMDModImmType6(Value);
6738 8 : Shift = 8;
6739 : }
6740 10 :
6741 : if (isAdvSIMDModImm) {
6742 10 : SDLoc dl(Op);
6743 : SDValue Mov;
6744 10 :
6745 4 : if (LHS)
6746 : Mov = DAG.getNode(NewOp, dl, MovTy, *LHS,
6747 : DAG.getConstant(Value, dl, MVT::i32),
6748 : DAG.getConstant(Shift, dl, MVT::i32));
6749 : else
6750 10 : Mov = DAG.getNode(NewOp, dl, MovTy,
6751 : DAG.getConstant(Value, dl, MVT::i32),
6752 : DAG.getConstant(Shift, dl, MVT::i32));
6753 :
6754 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6755 : }
6756 19 : }
6757 :
6758 60 : return SDValue();
6759 96 : }
6760 1 :
6761 : // Try 32-bit splatted SIMD immediate with shifted ones.
6762 47 : static SDValue tryAdvSIMDModImm321s(unsigned NewOp, SDValue Op,
6763 : SelectionDAG &DAG, const APInt &Bits) {
6764 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6765 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6766 36 : EVT VT = Op.getValueType();
6767 24 : MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v4i32 : MVT::v2i32;
6768 12 : bool isAdvSIMDModImm = false;
6769 : uint64_t Shift;
6770 :
6771 : if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType7(Value))) {
6772 12 : Value = AArch64_AM::encodeAdvSIMDModImmType7(Value);
6773 : Shift = 264;
6774 : }
6775 7 : else if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType8(Value))) {
6776 : Value = AArch64_AM::encodeAdvSIMDModImmType8(Value);
6777 : Shift = 272;
6778 1761 : }
6779 :
6780 3522 : if (isAdvSIMDModImm) {
6781 : SDLoc dl(Op);
6782 : SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
6783 : DAG.getConstant(Value, dl, MVT::i32),
6784 1761 : DAG.getConstant(Shift, dl, MVT::i32));
6785 1107 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6786 : }
6787 8148 : }
6788 7041 :
6789 7041 : return SDValue();
6790 14082 : }
6791 14156 :
6792 : // Try 8-bit splatted SIMD immediate.
6793 : static SDValue tryAdvSIMDModImm8(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
6794 : const APInt &Bits) {
6795 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6796 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6797 : EVT VT = Op.getValueType();
6798 : MVT MovTy = (VT.getSizeInBits() == 128) ? MVT::v16i8 : MVT::v8i8;
6799 :
6800 : if (AArch64_AM::isAdvSIMDModImmType9(Value)) {
6801 0 : Value = AArch64_AM::encodeAdvSIMDModImmType9(Value);
6802 :
6803 0 : SDLoc dl(Op);
6804 0 : SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
6805 0 : DAG.getConstant(Value, dl, MVT::i32));
6806 0 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6807 : }
6808 0 : }
6809 0 :
6810 : return SDValue();
6811 0 : }
6812 :
6813 0 : // Try FP splatted SIMD immediate.
6814 0 : static SDValue tryAdvSIMDModImmFP(unsigned NewOp, SDValue Op, SelectionDAG &DAG,
6815 : const APInt &Bits) {
6816 : if (Bits.getHiBits(64) == Bits.getLoBits(64)) {
6817 : uint64_t Value = Bits.zextOrTrunc(64).getZExtValue();
6818 0 : EVT VT = Op.getValueType();
6819 : bool isWide = (VT.getSizeInBits() == 128);
6820 : MVT MovTy;
6821 : bool isAdvSIMDModImm = false;
6822 0 :
6823 : if ((isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType11(Value))) {
6824 : Value = AArch64_AM::encodeAdvSIMDModImmType11(Value);
6825 0 : MovTy = isWide ? MVT::v4f32 : MVT::v2f32;
6826 0 : }
6827 0 : else if (isWide &&
6828 0 : (isAdvSIMDModImm = AArch64_AM::isAdvSIMDModImmType12(Value))) {
6829 : Value = AArch64_AM::encodeAdvSIMDModImmType12(Value);
6830 : MovTy = MVT::v2f64;
6831 : }
6832 :
6833 : if (isAdvSIMDModImm) {
6834 : SDLoc dl(Op);
6835 : SDValue Mov = DAG.getNode(NewOp, dl, MovTy,
6836 : DAG.getConstant(Value, dl, MVT::i32));
6837 : return DAG.getNode(AArch64ISD::NVCAST, dl, VT, Mov);
6838 : }
6839 : }
6840 :
6841 : return SDValue();
6842 : }
6843 :
6844 : SDValue AArch64TargetLowering::LowerVectorAND(SDValue Op,
6845 : SelectionDAG &DAG) const {
6846 : SDValue LHS = Op.getOperand(0);
6847 : EVT VT = Op.getValueType();
6848 :
6849 0 : BuildVectorSDNode *BVN =
6850 0 : dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
6851 0 : if (!BVN) {
6852 : // AND commutes, so try swapping the operands.
6853 0 : LHS = Op.getOperand(1);
6854 0 : BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
6855 : }
6856 0 : if (!BVN)
6857 : return Op;
6858 0 :
6859 : APInt DefBits(VT.getSizeInBits(), 0);
6860 0 : APInt UndefBits(VT.getSizeInBits(), 0);
6861 : if (resolveBuildVector(BVN, DefBits, UndefBits)) {
6862 0 : SDValue NewOp;
6863 :
6864 : // We only have BIC vector immediate instruction, which is and-not.
6865 : DefBits = ~DefBits;
6866 0 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
6867 : DefBits, &LHS)) ||
6868 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
6869 : DefBits, &LHS)))
6870 0 : return NewOp;
6871 :
6872 : UndefBits = ~UndefBits;
6873 0 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::BICi, Op, DAG,
6874 0 : UndefBits, &LHS)) ||
6875 0 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::BICi, Op, DAG,
6876 0 : UndefBits, &LHS)))
6877 : return NewOp;
6878 : }
6879 :
6880 : // We can always fall back to a non-immediate AND.
6881 : return Op;
6882 : }
6883 :
6884 : // Specialized code to quickly find if PotentialBVec is a BuildVector that
6885 : // consists of only the same constant int value, returned in reference arg
6886 : // ConstVal
6887 : static bool isAllConstantBuildVector(const SDValue &PotentialBVec,
6888 : uint64_t &ConstVal) {
6889 0 : BuildVectorSDNode *Bvec = dyn_cast<BuildVectorSDNode>(PotentialBVec);
6890 0 : if (!Bvec)
6891 0 : return false;
6892 : ConstantSDNode *FirstElt = dyn_cast<ConstantSDNode>(Bvec->getOperand(0));
6893 0 : if (!FirstElt)
6894 0 : return false;
6895 : EVT VT = Bvec->getValueType(0);
6896 0 : unsigned NumElts = VT.getVectorNumElements();
6897 : for (unsigned i = 1; i < NumElts; ++i)
6898 0 : if (dyn_cast<ConstantSDNode>(Bvec->getOperand(i)) != FirstElt)
6899 : return false;
6900 0 : ConstVal = FirstElt->getZExtValue();
6901 : return true;
6902 0 : }
6903 :
6904 : static unsigned getIntrinsicID(const SDNode *N) {
6905 : unsigned Opcode = N->getOpcode();
6906 0 : switch (Opcode) {
6907 : default:
6908 : return Intrinsic::not_intrinsic;
6909 : case ISD::INTRINSIC_WO_CHAIN: {
6910 0 : unsigned IID = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
6911 : if (IID < Intrinsic::num_intrinsics)
6912 0 : return IID;
6913 0 : return Intrinsic::not_intrinsic;
6914 0 : }
6915 0 : }
6916 : }
6917 :
6918 : // Attempt to form a vector S[LR]I from (or (and X, BvecC1), (lsl Y, C2)),
6919 : // to (SLI X, Y, C2), where X and Y have matching vector types, BvecC1 is a
6920 : // BUILD_VECTORs with constant element C1, C2 is a constant, and C1 == ~C2.
6921 : // Also, logical shift right -> sri, with the same structure.
6922 : static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) {
6923 : EVT VT = N->getValueType(0);
6924 :
6925 : if (!VT.isVector())
6926 : return SDValue();
6927 :
6928 0 : SDLoc DL(N);
6929 0 :
6930 : // Is the first op an AND?
6931 : const SDValue And = N->getOperand(0);
6932 0 : if (And.getOpcode() != ISD::AND)
6933 0 : return SDValue();
6934 :
6935 : // Is the second op an shl or lshr?
6936 : SDValue Shift = N->getOperand(1);
6937 0 : // This will have been turned into: AArch64ISD::VSHL vector, #shift
6938 : // or AArch64ISD::VLSHR vector, #shift
6939 : unsigned ShiftOpc = Shift.getOpcode();
6940 : if ((ShiftOpc != AArch64ISD::VSHL && ShiftOpc != AArch64ISD::VLSHR))
6941 0 : return SDValue();
6942 : bool IsShiftRight = ShiftOpc == AArch64ISD::VLSHR;
6943 0 :
6944 0 : // Is the shift amount constant?
6945 0 : ConstantSDNode *C2node = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
6946 0 : if (!C2node)
6947 : return SDValue();
6948 :
6949 : // Is the and mask vector all constant?
6950 : uint64_t C1;
6951 0 : if (!isAllConstantBuildVector(And.getOperand(1), C1))
6952 : return SDValue();
6953 0 :
6954 0 : // Is C1 == ~C2, taking into account how much one can shift elements of a
6955 : // particular size?
6956 : uint64_t C2 = C2node->getZExtValue();
6957 : unsigned ElemSizeInBits = VT.getScalarSizeInBits();
6958 0 : if (C2 > ElemSizeInBits)
6959 : return SDValue();
6960 : unsigned ElemMask = (1 << ElemSizeInBits) - 1;
6961 : if ((C1 & ElemMask) != (~C2 & ElemMask))
6962 0 : return SDValue();
6963 :
6964 0 : SDValue X = And.getOperand(0);
6965 0 : SDValue Y = Shift.getOperand(0);
6966 0 :
6967 0 : unsigned Intrin =
6968 : IsShiftRight ? Intrinsic::aarch64_neon_vsri : Intrinsic::aarch64_neon_vsli;
6969 : SDValue ResultSLI =
6970 : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
6971 : DAG.getConstant(Intrin, DL, MVT::i32), X, Y,
6972 0 : Shift.getOperand(1));
6973 0 :
6974 : LLVM_DEBUG(dbgs() << "aarch64-lower: transformed: \n");
6975 0 : LLVM_DEBUG(N->dump(&DAG));
6976 : LLVM_DEBUG(dbgs() << "into: \n");
6977 0 : LLVM_DEBUG(ResultSLI->dump(&DAG));
6978 :
6979 : ++NumShiftInserts;
6980 : return ResultSLI;
6981 0 : }
6982 0 :
6983 : SDValue AArch64TargetLowering::LowerVectorOR(SDValue Op,
6984 0 : SelectionDAG &DAG) const {
6985 0 : // Attempt to form a vector S[LR]I from (or (and X, C1), (lsl Y, C2))
6986 : if (EnableAArch64SlrGeneration) {
6987 : if (SDValue Res = tryLowerToSLI(Op.getNode(), DAG))
6988 : return Res;
6989 0 : }
6990 :
6991 : EVT VT = Op.getValueType();
6992 1006 :
6993 : SDValue LHS = Op.getOperand(0);
6994 1006 : BuildVectorSDNode *BVN =
6995 1006 : dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode());
6996 : if (!BVN) {
6997 : // OR commutes, so try swapping the operands.
6998 1006 : LHS = Op.getOperand(1);
6999 : BVN = dyn_cast<BuildVectorSDNode>(Op.getOperand(0).getNode());
7000 : }
7001 812 : if (!BVN)
7002 812 : return Op;
7003 :
7004 219 : APInt DefBits(VT.getSizeInBits(), 0);
7005 787 : APInt UndefBits(VT.getSizeInBits(), 0);
7006 : if (resolveBuildVector(BVN, DefBits, UndefBits)) {
7007 219 : SDValue NewOp;
7008 219 :
7009 219 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
7010 : DefBits, &LHS)) ||
7011 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
7012 : DefBits, &LHS)))
7013 217 : return NewOp;
7014 217 :
7015 217 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::ORRi, Op, DAG,
7016 137 : UndefBits, &LHS)) ||
7017 217 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::ORRi, Op, DAG,
7018 80 : UndefBits, &LHS)))
7019 : return NewOp;
7020 137 : }
7021 137 :
7022 137 : // We can always fall back to a non-immediate OR.
7023 137 : return Op;
7024 137 : }
7025 0 :
7026 : // Normalize the operands of BUILD_VECTOR. The value of constant operands will
7027 : // be truncated to fit element width.
7028 : static SDValue NormalizeBuildVector(SDValue Op,
7029 139 : SelectionDAG &DAG) {
7030 : assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!");
7031 : SDLoc dl(Op);
7032 : EVT VT = Op.getValueType();
7033 : EVT EltTy= VT.getVectorElementType();
7034 :
7035 8 : if (EltTy.isFloatingPoint() || EltTy.getSizeInBits() > 16)
7036 : return Op;
7037 :
7038 : SmallVector<SDValue, 16> Ops;
7039 : for (SDValue Lane : Op->ops()) {
7040 6 : // For integer vectors, type legalization would have promoted the
7041 : // operands already. Otherwise, if Op is a floating-point splat
7042 : // (with operands cast to integers), then the only possibilities
7043 12 : // are constants and UNDEFs.
7044 : if (auto *CstLane = dyn_cast<ConstantSDNode>(Lane)) {
7045 96 : APInt LowBits(EltTy.getSizeInBits(),
7046 90 : CstLane->getZExtValue());
7047 : Lane = DAG.getConstant(LowBits.getZExtValue(), dl, MVT::i32);
7048 6 : } else if (Lane.getNode()->isUndef()) {
7049 6 : Lane = DAG.getUNDEF(MVT::i32);
7050 : } else {
7051 : assert(Lane.getValueType() == MVT::i32 &&
7052 : "Unexpected BUILD_VECTOR operand type");
7053 0 : }
7054 0 : Ops.push_back(Lane);
7055 : }
7056 : return DAG.getBuildVector(VT, dl, Ops);
7057 0 : }
7058 108 :
7059 54 : static SDValue ConstantBuildVector(SDValue Op, SelectionDAG &DAG) {
7060 : EVT VT = Op.getValueType();
7061 :
7062 : APInt DefBits(VT.getSizeInBits(), 0);
7063 : APInt UndefBits(VT.getSizeInBits(), 0);
7064 : BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
7065 : if (resolveBuildVector(BVN, DefBits, UndefBits)) {
7066 : SDValue NewOp;
7067 : if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
7068 : (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
7069 : (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
7070 8 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
7071 16 : (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
7072 : (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
7073 8 : return NewOp;
7074 0 :
7075 : DefBits = ~DefBits;
7076 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
7077 : (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
7078 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
7079 8 : return NewOp;
7080 8 :
7081 0 : DefBits = UndefBits;
7082 : if ((NewOp = tryAdvSIMDModImm64(AArch64ISD::MOVIedit, Op, DAG, DefBits)) ||
7083 : (NewOp = tryAdvSIMDModImm32(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
7084 8 : (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MOVImsl, Op, DAG, DefBits)) ||
7085 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::MOVIshift, Op, DAG, DefBits)) ||
7086 : (NewOp = tryAdvSIMDModImm8(AArch64ISD::MOVI, Op, DAG, DefBits)) ||
7087 : (NewOp = tryAdvSIMDModImmFP(AArch64ISD::FMOV, Op, DAG, DefBits)))
7088 8 : return NewOp;
7089 0 :
7090 : DefBits = ~UndefBits;
7091 : if ((NewOp = tryAdvSIMDModImm32(AArch64ISD::MVNIshift, Op, DAG, DefBits)) ||
7092 : (NewOp = tryAdvSIMDModImm321s(AArch64ISD::MVNImsl, Op, DAG, DefBits)) ||
7093 : (NewOp = tryAdvSIMDModImm16(AArch64ISD::MVNIshift, Op, DAG, DefBits)))
7094 : return NewOp;
7095 0 : }
7096 :
7097 : return SDValue();
7098 : }
7099 8 :
7100 2 : SDValue AArch64TargetLowering::LowerBUILD_VECTOR(SDValue Op,
7101 : SelectionDAG &DAG) const {
7102 : EVT VT = Op.getValueType();
7103 :
7104 6 : // Try to build a simple constant vector.
7105 : Op = NormalizeBuildVector(Op, DAG);
7106 6 : if (VT.isInteger()) {
7107 0 : // Certain vector constants, used to express things like logical NOT and
7108 6 : // arithmetic NEG, are passed through unmodified. This allows special
7109 6 : // patterns for these operations to match, which will lower these constants
7110 4 : // to whatever is proven necessary.
7111 : BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
7112 2 : if (BVN->isConstant())
7113 2 : if (ConstantSDNode *Const = BVN->getConstantSplatNode()) {
7114 : unsigned BitSize = VT.getVectorElementType().getSizeInBits();
7115 : APInt Val(BitSize,
7116 2 : Const->getAPIntValue().zextOrTrunc(BitSize).getZExtValue());
7117 : if (Val.isNullValue() || Val.isAllOnesValue())
7118 : return Op;
7119 : }
7120 2 : }
7121 :
7122 : if (SDValue V = ConstantBuildVector(Op, DAG))
7123 : return V;
7124 :
7125 : // Scan through the operands to find some interesting properties we can
7126 : // exploit:
7127 : // 1) If only one value is used, we can use a DUP, or
7128 2 : // 2) if only the low element is not undef, we can just insert that, or
7129 : // 3) if only one constant value is used (w/ some non-constant lanes),
7130 : // we can splat the constant value into the whole vector then fill
7131 688 : // in the non-constant lanes.
7132 : // 4) FIXME: If different constant values are used, but we can intelligently
7133 : // select the values we'll be overwriting for the non-constant
7134 688 : // lanes such that we can directly materialize the vector
7135 8 : // some other way (MOVI, e.g.), we can be sneaky.
7136 2 : // 5) if all operands are EXTRACT_VECTOR_ELT, check for VUZP.
7137 : SDLoc dl(Op);
7138 : unsigned NumElts = VT.getVectorNumElements();
7139 686 : bool isOnlyLowElement = true;
7140 : bool usesOnlyOneValue = true;
7141 686 : bool usesOnlyOneConstantValue = true;
7142 : bool isConstant = true;
7143 686 : bool AllLanesExtractElt = true;
7144 : unsigned NumConstantLanes = 0;
7145 : SDValue Value;
7146 621 : SDValue ConstantValue;
7147 621 : for (unsigned i = 0; i < NumElts; ++i) {
7148 : SDValue V = Op.getOperand(i);
7149 65 : if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7150 621 : AllLanesExtractElt = false;
7151 : if (V.isUndef())
7152 65 : continue;
7153 65 : if (i > 0)
7154 65 : isOnlyLowElement = false;
7155 : if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7156 : isConstant = false;
7157 65 :
7158 65 : if (isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V)) {
7159 65 : ++NumConstantLanes;
7160 65 : if (!ConstantValue.getNode())
7161 63 : ConstantValue = V;
7162 : else if (ConstantValue != V)
7163 2 : usesOnlyOneConstantValue = false;
7164 2 : }
7165 2 :
7166 2 : if (!Value.getNode())
7167 0 : Value = V;
7168 : else if (V != Value)
7169 : usesOnlyOneValue = false;
7170 : }
7171 2 :
7172 : if (!Value.getNode()) {
7173 : LLVM_DEBUG(
7174 : dbgs() << "LowerBUILD_VECTOR: value undefined, creating undef node\n");
7175 : return DAG.getUNDEF(VT);
7176 2165 : }
7177 :
7178 : if (isOnlyLowElement) {
7179 : LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: only low element used, creating 1 "
7180 2165 : "SCALAR_TO_VECTOR node\n");
7181 2165 : return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
7182 : }
7183 2165 :
7184 1388 : if (AllLanesExtractElt) {
7185 : SDNode *Vector = nullptr;
7186 : bool Even = false;
7187 7201 : bool Odd = false;
7188 : // Check whether the extract elements match the Even pattern <0,2,4,...> or
7189 : // the Odd pattern <1,3,5,...>.
7190 : for (unsigned i = 0; i < NumElts; ++i) {
7191 : SDValue V = Op.getOperand(i);
7192 : const SDNode *N = V.getNode();
7193 : if (!isa<ConstantSDNode>(N->getOperand(1)))
7194 9756 : break;
7195 4878 : SDValue N0 = N->getOperand(0);
7196 1546 :
7197 602 : // All elements are extracted from the same vector.
7198 : if (!Vector) {
7199 : Vector = N0.getNode();
7200 : // Check that the type of EXTRACT_VECTOR_ELT matches the type of
7201 : // BUILD_VECTOR.
7202 6424 : if (VT.getVectorElementType() !=
7203 : N0.getValueType().getVectorElementType())
7204 777 : break;
7205 : } else if (Vector != N0.getNode()) {
7206 : Odd = false;
7207 0 : Even = false;
7208 0 : break;
7209 : }
7210 0 :
7211 0 : // Extracted values are either at Even indices <0,2,4,...> or at Odd
7212 : // indices <1,3,5,...>.
7213 0 : uint64_t Val = N->getConstantOperandVal(1);
7214 : if (Val == 2 * i) {
7215 0 : Even = true;
7216 0 : continue;
7217 0 : }
7218 0 : if (Val - 1 == 2 * i) {
7219 0 : Odd = true;
7220 0 : continue;
7221 0 : }
7222 :
7223 0 : // Something does not match: abort.
7224 0 : Odd = false;
7225 0 : Even = false;
7226 0 : break;
7227 0 : }
7228 : if (Even || Odd) {
7229 0 : SDValue LHS =
7230 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
7231 0 : DAG.getConstant(0, dl, MVT::i64));
7232 0 : SDValue RHS =
7233 0 : DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, SDValue(Vector, 0),
7234 0 : DAG.getConstant(NumElts, dl, MVT::i64));
7235 0 :
7236 0 : if (Even && !Odd)
7237 : return DAG.getNode(AArch64ISD::UZP1, dl, DAG.getVTList(VT, VT), LHS,
7238 0 : RHS);
7239 0 : if (Odd && !Even)
7240 0 : return DAG.getNode(AArch64ISD::UZP2, dl, DAG.getVTList(VT, VT), LHS,
7241 0 : RHS);
7242 0 : }
7243 : }
7244 :
7245 0 : // Use DUP for non-constant splats. For f32 constant splats, reduce to
7246 : // i32 and try again.
7247 : if (usesOnlyOneValue) {
7248 2165 : if (!isConstant) {
7249 : if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7250 2165 : Value.getValueType() != VT) {
7251 : LLVM_DEBUG(
7252 : dbgs() << "LowerBUILD_VECTOR: use DUP for non-constant splats\n");
7253 2165 : return DAG.getNode(AArch64ISD::DUP, dl, VT, Value);
7254 2165 : }
7255 :
7256 : // This is actually a DUPLANExx operation, which keeps everything vectory.
7257 :
7258 : SDValue Lane = Value.getOperand(1);
7259 : Value = Value.getOperand(0);
7260 1872 : if (Value.getValueSizeInBits() == 64) {
7261 1461 : LLVM_DEBUG(
7262 1316 : dbgs() << "LowerBUILD_VECTOR: DUPLANE works on 128-bit vectors, "
7263 : "widening it\n");
7264 3948 : Value = WidenVector(Value, DAG);
7265 2180 : }
7266 1005 :
7267 : unsigned Opcode = getDUPLANEOp(VT.getVectorElementType());
7268 : return DAG.getNode(Opcode, dl, VT, Value, Lane);
7269 : }
7270 1160 :
7271 382 : if (VT.getVectorElementType().isFloatingPoint()) {
7272 : SmallVector<SDValue, 8> Ops;
7273 : EVT EltTy = VT.getVectorElementType();
7274 : assert ((EltTy == MVT::f16 || EltTy == MVT::f32 || EltTy == MVT::f64) &&
7275 : "Unsupported floating-point vector type");
7276 : LLVM_DEBUG(
7277 : dbgs() << "LowerBUILD_VECTOR: float constant splats, creating int "
7278 : "BITCASTS, and try again\n");
7279 : MVT NewType = MVT::getIntegerVT(EltTy.getSizeInBits());
7280 : for (unsigned i = 0; i < NumElts; ++i)
7281 : Ops.push_back(DAG.getNode(ISD::BITCAST, dl, NewType, Op.getOperand(i)));
7282 : EVT VecVT = EVT::getVectorVT(*DAG.getContext(), NewType, NumElts);
7283 : SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7284 : LLVM_DEBUG(dbgs() << "LowerBUILD_VECTOR: trying to lower new vector: ";
7285 : Val.dump(););
7286 : Val = LowerBUILD_VECTOR(Val, DAG);
7287 : if (Val.getNode())
7288 : return DAG.getNode(ISD::BITCAST, dl, VT, Val);
7289 : }
7290 : }
7291 :
7292 : // If there was only one constant value used and for more than one lane,
7293 778 : // start by splatting that value, then replace the non-constant lanes. This
7294 : // is better than the default, which will perform a separate initialization
7295 4350 : // for each lane.
7296 3572 : if (NumConstantLanes > 0 && usesOnlyOneConstantValue) {
7297 3572 : // Firstly, try to materialize the splat constant.
7298 : SDValue Vec = DAG.getSplatBuildVector(VT, dl, ConstantValue),
7299 3572 : Val = ConstantBuildVector(Vec, DAG);
7300 737 : if (!Val) {
7301 2835 : // Otherwise, materialize the constant and splat it.
7302 : Val = DAG.getNode(AArch64ISD::DUP, dl, VT, ConstantValue);
7303 : DAG.ReplaceAllUsesWith(Vec.getNode(), &Val);
7304 : }
7305 :
7306 : // Now insert the non-constant lanes.
7307 915 : for (unsigned i = 0; i < NumElts; ++i) {
7308 915 : SDValue V = Op.getOperand(i);
7309 155 : SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
7310 : if (!isa<ConstantSDNode>(V) && !isa<ConstantFPSDNode>(V))
7311 : // Note that type legalization likely mucked about with the VT of the
7312 : // source operand, so we may have to convert it here before inserting.
7313 : Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Val, V, LaneIdx);
7314 2835 : }
7315 778 : return Val;
7316 : }
7317 :
7318 : // This will generate a load from the constant pool.
7319 : if (isConstant) {
7320 778 : LLVM_DEBUG(
7321 : dbgs() << "LowerBUILD_VECTOR: all elements are constant, use default "
7322 : "expansion\n");
7323 0 : return SDValue();
7324 : }
7325 :
7326 778 : // Empirical tests suggest this is rarely worth it for vectors of length <= 2.
7327 : if (NumElts >= 4) {
7328 : if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7329 236 : return shuffle;
7330 : }
7331 :
7332 542 : // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7333 : // know the default expansion would otherwise fall back on something even
7334 : // worse. For a vector with one or two non-undef values, that's
7335 : // scalar_to_vector for the elements followed by a shuffle (provided the
7336 : // shuffle is valid for the target) and materialization element by element
7337 : // on the stack followed by a load for everything else.
7338 10 : if (!isConstant && !usesOnlyOneValue) {
7339 10 : LLVM_DEBUG(
7340 : dbgs() << "LowerBUILD_VECTOR: alternatives failed, creating sequence "
7341 10 : "of INSERT_VECTOR_ELT\n");
7342 :
7343 9 : SDValue Vec = DAG.getUNDEF(VT);
7344 : SDValue Op0 = Op.getOperand(0);
7345 : unsigned i = 0;
7346 9 :
7347 : // Use SCALAR_TO_VECTOR for lane zero to
7348 : // a) Avoid a RMW dependency on the full vector register, and
7349 : // b) Allow the register coalescer to fold away the copy if the
7350 9 : // value is already in an S or D register, and we're forced to emit an
7351 18 : // INSERT_SUBREG that we can't fold anywhere.
7352 : //
7353 0 : // We also allow types like i8 and i16 which are illegal scalar but legal
7354 : // vector element types. After type-legalization the inserted value is
7355 : // extended (i32) and it is safe to cast them to the vector type by ignoring
7356 : // the upper bits of the lowest lane (e.g. v8i8, v4i16).
7357 : if (!Op0.isUndef()) {
7358 : LLVM_DEBUG(dbgs() << "Creating node for op0, it is not undefined:\n");
7359 : Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op0);
7360 : ++i;
7361 : }
7362 0 : LLVM_DEBUG(if (i < NumElts) dbgs()
7363 : << "Creating nodes for the other vector elements:\n";);
7364 : for (; i < NumElts; ++i) {
7365 : SDValue V = Op.getOperand(i);
7366 0 : if (V.isUndef())
7367 : continue;
7368 : SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i64);
7369 : Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
7370 : }
7371 : return Vec;
7372 : }
7373 :
7374 : LLVM_DEBUG(
7375 : dbgs() << "LowerBUILD_VECTOR: use default expansion, failed to find "
7376 10 : "better alternative\n");
7377 : return SDValue();
7378 : }
7379 0 :
7380 : SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
7381 : SelectionDAG &DAG) const {
7382 0 : assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT && "Unknown opcode!");
7383 :
7384 0 : // Check for non-constant or out of range lane.
7385 : EVT VT = Op.getOperand(0).getValueType();
7386 0 : ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(2));
7387 0 : if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
7388 : return SDValue();
7389 0 :
7390 :
7391 : // Insertion/extraction are legal for V128 types.
7392 : if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
7393 : VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
7394 : VT == MVT::v8f16)
7395 542 : return Op;
7396 195 :
7397 320 : if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
7398 18 : VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16)
7399 : return SDValue();
7400 :
7401 160 : // For V64 types, we perform insertion by expanding the value
7402 : // to a V128 type and perform the insertion on that.
7403 : SDLoc DL(Op);
7404 : SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
7405 : EVT WideTy = WideVec.getValueType();
7406 0 :
7407 0 : SDValue Node = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideTy, WideVec,
7408 0 : Op.getOperand(1), Op.getOperand(2));
7409 : // Re-narrow the resultant vector.
7410 : return NarrowVector(Node, DAG);
7411 : }
7412 0 :
7413 : SDValue
7414 : AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
7415 0 : SelectionDAG &DAG) const {
7416 0 : assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
7417 :
7418 : // Check for non-constant or out of range lane.
7419 35 : EVT VT = Op.getOperand(0).getValueType();
7420 : ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1));
7421 4 : if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
7422 : return SDValue();
7423 :
7424 :
7425 : // Insertion/extraction are legal for V128 types.
7426 : if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
7427 4 : VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
7428 18 : VT == MVT::v8f16)
7429 14 : return Op;
7430 4 :
7431 4 : if (VT != MVT::v8i8 && VT != MVT::v4i16 && VT != MVT::v2i32 &&
7432 : VT != MVT::v1i64 && VT != MVT::v2f32 && VT != MVT::v4f16)
7433 : return SDValue();
7434 4 :
7435 4 : // For V64 types, we perform extraction by expanding the value
7436 4 : // to a V128 type and perform the extraction on that.
7437 : SDLoc DL(Op);
7438 : SDValue WideVec = WidenVector(Op.getOperand(0), DAG);
7439 : EVT WideTy = WideVec.getValueType();
7440 :
7441 : EVT ExtrTy = WideTy.getVectorElementType();
7442 : if (ExtrTy == MVT::i16 || ExtrTy == MVT::i8)
7443 : ExtrTy = MVT::i32;
7444 378 :
7445 : // For extractions, we just return the result directly.
7446 60 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtrTy, WideVec,
7447 60 : Op.getOperand(1));
7448 60 : }
7449 :
7450 31 : SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op,
7451 31 : SelectionDAG &DAG) const {
7452 : EVT VT = Op.getOperand(0).getValueType();
7453 : SDLoc dl(Op);
7454 : // Just in case...
7455 326 : if (!VT.isVector())
7456 266 : return SDValue();
7457 266 :
7458 : ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op.getOperand(1));
7459 : if (!Cst)
7460 : return SDValue();
7461 31 : unsigned Val = Cst->getZExtValue();
7462 :
7463 60 : unsigned Size = Op.getValueSizeInBits();
7464 :
7465 : // This will get lowered to an appropriate EXTRACT_SUBREG in ISel.
7466 : if (Val == 0)
7467 318 : return Op;
7468 :
7469 : // If this is extracting the upper 64-bits of a 128-bit vector, we match
7470 : // that directly.
7471 87 : if (Size == 64 && Val * VT.getScalarSizeInBits() == 64)
7472 : return Op;
7473 :
7474 : return SDValue();
7475 231 : }
7476 140 :
7477 3 : bool AArch64TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7478 : if (VT.getVectorNumElements() == 4 &&
7479 : (VT.is128BitVector() || VT.is64BitVector())) {
7480 : unsigned PFIndexes[4];
7481 : for (unsigned i = 0; i != 4; ++i) {
7482 : if (M[i] < 0)
7483 : PFIndexes[i] = 8;
7484 : else
7485 : PFIndexes[i] = M[i];
7486 228 : }
7487 :
7488 : // Compute the index in the perfect shuffle table.
7489 : unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
7490 : PFIndexes[2] * 9 + PFIndexes[3];
7491 228 : unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
7492 228 : unsigned Cost = (PFEntry >> 30);
7493 :
7494 : if (Cost <= 4)
7495 : return true;
7496 : }
7497 :
7498 : bool DummyBool;
7499 : int DummyInt;
7500 : unsigned DummyUnsigned;
7501 :
7502 : return (ShuffleVectorSDNode::isSplatMask(&M[0], VT) || isREVMask(M, VT, 64) ||
7503 : isREVMask(M, VT, 32) || isREVMask(M, VT, 16) ||
7504 : isEXTMask(M, VT, DummyBool, DummyUnsigned) ||
7505 228 : // isTBLMask(M, VT) || // FIXME: Port TBL support from ARM.
7506 : isTRNMask(M, VT, DummyUnsigned) || isUZPMask(M, VT, DummyUnsigned) ||
7507 228 : isZIPMask(M, VT, DummyUnsigned) ||
7508 : isTRN_v_undef_Mask(M, VT, DummyUnsigned) ||
7509 : isUZP_v_undef_Mask(M, VT, DummyUnsigned) ||
7510 : isZIP_v_undef_Mask(M, VT, DummyUnsigned) ||
7511 : isINSMask(M, VT.getVectorNumElements(), DummyBool, DummyInt) ||
7512 1002 : isConcatMask(M, VT, VT.getSizeInBits() == 128));
7513 774 : }
7514 774 :
7515 8 : /// getVShiftImm - Check if this is a valid build_vector for the immediate
7516 766 : /// operand of a vector shift operation, where all the elements of the
7517 766 : /// build_vector must have the same constant integer value.
7518 : static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
7519 228 : // Ignore bit_converts.
7520 : while (Op.getOpcode() == ISD::BITCAST)
7521 : Op = Op.getOperand(0);
7522 : BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
7523 : APInt SplatBits, SplatUndef;
7524 : unsigned SplatBitSize;
7525 0 : bool HasAnyUndefs;
7526 : if (!BVN || !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize,
7527 : HasAnyUndefs, ElementBits) ||
7528 2130 : SplatBitSize > ElementBits)
7529 : return false;
7530 : Cnt = SplatBits.getSExtValue();
7531 : return true;
7532 : }
7533 4260 :
7534 : /// isVShiftLImm - Check if this is a valid build_vector for the immediate
7535 4252 : /// operand of a vector shift left operation. That value must be in the range:
7536 4 : /// 0 <= Value < ElementBits for a left shift; or
7537 : /// 0 <= Value <= ElementBits for a long left shift.
7538 : static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
7539 : assert(VT.isVector() && "vector shift count is not a vector type");
7540 : int64_t ElementBits = VT.getScalarSizeInBits();
7541 : if (!getVShiftImm(Op, ElementBits, Cnt))
7542 : return false;
7543 1874 : return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
7544 : }
7545 :
7546 : /// isVShiftRImm - Check if this is a valid build_vector for the immediate
7547 2 : /// operand of a vector shift right operation. The value must be in the range:
7548 : /// 1 <= Value <= ElementBits for a right shift; or
7549 : static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) {
7550 : assert(VT.isVector() && "vector shift count is not a vector type");
7551 : int64_t ElementBits = VT.getScalarSizeInBits();
7552 250 : if (!getVShiftImm(Op, ElementBits, Cnt))
7553 500 : return false;
7554 : return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
7555 : }
7556 250 :
7557 : SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op,
7558 250 : SelectionDAG &DAG) const {
7559 : EVT VT = Op.getValueType();
7560 : SDLoc DL(Op);
7561 : int64_t Cnt;
7562 5267 :
7563 : if (!Op.getOperand(1).getValueType().isVector())
7564 : return Op;
7565 : unsigned EltSize = VT.getScalarSizeInBits();
7566 :
7567 10534 : switch (Op.getOpcode()) {
7568 : default:
7569 10528 : llvm_unreachable("unexpected shift opcode");
7570 3 :
7571 : case ISD::SHL:
7572 : if (isVShiftLImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize)
7573 : return DAG.getNode(AArch64ISD::VSHL, DL, VT, Op.getOperand(0),
7574 : DAG.getConstant(Cnt, DL, MVT::i32));
7575 : return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
7576 : DAG.getConstant(Intrinsic::aarch64_neon_ushl, DL,
7577 4534 : MVT::i32),
7578 : Op.getOperand(0), Op.getOperand(1));
7579 : case ISD::SRA:
7580 : case ISD::SRL:
7581 69 : // Right shift immediate
7582 : if (isVShiftRImm(Op.getOperand(1), VT, false, Cnt) && Cnt < EltSize) {
7583 : unsigned Opc =
7584 : (Op.getOpcode() == ISD::SRA) ? AArch64ISD::VASHR : AArch64ISD::VLSHR;
7585 : return DAG.getNode(Opc, DL, VT, Op.getOperand(0),
7586 661 : DAG.getConstant(Cnt, DL, MVT::i32));
7587 661 : }
7588 :
7589 661 : // Right shift register. Note, there is not a shift right register
7590 : // instruction, but the shift left register instruction takes a signed
7591 : // value, where negative numbers specify a right shift.
7592 : unsigned Opc = (Op.getOpcode() == ISD::SRA) ? Intrinsic::aarch64_neon_sshl
7593 : : Intrinsic::aarch64_neon_ushl;
7594 : // negate the shift amount
7595 661 : SDValue NegShift = DAG.getNode(AArch64ISD::NEG, DL, VT, Op.getOperand(1));
7596 : SDValue NegShiftLeft =
7597 : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT,
7598 1736 : DAG.getConstant(Opc, DL, MVT::i32), Op.getOperand(0),
7599 : NegShift);
7600 5208 : return NegShiftLeft;
7601 : }
7602 :
7603 1736 : return SDValue();
7604 0 : }
7605 :
7606 1736 : static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS,
7607 : AArch64CC::CondCode CC, bool NoNans, EVT VT,
7608 0 : const SDLoc &dl, SelectionDAG &DAG) {
7609 1736 : EVT SrcVT = LHS.getValueType();
7610 : assert(VT.getSizeInBits() == SrcVT.getSizeInBits() &&
7611 1736 : "function only supposed to emit natural comparisons");
7612 :
7613 : BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(RHS.getNode());
7614 1736 : APInt CnstBits(VT.getSizeInBits(), 0);
7615 333 : APInt UndefBits(VT.getSizeInBits(), 0);
7616 : bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits);
7617 : bool IsZero = IsCnst && (CnstBits == 0);
7618 :
7619 2806 : if (SrcVT.getVectorElementType().isFloatingPoint()) {
7620 1403 : switch (CC) {
7621 : default:
7622 0 : return SDValue();
7623 : case AArch64CC::NE: {
7624 : SDValue Fcmeq;
7625 62 : if (IsZero)
7626 62 : Fcmeq = DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
7627 10 : else
7628 : Fcmeq = DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
7629 : return DAG.getNode(AArch64ISD::NOT, dl, VT, Fcmeq);
7630 : }
7631 : case AArch64CC::EQ:
7632 : if (IsZero)
7633 : return DAG.getNode(AArch64ISD::FCMEQz, dl, VT, LHS);
7634 : return DAG.getNode(AArch64ISD::FCMEQ, dl, VT, LHS, RHS);
7635 : case AArch64CC::GE:
7636 : if (IsZero)
7637 : return DAG.getNode(AArch64ISD::FCMGEz, dl, VT, LHS);
7638 : return DAG.getNode(AArch64ISD::FCMGE, dl, VT, LHS, RHS);
7639 : case AArch64CC::GT:
7640 : if (IsZero)
7641 : return DAG.getNode(AArch64ISD::FCMGTz, dl, VT, LHS);
7642 : return DAG.getNode(AArch64ISD::FCMGT, dl, VT, LHS, RHS);
7643 : case AArch64CC::LS:
7644 : if (IsZero)
7645 : return DAG.getNode(AArch64ISD::FCMLEz, dl, VT, LHS);
7646 : return DAG.getNode(AArch64ISD::FCMGE, dl, VT, RHS, LHS);
7647 : case AArch64CC::LT:
7648 : if (!NoNans)
7649 : return SDValue();
7650 142 : // If we ignore NaNs then we can use to the MI implementation.
7651 133 : LLVM_FALLTHROUGH;
7652 84 : case AArch64CC::MI:
7653 : if (IsZero)
7654 72 : return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS);
7655 31 : return DAG.getNode(AArch64ISD::FCMGT, dl, VT, RHS, LHS);
7656 30 : }
7657 30 : }
7658 30 :
7659 67 : switch (CC) {
7660 13 : default:
7661 : return SDValue();
7662 : case AArch64CC::NE: {
7663 : SDValue Cmeq;
7664 : if (IsZero)
7665 : Cmeq = DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
7666 280 : else
7667 : Cmeq = DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
7668 280 : return DAG.getNode(AArch64ISD::NOT, dl, VT, Cmeq);
7669 0 : }
7670 : case AArch64CC::EQ:
7671 : if (IsZero)
7672 : return DAG.getNode(AArch64ISD::CMEQz, dl, VT, LHS);
7673 : return DAG.getNode(AArch64ISD::CMEQ, dl, VT, LHS, RHS);
7674 272 : case AArch64CC::GE:
7675 280 : if (IsZero)
7676 272 : return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS);
7677 14 : return DAG.getNode(AArch64ISD::CMGE, dl, VT, LHS, RHS);
7678 266 : case AArch64CC::GT:
7679 266 : if (IsZero)
7680 : return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS);
7681 : return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS);
7682 : case AArch64CC::LE:
7683 : if (IsZero)
7684 : return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS);
7685 : return DAG.getNode(AArch64ISD::CMGE, dl, VT, RHS, LHS);
7686 57 : case AArch64CC::LS:
7687 : return DAG.getNode(AArch64ISD::CMHS, dl, VT, RHS, LHS);
7688 57 : case AArch64CC::LO:
7689 57 : return DAG.getNode(AArch64ISD::CMHI, dl, VT, RHS, LHS);
7690 : case AArch64CC::LT:
7691 55 : if (IsZero)
7692 : return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS);
7693 : return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS);
7694 : case AArch64CC::HI:
7695 : return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS);
7696 : case AArch64CC::HS:
7697 223 : return DAG.getNode(AArch64ISD::CMHS, dl, VT, LHS, RHS);
7698 : }
7699 223 : }
7700 223 :
7701 : SDValue AArch64TargetLowering::LowerVSETCC(SDValue Op,
7702 211 : SelectionDAG &DAG) const {
7703 : ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
7704 : SDValue LHS = Op.getOperand(0);
7705 280 : SDValue RHS = Op.getOperand(1);
7706 : EVT CmpVT = LHS.getValueType().changeVectorElementTypeToInteger();
7707 280 : SDLoc dl(Op);
7708 :
7709 : if (LHS.getValueType().getVectorElementType().isInteger()) {
7710 : assert(LHS.getValueType() == RHS.getValueType());
7711 840 : AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
7712 0 : SDValue Cmp =
7713 : EmitVectorComparison(LHS, RHS, AArch64CC, false, CmpVT, dl, DAG);
7714 : return DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
7715 280 : }
7716 0 :
7717 0 : const bool FullFP16 =
7718 : static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
7719 57 :
7720 57 : // Make v4f16 (only) fcmp operations utilise vector instructions
7721 : // v8f16 support will be a litle more complicated
7722 55 : if (LHS.getValueType().getVectorElementType() == MVT::f16) {
7723 : if (!FullFP16 && LHS.getValueType().getVectorNumElements() == 4) {
7724 : LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, LHS);
7725 : RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::v4f32, RHS);
7726 2 : SDValue NewSetcc = DAG.getSetCC(dl, MVT::v4i16, LHS, RHS, CC);
7727 223 : DAG.ReplaceAllUsesWith(Op, NewSetcc);
7728 : CmpVT = MVT::v4i32;
7729 : } else
7730 223 : return SDValue();
7731 : }
7732 211 :
7733 : assert(LHS.getValueType().getVectorElementType() == MVT::f32 ||
7734 211 : LHS.getValueType().getVectorElementType() == MVT::f64);
7735 :
7736 : // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
7737 : // clean. Some of them require two branches to implement.
7738 : AArch64CC::CondCode CC1, CC2;
7739 : bool ShouldInvert;
7740 12 : changeVectorFPCCToAArch64CC(CC, CC1, CC2, ShouldInvert);
7741 :
7742 : bool NoNaNs = getTargetMachine().Options.NoNaNsFPMath;
7743 12 : SDValue Cmp =
7744 : EmitVectorComparison(LHS, RHS, CC1, NoNaNs, CmpVT, dl, DAG);
7745 : if (!Cmp.getNode())
7746 : return SDValue();
7747 12 :
7748 12 : if (CC2 != AArch64CC::AL) {
7749 : SDValue Cmp2 =
7750 : EmitVectorComparison(LHS, RHS, CC2, NoNaNs, CmpVT, dl, DAG);
7751 : if (!Cmp2.getNode())
7752 : return SDValue();
7753 :
7754 555 : Cmp = DAG.getNode(ISD::OR, dl, CmpVT, Cmp, Cmp2);
7755 : }
7756 :
7757 555 : Cmp = DAG.getSExtOrTrunc(Cmp, dl, Op.getValueType());
7758 :
7759 : if (ShouldInvert)
7760 : return Cmp = DAG.getNOT(dl, Cmp, Cmp.getValueType());
7761 555 :
7762 555 : return Cmp;
7763 555 : }
7764 555 :
7765 257 : static SDValue getReductionSDNode(unsigned Op, SDLoc DL, SDValue ScalarOp,
7766 : SelectionDAG &DAG) {
7767 555 : SDValue VecOp = ScalarOp.getOperand(0);
7768 182 : auto Rdx = DAG.getNode(Op, DL, VecOp.getSimpleValueType(), VecOp);
7769 0 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ScalarOp.getValueType(), Rdx,
7770 0 : DAG.getConstant(0, DL, MVT::i64));
7771 8 : }
7772 8 :
7773 8 : SDValue AArch64TargetLowering::LowerVECREDUCE(SDValue Op,
7774 3 : SelectionDAG &DAG) const {
7775 : SDLoc dl(Op);
7776 5 : switch (Op.getOpcode()) {
7777 8 : case ISD::VECREDUCE_ADD:
7778 : return getReductionSDNode(AArch64ISD::UADDV, dl, Op, DAG);
7779 33 : case ISD::VECREDUCE_SMAX:
7780 33 : return getReductionSDNode(AArch64ISD::SMAXV, dl, Op, DAG);
7781 11 : case ISD::VECREDUCE_SMIN:
7782 22 : return getReductionSDNode(AArch64ISD::SMINV, dl, Op, DAG);
7783 34 : case ISD::VECREDUCE_UMAX:
7784 34 : return getReductionSDNode(AArch64ISD::UMAXV, dl, Op, DAG);
7785 13 : case ISD::VECREDUCE_UMIN:
7786 21 : return getReductionSDNode(AArch64ISD::UMINV, dl, Op, DAG);
7787 35 : case ISD::VECREDUCE_FMAX: {
7788 35 : assert(Op->getFlags().hasNoNaNs() && "fmax vector reduction needs NoNaN flag");
7789 13 : return DAG.getNode(
7790 22 : ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
7791 18 : DAG.getConstant(Intrinsic::aarch64_neon_fmaxnmv, dl, MVT::i32),
7792 18 : Op.getOperand(0));
7793 7 : }
7794 11 : case ISD::VECREDUCE_FMIN: {
7795 0 : assert(Op->getFlags().hasNoNaNs() && "fmin vector reduction needs NoNaN flag");
7796 0 : return DAG.getNode(
7797 0 : ISD::INTRINSIC_WO_CHAIN, dl, Op.getValueType(),
7798 : DAG.getConstant(Intrinsic::aarch64_neon_fminnmv, dl, MVT::i32),
7799 : Op.getOperand(0));
7800 : }
7801 54 : default:
7802 20 : llvm_unreachable("Unhandled reduction");
7803 34 : }
7804 : }
7805 :
7806 : SDValue AArch64TargetLowering::LowerATOMIC_LOAD_SUB(SDValue Op,
7807 373 : SelectionDAG &DAG) const {
7808 0 : auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
7809 0 : if (!Subtarget.hasLSE())
7810 46 : return SDValue();
7811 46 :
7812 46 : // LSE has an atomic load-add instruction, but not a load-sub.
7813 31 : SDLoc dl(Op);
7814 : MVT VT = Op.getSimpleValueType();
7815 15 : SDValue RHS = Op.getOperand(2);
7816 46 : AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
7817 : RHS = DAG.getNode(ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT), RHS);
7818 75 : return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, dl, AN->getMemoryVT(),
7819 75 : Op.getOperand(0), Op.getOperand(1), RHS,
7820 32 : AN->getMemOperand());
7821 43 : }
7822 30 :
7823 30 : SDValue AArch64TargetLowering::LowerATOMIC_LOAD_AND(SDValue Op,
7824 15 : SelectionDAG &DAG) const {
7825 15 : auto &Subtarget = static_cast<const AArch64Subtarget &>(DAG.getSubtarget());
7826 31 : if (!Subtarget.hasLSE())
7827 31 : return SDValue();
7828 15 :
7829 16 : // LSE has an atomic load-clear instruction, but not a load-and.
7830 29 : SDLoc dl(Op);
7831 29 : MVT VT = Op.getSimpleValueType();
7832 15 : SDValue RHS = Op.getOperand(2);
7833 14 : AtomicSDNode *AN = cast<AtomicSDNode>(Op.getNode());
7834 : RHS = DAG.getNode(ISD::XOR, dl, VT, DAG.getConstant(-1ULL, dl, VT), RHS);
7835 28 : return DAG.getAtomic(ISD::ATOMIC_LOAD_CLR, dl, AN->getMemoryVT(),
7836 : Op.getOperand(0), Op.getOperand(1), RHS,
7837 30 : AN->getMemOperand());
7838 30 : }
7839 30 :
7840 15 : SDValue AArch64TargetLowering::LowerWindowsDYNAMIC_STACKALLOC(
7841 15 : SDValue Op, SDValue Chain, SDValue &Size, SelectionDAG &DAG) const {
7842 : SDLoc dl(Op);
7843 45 : EVT PtrVT = getPointerTy(DAG.getDataLayout());
7844 : SDValue Callee = DAG.getTargetExternalSymbol("__chkstk", PtrVT, 0);
7845 29 :
7846 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
7847 : const uint32_t *Mask = TRI->getWindowsStackProbePreservedMask();
7848 : if (Subtarget->hasCustomCallingConv())
7849 564 : TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
7850 :
7851 564 : Size = DAG.getNode(ISD::SRL, dl, MVT::i64, Size,
7852 564 : DAG.getConstant(4, dl, MVT::i64));
7853 564 : Chain = DAG.getCopyToReg(Chain, dl, AArch64::X15, Size, SDValue());
7854 564 : Chain =
7855 : DAG.getNode(AArch64ISD::CALL, dl, DAG.getVTList(MVT::Other, MVT::Glue),
7856 : Chain, Callee, DAG.getRegister(AArch64::X15, MVT::i64),
7857 564 : DAG.getRegisterMask(Mask), Chain.getValue(1));
7858 : // To match the actual intent better, we should read the output from X15 here
7859 373 : // again (instead of potentially spilling it to the stack), but rereading Size
7860 : // from X15 here doesn't work at -O0, since it thinks that X15 is undefined
7861 373 : // here.
7862 373 :
7863 : Size = DAG.getNode(ISD::SHL, dl, MVT::i64, Size,
7864 : DAG.getConstant(4, dl, MVT::i64));
7865 : return Chain;
7866 382 : }
7867 :
7868 : SDValue
7869 : AArch64TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
7870 326 : SelectionDAG &DAG) const {
7871 84 : assert(Subtarget->isTargetWindows() &&
7872 14 : "Only Windows alloca probing supported");
7873 14 : SDLoc dl(Op);
7874 14 : // Get the inputs.
7875 14 : SDNode *Node = Op.getNode();
7876 14 : SDValue Chain = Op.getOperand(0);
7877 : SDValue Size = Op.getOperand(1);
7878 42 : unsigned Align = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
7879 : EVT VT = Node->getValueType(0);
7880 :
7881 : if (DAG.getMachineFunction().getFunction().hasFnAttribute(
7882 : "no-stack-arg-probe")) {
7883 : SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
7884 : Chain = SP.getValue(1);
7885 : SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
7886 : if (Align)
7887 : SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
7888 149 : DAG.getConstant(-(uint64_t)Align, dl, VT));
7889 : Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
7890 149 : SDValue Ops[2] = {SP, Chain};
7891 : return DAG.getMergeValues(Ops, dl);
7892 149 : }
7893 149 :
7894 0 : Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
7895 :
7896 149 : Chain = LowerWindowsDYNAMIC_STACKALLOC(Op, Chain, Size, DAG);
7897 :
7898 33 : SDValue SP = DAG.getCopyFromReg(Chain, dl, AArch64::SP, MVT::i64);
7899 33 : Chain = SP.getValue(1);
7900 0 : SP = DAG.getNode(ISD::SUB, dl, MVT::i64, SP, Size);
7901 : if (Align)
7902 33 : SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
7903 : DAG.getConstant(-(uint64_t)Align, dl, VT));
7904 : Chain = DAG.getCopyToReg(Chain, dl, AArch64::SP, SP);
7905 149 :
7906 : Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
7907 149 : DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
7908 48 :
7909 : SDValue Ops[2] = {SP, Chain};
7910 101 : return DAG.getMergeValues(Ops, dl);
7911 : }
7912 :
7913 0 : /// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
7914 : /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
7915 0 : /// specified in the intrinsic calls.
7916 0 : bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
7917 : const CallInst &I,
7918 0 : MachineFunction &MF,
7919 : unsigned Intrinsic) const {
7920 : auto &DL = I.getModule()->getDataLayout();
7921 58 : switch (Intrinsic) {
7922 : case Intrinsic::aarch64_neon_ld2:
7923 : case Intrinsic::aarch64_neon_ld3:
7924 58 : case Intrinsic::aarch64_neon_ld4:
7925 : case Intrinsic::aarch64_neon_ld1x2:
7926 28 : case Intrinsic::aarch64_neon_ld1x3:
7927 : case Intrinsic::aarch64_neon_ld1x4:
7928 20 : case Intrinsic::aarch64_neon_ld2lane:
7929 : case Intrinsic::aarch64_neon_ld3lane:
7930 20 : case Intrinsic::aarch64_neon_ld4lane:
7931 : case Intrinsic::aarch64_neon_ld2r:
7932 20 : case Intrinsic::aarch64_neon_ld3r:
7933 : case Intrinsic::aarch64_neon_ld4r: {
7934 20 : Info.opc = ISD::INTRINSIC_W_CHAIN;
7935 : // Conservatively set memVT to the entire set of vectors loaded.
7936 : uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
7937 : Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
7938 : Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
7939 : Info.offset = 0;
7940 2 : Info.align = 0;
7941 : // volatile loads with NEON intrinsics not supported
7942 : Info.flags = MachineMemOperand::MOLoad;
7943 : return true;
7944 : }
7945 : case Intrinsic::aarch64_neon_st2:
7946 : case Intrinsic::aarch64_neon_st3:
7947 2 : case Intrinsic::aarch64_neon_st4:
7948 : case Intrinsic::aarch64_neon_st1x2:
7949 0 : case Intrinsic::aarch64_neon_st1x3:
7950 0 : case Intrinsic::aarch64_neon_st1x4:
7951 : case Intrinsic::aarch64_neon_st2lane:
7952 : case Intrinsic::aarch64_neon_st3lane:
7953 : case Intrinsic::aarch64_neon_st4lane: {
7954 176 : Info.opc = ISD::INTRINSIC_VOID;
7955 : // Conservatively set memVT to the entire set of vectors stored.
7956 176 : unsigned NumElts = 0;
7957 176 : for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
7958 0 : Type *ArgTy = I.getArgOperand(ArgI)->getType();
7959 : if (!ArgTy->isVectorTy())
7960 : break;
7961 : NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
7962 : }
7963 176 : Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
7964 : Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
7965 176 : Info.offset = 0;
7966 : Info.align = 0;
7967 : // volatile stores with NEON intrinsics not supported
7968 352 : Info.flags = MachineMemOperand::MOStore;
7969 : return true;
7970 : }
7971 176 : case Intrinsic::aarch64_ldaxr:
7972 : case Intrinsic::aarch64_ldxr: {
7973 176 : PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
7974 176 : Info.opc = ISD::INTRINSIC_W_CHAIN;
7975 0 : Info.memVT = MVT::getVT(PtrTy->getElementType());
7976 : Info.ptrVal = I.getArgOperand(0);
7977 : Info.offset = 0;
7978 : Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
7979 : Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
7980 176 : return true;
7981 : }
7982 176 : case Intrinsic::aarch64_stlxr:
7983 : case Intrinsic::aarch64_stxr: {
7984 : PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
7985 352 : Info.opc = ISD::INTRINSIC_W_CHAIN;
7986 : Info.memVT = MVT::getVT(PtrTy->getElementType());
7987 : Info.ptrVal = I.getArgOperand(1);
7988 3 : Info.offset = 0;
7989 : Info.align = DL.getABITypeAlignment(PtrTy->getElementType());
7990 : Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
7991 3 : return true;
7992 3 : }
7993 : case Intrinsic::aarch64_ldaxp:
7994 3 : case Intrinsic::aarch64_ldxp:
7995 3 : Info.opc = ISD::INTRINSIC_W_CHAIN;
7996 6 : Info.memVT = MVT::i128;
7997 0 : Info.ptrVal = I.getArgOperand(0);
7998 : Info.offset = 0;
7999 3 : Info.align = 16;
8000 3 : Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
8001 3 : return true;
8002 3 : case Intrinsic::aarch64_stlxp:
8003 3 : case Intrinsic::aarch64_stxp:
8004 : Info.opc = ISD::INTRINSIC_W_CHAIN;
8005 3 : Info.memVT = MVT::i128;
8006 : Info.ptrVal = I.getArgOperand(2);
8007 : Info.offset = 0;
8008 : Info.align = 16;
8009 : Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
8010 : return true;
8011 3 : default:
8012 3 : break;
8013 3 : }
8014 :
8015 : return false;
8016 : }
8017 4 :
8018 : bool AArch64TargetLowering::shouldReduceLoadWidth(SDNode *Load,
8019 : ISD::LoadExtType ExtTy,
8020 : EVT NewVT) const {
8021 : // If we're reducing the load width in order to avoid having to use an extra
8022 : // instruction to do extension then it's probably a good idea.
8023 : if (ExtTy != ISD::NON_EXTLOAD)
8024 4 : return true;
8025 4 : // Don't reduce load width if it would prevent us from combining a shift into
8026 4 : // the offset.
8027 4 : MemSDNode *Mem = dyn_cast<MemSDNode>(Load);
8028 : assert(Mem);
8029 4 : const SDValue &Base = Mem->getBasePtr();
8030 : if (Base.getOpcode() == ISD::ADD &&
8031 1 : Base.getOperand(1).getOpcode() == ISD::SHL &&
8032 1 : Base.getOperand(1).hasOneUse() &&
8033 1 : Base.getOperand(1).getOperand(1).getOpcode() == ISD::Constant) {
8034 1 : // The shift can be combined if it matches the size of the value being
8035 0 : // loaded (and so reducing the width would make it not match).
8036 0 : uint64_t ShiftAmount = Base.getOperand(1).getConstantOperandVal(1);
8037 1 : uint64_t LoadBytes = Mem->getMemoryVT().getSizeInBits()/8;
8038 1 : if (ShiftAmount == Log2_32(LoadBytes))
8039 1 : return false;
8040 : }
8041 : // We have no reason to disallow reducing the load width, so allow it.
8042 3 : return true;
8043 : }
8044 3 :
8045 : // Truncations from 64-bit GPR to 32-bit GPR is free.
8046 3 : bool AArch64TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
8047 3 : if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
8048 3 : return false;
8049 3 : unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
8050 0 : unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
8051 0 : return NumBits1 > NumBits2;
8052 3 : }
8053 : bool AArch64TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
8054 3 : if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
8055 3 : return false;
8056 : unsigned NumBits1 = VT1.getSizeInBits();
8057 3 : unsigned NumBits2 = VT2.getSizeInBits();
8058 3 : return NumBits1 > NumBits2;
8059 : }
8060 :
8061 : /// Check if it is profitable to hoist instruction in then/else to if.
8062 : /// Not profitable if I and it's user can form a FMA instruction
8063 : /// because we prefer FMSUB/FMADD.
8064 4319 : bool AArch64TargetLowering::isProfitableToHoist(Instruction *I) const {
8065 : if (I->getOpcode() != Instruction::FMul)
8066 : return true;
8067 :
8068 4319 : if (!I->hasOneUse())
8069 4319 : return true;
8070 434 :
8071 : Instruction *User = I->user_back();
8072 :
8073 : if (User &&
8074 : !(User->getOpcode() == Instruction::FSub ||
8075 : User->getOpcode() == Instruction::FAdd))
8076 : return true;
8077 :
8078 : const TargetOptions &Options = getTargetMachine().Options;
8079 : const DataLayout &DL = I->getModule()->getDataLayout();
8080 : EVT VT = getValueType(DL, User->getOperand(0)->getType());
8081 :
8082 434 : return !(isFMAFasterThanFMulAndFAdd(VT) &&
8083 : isOperationLegalOrCustom(ISD::FMA, VT) &&
8084 434 : (Options.AllowFPOpFusion == FPOpFusion::Fast ||
8085 868 : Options.UnsafeFPMath));
8086 434 : }
8087 434 :
8088 434 : // All 32-bit GPR operations implicitly zero the high-half of the corresponding
8089 : // 64-bit GPR.
8090 434 : bool AArch64TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
8091 434 : if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
8092 : return false;
8093 401 : unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
8094 : unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
8095 : return NumBits1 == 32 && NumBits2 == 64;
8096 : }
8097 : bool AArch64TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
8098 : if (VT1.isVector() || VT2.isVector() || !VT1.isInteger() || !VT2.isInteger())
8099 : return false;
8100 : unsigned NumBits1 = VT1.getSizeInBits();
8101 : unsigned NumBits2 = VT2.getSizeInBits();
8102 401 : return NumBits1 == 32 && NumBits2 == 64;
8103 : }
8104 :
8105 1592 : bool AArch64TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
8106 1191 : EVT VT1 = Val.getValueType();
8107 1191 : if (isZExtFree(VT1, VT2)) {
8108 : return true;
8109 790 : }
8110 :
8111 401 : if (Val.getOpcode() != ISD::LOAD)
8112 401 : return false;
8113 401 :
8114 401 : // 8-, 16-, and 32-bit integer loads all implicitly zero-extend.
8115 : return (VT1.isSimple() && !VT1.isVector() && VT1.isInteger() &&
8116 401 : VT2.isSimple() && !VT2.isVector() && VT2.isInteger() &&
8117 401 : VT1.getSizeInBits() <= 32);
8118 : }
8119 112 :
8120 : bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
8121 112 : if (isa<FPExtInst>(Ext))
8122 112 : return false;
8123 112 :
8124 : // Vector types are not free.
8125 112 : if (Ext->getType()->isVectorTy())
8126 112 : return false;
8127 112 :
8128 112 : for (const Use &U : Ext->uses()) {
8129 : // The extension is free if we can fold it with a left shift in an
8130 111 : // addressing mode or an arithmetic operation: add, sub, and cmp.
8131 :
8132 111 : // Is there a shift?
8133 111 : const Instruction *Instr = cast<Instruction>(U.getUser());
8134 111 :
8135 : // Is this a constant shift?
8136 111 : switch (Instr->getOpcode()) {
8137 111 : case Instruction::Shl:
8138 111 : if (!isa<ConstantInt>(Instr->getOperand(1)))
8139 111 : return false;
8140 : break;
8141 16 : case Instruction::GetElementPtr: {
8142 : gep_type_iterator GTI = gep_type_begin(Instr);
8143 16 : auto &DL = Ext->getModule()->getDataLayout();
8144 16 : std::advance(GTI, U.getOperandNo()-1);
8145 16 : Type *IdxTy = GTI.getIndexedType();
8146 16 : // This extension will end up with a shift because of the scaling factor.
8147 16 : // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0.
8148 16 : // Get the shift amount based on the scaling factor:
8149 16 : // log2(sizeof(IdxTy)) - log2(8).
8150 16 : uint64_t ShiftAmt =
8151 : countTrailingZeros(DL.getTypeStoreSizeInBits(IdxTy)) - 3;
8152 16 : // Is the constant foldable in the shift of the addressing mode?
8153 16 : // I.e., shift amount is between 1 and 4 inclusive.
8154 16 : if (ShiftAmt == 0 || ShiftAmt > 4)
8155 16 : return false;
8156 16 : break;
8157 16 : }
8158 16 : case Instruction::Trunc:
8159 : // Check if this is a noop.
8160 : // trunc(sext ty1 to ty2) to ty1.
8161 : if (Instr->getType() == Ext->getOperand(0)->getType())
8162 : continue;
8163 : LLVM_FALLTHROUGH;
8164 : default:
8165 : return false;
8166 88 : }
8167 :
8168 : // At this point we can use the bfm family, so this extension is free
8169 : // for that use.
8170 : }
8171 88 : return true;
8172 : }
8173 :
8174 : bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType,
8175 : unsigned &RequiredAligment) const {
8176 : if (!LoadedType.isSimple() ||
8177 : (!LoadedType.isInteger() && !LoadedType.isFloatingPoint()))
8178 32 : return false;
8179 56 : // Cyclone supports unaligned accesses.
8180 82 : RequiredAligment = 0;
8181 66 : unsigned NumBits = LoadedType.getSizeInBits();
8182 : return NumBits == 32 || NumBits == 64;
8183 : }
8184 :
8185 22 : /// A helper function for determining the number of interleaved accesses we
8186 22 : /// will generate when lowering accesses of the given type.
8187 22 : unsigned
8188 : AArch64TargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
8189 : const DataLayout &DL) const {
8190 : return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
8191 : }
8192 :
8193 : MachineMemOperand::Flags
8194 823 : AArch64TargetLowering::getMMOFlags(const Instruction &I) const {
8195 823 : if (Subtarget->getProcFamily() == AArch64Subtarget::Falkor &&
8196 : I.getMetadata(FALKOR_STRIDED_ACCESS_MD) != nullptr)
8197 702 : return MOStridedAccess;
8198 702 : return MachineMemOperand::MONone;
8199 702 : }
8200 :
8201 1047 : bool AArch64TargetLowering::isLegalInterleavedAccessType(
8202 2092 : VectorType *VecTy, const DataLayout &DL) const {
8203 :
8204 1045 : unsigned VecSize = DL.getTypeSizeInBits(VecTy);
8205 1045 : unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
8206 1045 :
8207 : // Ensure the number of vector elements is greater than 1.
8208 : if (VecTy->getNumElements() < 2)
8209 : return false;
8210 :
8211 : // Ensure the element type is legal.
8212 15 : if (ElSize != 8 && ElSize != 16 && ElSize != 32 && ElSize != 64)
8213 15 : return false;
8214 :
8215 : // Ensure the total vector size is 64 or a multiple of 128. Types larger than
8216 3 : // 128 will be split into multiple interleaved accesses.
8217 : return VecSize == 64 || VecSize % 128 == 0;
8218 : }
8219 :
8220 : /// Lower an interleaved load into a ldN intrinsic.
8221 3 : ///
8222 3 : /// E.g. Lower an interleaved load (Factor = 2):
8223 : /// %wide.vec = load <8 x i32>, <8 x i32>* %ptr
8224 : /// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
8225 : /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
8226 3 : ///
8227 3 : /// Into:
8228 6 : /// %ld2 = { <4 x i32>, <4 x i32> } call llvm.aarch64.neon.ld2(%ptr)
8229 : /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
8230 3 : /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
8231 : bool AArch64TargetLowering::lowerInterleavedLoad(
8232 3 : LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
8233 3 : ArrayRef<unsigned> Indices, unsigned Factor) const {
8234 : assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
8235 : "Invalid interleave factor");
8236 : assert(!Shuffles.empty() && "Empty shufflevector input");
8237 : assert(Shuffles.size() == Indices.size() &&
8238 390 : "Unmatched number of shufflevectors and indices");
8239 390 :
8240 : const DataLayout &DL = LI->getModule()->getDataLayout();
8241 390 :
8242 390 : VectorType *VecTy = Shuffles[0]->getType();
8243 390 :
8244 : // Skip if we do not have NEON and skip illegal vector types. We can
8245 3079 : // "legalize" wide vector types into multiple interleaved accesses as long as
8246 5603 : // the vector types are divisible by 128.
8247 : if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(VecTy, DL))
8248 2366 : return false;
8249 2366 :
8250 2366 : unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
8251 :
8252 : // A pointer vector can not be the return type of the ldN intrinsics. Need to
8253 2020 : // load integer vectors first and then convert to pointer vectors.
8254 2020 : Type *EltTy = VecTy->getVectorElementType();
8255 2020 : if (EltTy->isPointerTy())
8256 : VecTy =
8257 : VectorType::get(DL.getIntPtrType(EltTy), VecTy->getVectorNumElements());
8258 :
8259 2020 : IRBuilder<> Builder(LI);
8260 :
8261 : // The base address of the load.
8262 : Value *BaseAddr = LI->getPointerOperand();
8263 353 :
8264 602 : if (NumLoads > 1) {
8265 249 : // If we're going to generate more than one load, reset the sub-vector type
8266 : // to something legal.
8267 : VecTy = VectorType::get(VecTy->getVectorElementType(),
8268 471 : VecTy->getVectorNumElements() / NumLoads);
8269 471 :
8270 : // We will compute the pointer operand of each load from the original base
8271 : // address using GEPs. Cast the base address to a pointer to the scalar
8272 : // element type.
8273 942 : BaseAddr = Builder.CreateBitCast(
8274 : BaseAddr, VecTy->getVectorElementType()->getPointerTo(
8275 : LI->getPointerAddressSpace()));
8276 586 : }
8277 :
8278 : Type *PtrTy = VecTy->getPointerTo(LI->getPointerAddressSpace());
8279 : Type *Tys[2] = {VecTy, PtrTy};
8280 : static const Intrinsic::ID LoadInts[3] = {Intrinsic::aarch64_neon_ld2,
8281 484 : Intrinsic::aarch64_neon_ld3,
8282 : Intrinsic::aarch64_neon_ld4};
8283 : Function *LdNFunc =
8284 484 : Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
8285 28 :
8286 56 : // Holds sub-vectors extracted from the load intrinsic return values. The
8287 : // sub-vectors are associated with the shufflevector instructions they will
8288 : // replace.
8289 115 : DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
8290 115 :
8291 115 : for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
8292 115 :
8293 115 : // If we're generating more than one load, compute the base address of
8294 : // subsequent loads as an offset from the previous.
8295 : if (LoadCount > 0)
8296 : BaseAddr = Builder.CreateConstGEP1_32(
8297 : BaseAddr, VecTy->getVectorNumElements() * Factor);
8298 :
8299 : CallInst *LdN = Builder.CreateCall(
8300 : LdNFunc, Builder.CreateBitCast(BaseAddr, PtrTy), "ldN");
8301 :
8302 115 : // Extract and store the sub-vectors returned by the load intrinsic.
8303 28 : for (unsigned i = 0; i < Shuffles.size(); i++) {
8304 87 : ShuffleVectorInst *SVI = Shuffles[i];
8305 : unsigned Index = Indices[i];
8306 0 :
8307 : Value *SubVec = Builder.CreateExtractValue(LdN, Index);
8308 :
8309 0 : // Convert the integer vector to pointer vector if the element is pointer.
8310 : if (EltTy->isPointerTy())
8311 : SubVec = Builder.CreateIntToPtr(
8312 : SubVec, VectorType::get(SVI->getType()->getVectorElementType(),
8313 : VecTy->getVectorNumElements()));
8314 : SubVecs[SVI].push_back(SubVec);
8315 : }
8316 : }
8317 :
8318 : // Replace uses of the shufflevector instructions with the sub-vectors
8319 : // returned by the load intrinsic. If a shufflevector instruction is
8320 : // associated with more than one sub-vector, those sub-vectors will be
8321 : // concatenated into a single wide vector.
8322 622 : for (ShuffleVectorInst *SVI : Shuffles) {
8323 : auto &SubVec = SubVecs[SVI];
8324 622 : auto *WideVec =
8325 622 : SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
8326 : SVI->replaceAllUsesWith(WideVec);
8327 : }
8328 622 :
8329 622 : return true;
8330 622 : }
8331 :
8332 : /// Lower an interleaved store into a stN intrinsic.
8333 : ///
8334 : /// E.g. Lower an interleaved store (Factor = 3):
8335 : /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
8336 46 : /// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
8337 : /// store <12 x i32> %i.vec, <12 x i32>* %ptr
8338 46 : ///
8339 : /// Into:
8340 : /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
8341 : /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
8342 9978 : /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
8343 9978 : /// call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
8344 16 : ///
8345 8 : /// Note that the new shufflevectors will be removed and we'll only generate one
8346 : /// st3 instruction in CodeGen.
8347 : ///
8348 : /// Example for a more general valid mask (Factor 3). Lower:
8349 57 : /// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
8350 : /// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
8351 : /// store <12 x i32> %i.vec, <12 x i32>* %ptr
8352 57 : ///
8353 57 : /// Into:
8354 : /// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
8355 : /// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
8356 57 : /// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
8357 : /// call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr)
8358 : bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI,
8359 : ShuffleVectorInst *SVI,
8360 49 : unsigned Factor) const {
8361 : assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&
8362 : "Invalid interleave factor");
8363 :
8364 : VectorType *VecTy = SVI->getType();
8365 48 : assert(VecTy->getVectorNumElements() % Factor == 0 &&
8366 : "Invalid interleaved store");
8367 :
8368 : unsigned LaneLen = VecTy->getVectorNumElements() / Factor;
8369 : Type *EltTy = VecTy->getVectorElementType();
8370 : VectorType *SubVecTy = VectorType::get(EltTy, LaneLen);
8371 :
8372 : const DataLayout &DL = SI->getModule()->getDataLayout();
8373 :
8374 : // Skip if we do not have NEON and skip illegal vector types. We can
8375 : // "legalize" wide vector types into multiple interleaved accesses as long as
8376 : // the vector types are divisible by 128.
8377 : if (!Subtarget->hasNEON() || !isLegalInterleavedAccessType(SubVecTy, DL))
8378 : return false;
8379 35 :
8380 : unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
8381 :
8382 : Value *Op0 = SVI->getOperand(0);
8383 : Value *Op1 = SVI->getOperand(1);
8384 : IRBuilder<> Builder(SI);
8385 :
8386 : // StN intrinsics don't support pointer vectors as arguments. Convert pointer
8387 : // vectors to integer vectors.
8388 35 : if (EltTy->isPointerTy()) {
8389 : Type *IntTy = DL.getIntPtrType(EltTy);
8390 35 : unsigned NumOpElts = Op0->getType()->getVectorNumElements();
8391 :
8392 : // Convert to the corresponding integer vector.
8393 : Type *IntVecTy = VectorType::get(IntTy, NumOpElts);
8394 : Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
8395 35 : Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
8396 19 :
8397 : SubVecTy = VectorType::get(IntTy, LaneLen);
8398 16 : }
8399 :
8400 : // The base address of the store.
8401 : Value *BaseAddr = SI->getPointerOperand();
8402 16 :
8403 16 : if (NumStores > 1) {
8404 : // If we're going to generate more than one store, reset the lane length
8405 4 : // and sub-vector type to something legal.
8406 : LaneLen /= NumStores;
8407 16 : SubVecTy = VectorType::get(SubVecTy->getVectorElementType(), LaneLen);
8408 :
8409 : // We will compute the pointer operand of each store from the original base
8410 : // address using GEPs. Cast the base address to a pointer to the scalar
8411 : // element type.
8412 16 : BaseAddr = Builder.CreateBitCast(
8413 : BaseAddr, SubVecTy->getVectorElementType()->getPointerTo(
8414 : SI->getPointerAddressSpace()));
8415 10 : }
8416 :
8417 : auto Mask = SVI->getShuffleMask();
8418 :
8419 : Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace());
8420 : Type *Tys[2] = {SubVecTy, PtrTy};
8421 5 : static const Intrinsic::ID StoreInts[3] = {Intrinsic::aarch64_neon_st2,
8422 10 : Intrinsic::aarch64_neon_st3,
8423 : Intrinsic::aarch64_neon_st4};
8424 : Function *StNFunc =
8425 : Intrinsic::getDeclaration(SI->getModule(), StoreInts[Factor - 2], Tys);
8426 32 :
8427 16 : for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
8428 :
8429 : SmallVector<Value *, 5> Ops;
8430 :
8431 : // Split the shufflevector operands into sub vectors for the new stN call.
8432 16 : for (unsigned i = 0; i < Factor; i++) {
8433 : unsigned IdxI = StoreCount * LaneLen * Factor + i;
8434 : if (Mask[IdxI] >= 0) {
8435 : Ops.push_back(Builder.CreateShuffleVector(
8436 : Op0, Op1, createSequentialMask(Builder, Mask[IdxI], LaneLen, 0)));
8437 : } else {
8438 : unsigned StartMask = 0;
8439 38 : for (unsigned j = 1; j < LaneLen; j++) {
8440 : unsigned IdxJ = StoreCount * LaneLen * Factor + j;
8441 : if (Mask[IdxJ * Factor + IdxI] >= 0) {
8442 : StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
8443 22 : break;
8444 6 : }
8445 : }
8446 : // Note: Filling undef gaps with random elements is ok, since
8447 44 : // those elements were being written anyway (with undefs).
8448 22 : // In the case of all undefs we're defaulting to using elems from 0
8449 : // Note: StartMask cannot be negative, it's checked in
8450 : // isReInterleaveMask
8451 79 : Ops.push_back(Builder.CreateShuffleVector(
8452 57 : Op0, Op1, createSequentialMask(Builder, StartMask, LaneLen, 0)));
8453 114 : }
8454 : }
8455 57 :
8456 : // If we generating more than one store, we compute the base address of
8457 : // subsequent stores as an offset from the previous.
8458 57 : if (StoreCount > 0)
8459 13 : BaseAddr = Builder.CreateConstGEP1_32(BaseAddr, LaneLen * Factor);
8460 39 :
8461 : Ops.push_back(Builder.CreateBitCast(BaseAddr, PtrTy));
8462 57 : Builder.CreateCall(StNFunc, Ops);
8463 : }
8464 : return true;
8465 : }
8466 :
8467 : static bool memOpAlign(unsigned DstAlign, unsigned SrcAlign,
8468 : unsigned AlignCheck) {
8469 : return ((SrcAlign == 0 || SrcAlign % AlignCheck == 0) &&
8470 58 : (DstAlign == 0 || DstAlign % AlignCheck == 0));
8471 : }
8472 :
8473 84 : EVT AArch64TargetLowering::getOptimalMemOpType(uint64_t Size, unsigned DstAlign,
8474 42 : unsigned SrcAlign, bool IsMemset,
8475 : bool ZeroMemset,
8476 : bool MemcpyStrSrc,
8477 : MachineFunction &MF) const {
8478 : const Function &F = MF.getFunction();
8479 : bool CanImplicitFloat = !F.hasFnAttribute(Attribute::NoImplicitFloat);
8480 : bool CanUseNEON = Subtarget->hasNEON() && CanImplicitFloat;
8481 : bool CanUseFP = Subtarget->hasFPARMv8() && CanImplicitFloat;
8482 : // Only use AdvSIMD to implement memset of 32-byte and above. It would have
8483 : // taken one instruction to materialize the v2i64 zero and one store (with
8484 : // restrictive addressing mode). Just do i64 stores.
8485 : bool IsSmallMemset = IsMemset && Size < 32;
8486 : auto AlignmentIsAcceptable = [&](EVT VT, unsigned AlignCheck) {
8487 : if (memOpAlign(SrcAlign, DstAlign, AlignCheck))
8488 : return true;
8489 : bool Fast;
8490 : return allowsMisalignedMemoryAccesses(VT, 0, 1, &Fast) && Fast;
8491 : };
8492 :
8493 : if (CanUseNEON && IsMemset && !IsSmallMemset &&
8494 : AlignmentIsAcceptable(MVT::v2i64, 16))
8495 : return MVT::v2i64;
8496 : if (CanUseFP && !IsSmallMemset && AlignmentIsAcceptable(MVT::f128, 16))
8497 : return MVT::f128;
8498 : if (Size >= 8 && AlignmentIsAcceptable(MVT::i64, 8))
8499 : return MVT::i64;
8500 : if (Size >= 4 && AlignmentIsAcceptable(MVT::i32, 4))
8501 : return MVT::i32;
8502 : return MVT::Other;
8503 : }
8504 :
8505 : // 12-bit optionally shifted immediates are legal for adds.
8506 49 : bool AArch64TargetLowering::isLegalAddImmediate(int64_t Immed) const {
8507 : if (Immed == std::numeric_limits<int64_t>::min()) {
8508 : LLVM_DEBUG(dbgs() << "Illegal add imm " << Immed
8509 : << ": avoid UB for INT64_MIN\n");
8510 : return false;
8511 : }
8512 : // Same encoding for add/sub, just flip the sign.
8513 : Immed = std::abs(Immed);
8514 : bool IsLegal = ((Immed >> 12) == 0 ||
8515 : ((Immed & 0xfff) == 0 && Immed >> 24 == 0));
8516 49 : LLVM_DEBUG(dbgs() << "Is " << Immed
8517 49 : << " legal add imm: " << (IsLegal ? "yes" : "no") << "\n");
8518 49 : return IsLegal;
8519 : }
8520 49 :
8521 : // Integer comparisons are implemented with ADDS/SUBS, so the range of valid
8522 : // immediates is the same as for an add or a sub.
8523 : bool AArch64TargetLowering::isLegalICmpImmediate(int64_t Immed) const {
8524 : return isLegalAddImmediate(Immed);
8525 49 : }
8526 29 :
8527 : /// isLegalAddressingMode - Return true if the addressing mode represented
8528 20 : /// by AM is legal for this target, for a load/store of the specified type.
8529 : bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL,
8530 : const AddrMode &AM, Type *Ty,
8531 : unsigned AS, Instruction *I) const {
8532 20 : // AArch64 has five basic addressing modes:
8533 : // reg
8534 : // reg + 9-bit signed offset
8535 : // reg + SIZE_IN_BYTES * 12-bit unsigned offset
8536 20 : // reg1 + reg2
8537 3 : // reg + SIZE_IN_BYTES * reg
8538 3 :
8539 : // No global is ever allowed as a base.
8540 : if (AM.BaseGV)
8541 3 : return false;
8542 3 :
8543 3 : // No reg+reg+imm addressing.
8544 : if (AM.HasBaseReg && AM.BaseOffs && AM.Scale)
8545 3 : return false;
8546 :
8547 : // check reg + imm case:
8548 : // i.e., reg + 0, reg + imm9, reg + SIZE_IN_BYTES * uimm12
8549 : uint64_t NumBytes = 0;
8550 : if (Ty->isSized()) {
8551 20 : uint64_t NumBits = DL.getTypeSizeInBits(Ty);
8552 : NumBytes = NumBits / 8;
8553 : if (!isPowerOf2_64(NumBits))
8554 3 : NumBytes = 0;
8555 6 : }
8556 :
8557 : if (!AM.Scale) {
8558 : int64_t Offset = AM.BaseOffs;
8559 :
8560 3 : // 9-bit signed offset
8561 6 : if (isInt<9>(Offset))
8562 : return true;
8563 :
8564 : // 12-bit unsigned offset
8565 : unsigned shift = Log2_64(NumBytes);
8566 : if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
8567 40 : // Must be a multiple of NumBytes (NumBytes is a power of 2)
8568 20 : (Offset >> shift) << shift == Offset)
8569 : return true;
8570 : return false;
8571 : }
8572 :
8573 20 : // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
8574 :
8575 43 : return AM.Scale == 1 || (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes);
8576 : }
8577 :
8578 : bool AArch64TargetLowering::shouldConsiderGEPOffsetSplit() const {
8579 : // Consider splitting large offset of struct or array.
8580 97 : return true;
8581 74 : }
8582 148 :
8583 65 : int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL,
8584 65 : const AddrMode &AM, Type *Ty,
8585 : unsigned AS) const {
8586 : // Scaling factors are not free at all.
8587 16 : // Operands | Rt Latency
8588 13 : // -------------------------------------------
8589 26 : // Rt, [Xn, Xm] | 4
8590 6 : // -------------------------------------------
8591 6 : // Rt, [Xn, Xm, lsl #imm] | Rn: 4 Rm: 5
8592 : // Rt, [Xn, Wm, <extend> #imm] |
8593 : if (isLegalAddressingMode(DL, AM, Ty, AS))
8594 : // Scale represents reg2 * scale, thus account for 1 if
8595 : // it is not equal to 0 or 1.
8596 : return AM.Scale != 0 && AM.Scale != 1;
8597 : return -1;
8598 : }
8599 9 :
8600 9 : bool AArch64TargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const {
8601 : VT = VT.getScalarType();
8602 :
8603 : if (!VT.isSimple())
8604 : return false;
8605 :
8606 23 : switch (VT.getSimpleVT().SimpleTy) {
8607 3 : case MVT::f32:
8608 : case MVT::f64:
8609 23 : return true;
8610 23 : default:
8611 : break;
8612 : }
8613 :
8614 : return false;
8615 : }
8616 :
8617 : const MCPhysReg *
8618 : AArch64TargetLowering::getScratchRegisters(CallingConv::ID) const {
8619 : // LR is a callee-save register, but we must treat it as clobbered by any call
8620 : // site. Hence we include LR in the scratch registers, which are in turn added
8621 93 : // as implicit-defs for stackmaps and patchpoints.
8622 : static const MCPhysReg ScratchRegs[] = {
8623 : AArch64::X16, AArch64::X17, AArch64::LR, 0
8624 : };
8625 : return ScratchRegs;
8626 93 : }
8627 93 :
8628 93 : bool
8629 93 : AArch64TargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
8630 : CombineLevel Level) const {
8631 : N = N->getOperand(0).getNode();
8632 : EVT VT = N->getValueType(0);
8633 93 : // If N is unsigned bit extraction: ((x >> C) & mask), then do not combine
8634 : // it with shift to let it be lowered to UBFX.
8635 : if (N->getOpcode() == ISD::AND && (VT == MVT::i32 || VT == MVT::i64) &&
8636 : isa<ConstantSDNode>(N->getOperand(1))) {
8637 : uint64_t TruncMask = N->getConstantOperandVal(1);
8638 : if (isMask_64(TruncMask) &&
8639 93 : N->getOperand(0).getOpcode() == ISD::SRL &&
8640 : isa<ConstantSDNode>(N->getOperand(0)->getOperand(1)))
8641 123 : return false;
8642 30 : }
8643 30 : return true;
8644 63 : }
8645 38 :
8646 25 : bool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
8647 20 : Type *Ty) const {
8648 5 : assert(Ty->isIntegerTy());
8649 3 :
8650 2 : unsigned BitSize = Ty->getPrimitiveSizeInBits();
8651 : if (BitSize == 0)
8652 : return false;
8653 :
8654 2614 : int64_t Val = Imm.getSExtValue();
8655 2614 : if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, BitSize))
8656 : return true;
8657 :
8658 : if ((int64_t)Val < 0)
8659 : Val = ~Val;
8660 : if (BitSize == 32)
8661 : Val &= (1LL << 32) - 1;
8662 2614 :
8663 111 : unsigned LZ = countLeadingZeros((uint64_t)Val);
8664 : unsigned Shift = (63 - LZ) / 16;
8665 : // MOVZ is free so return true for one or fewer MOVK.
8666 : return Shift < 3;
8667 : }
8668 :
8669 : bool AArch64TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
8670 : unsigned Index) const {
8671 2253 : if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
8672 2253 : return false;
8673 :
8674 : return (Index == 0 || Index == ResVT.getVectorNumElements());
8675 : }
8676 :
8677 34539 : /// Turn vector tests of the signbit in the form of:
8678 : /// xor (sra X, elt_size(X)-1), -1
8679 : /// into:
8680 : /// cmge X, X, #0
8681 : static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
8682 : const AArch64Subtarget *Subtarget) {
8683 : EVT VT = N->getValueType(0);
8684 : if (!Subtarget->hasNEON() || !VT.isVector())
8685 : return SDValue();
8686 :
8687 : // There must be a shift right algebraic before the xor, and the xor must be a
8688 34539 : // 'not' operation.
8689 : SDValue Shift = N->getOperand(0);
8690 : SDValue Ones = N->getOperand(1);
8691 : if (Shift.getOpcode() != AArch64ISD::VASHR || !Shift.hasOneUse() ||
8692 30752 : !ISD::isBuildVectorAllOnes(Ones.getNode()))
8693 : return SDValue();
8694 :
8695 : // The shift should be smearing the sign bit across each vector element.
8696 : auto *ShiftAmt = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
8697 : EVT ShiftEltTy = Shift.getValueType().getVectorElementType();
8698 28719 : if (!ShiftAmt || ShiftAmt->getZExtValue() != ShiftEltTy.getSizeInBits() - 1)
8699 28689 : return SDValue();
8700 28689 :
8701 : return DAG.getNode(AArch64ISD::CMGEz, SDLoc(N), VT, Shift.getOperand(0));
8702 : }
8703 :
8704 : // Generate SUBS and CSEL for integer abs.
8705 28719 : static SDValue performIntegerAbsCombine(SDNode *N, SelectionDAG &DAG) {
8706 21895 : EVT VT = N->getValueType(0);
8707 :
8708 : SDValue N0 = N->getOperand(0);
8709 21895 : SDValue N1 = N->getOperand(1);
8710 : SDLoc DL(N);
8711 :
8712 : // Check pattern of XOR(ADD(X,Y), Y) where Y is SRA(X, size(X)-1)
8713 : // and change it to SUB and CSEL.
8714 459 : if (VT.isInteger() && N->getOpcode() == ISD::XOR &&
8715 : N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1 &&
8716 357 : N1.getOpcode() == ISD::SRA && N1.getOperand(0) == N0.getOperand(0))
8717 : if (ConstantSDNode *Y1C = dyn_cast<ConstantSDNode>(N1.getOperand(1)))
8718 110 : if (Y1C->getAPIntValue() == VT.getSizeInBits() - 1) {
8719 : SDValue Neg = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
8720 : N0.getOperand(0));
8721 : // Generate SUBS & CSEL.
8722 : SDValue Cmp =
8723 6824 : DAG.getNode(AArch64ISD::SUBS, DL, DAG.getVTList(VT, MVT::i32),
8724 : N0.getOperand(0), DAG.getConstant(0, DL, VT));
8725 : return DAG.getNode(AArch64ISD::CSEL, DL, VT, N0.getOperand(0), Neg,
8726 56 : DAG.getConstant(AArch64CC::PL, DL, MVT::i32),
8727 : SDValue(Cmp.getNode(), 1));
8728 56 : }
8729 : return SDValue();
8730 : }
8731 1372 :
8732 : static SDValue performXorCombine(SDNode *N, SelectionDAG &DAG,
8733 : TargetLowering::DAGCombinerInfo &DCI,
8734 : const AArch64Subtarget *Subtarget) {
8735 : if (DCI.isBeforeLegalizeOps())
8736 : return SDValue();
8737 :
8738 : if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
8739 : return Cmp;
8740 :
8741 1372 : return performIntegerAbsCombine(N, DAG);
8742 : }
8743 :
8744 1372 : SDValue
8745 : AArch64TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
8746 : SelectionDAG &DAG,
8747 : SmallVectorImpl<SDNode *> &Created) const {
8748 1868 : AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
8749 1868 : if (isIntDivCheap(N->getValueType(0), Attr))
8750 : return SDValue(N,0); // Lower SDIV as SDIV
8751 1868 :
8752 : // fold (sdiv X, pow2)
8753 : EVT VT = N->getValueType(0);
8754 1868 : if ((VT != MVT::i32 && VT != MVT::i64) ||
8755 : !(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()))
8756 : return SDValue();
8757 :
8758 : SDLoc DL(N);
8759 : SDValue N0 = N->getOperand(0);
8760 : unsigned Lg2 = Divisor.countTrailingZeros();
8761 : SDValue Zero = DAG.getConstant(0, DL, VT);
8762 195 : SDValue Pow2MinusOne = DAG.getConstant((1ULL << Lg2) - 1, DL, VT);
8763 :
8764 : // Add (N0 < 0) ? Pow2 - 1 : 0;
8765 : SDValue CCVal;
8766 64 : SDValue Cmp = getAArch64Cmp(N0, Zero, ISD::SETLT, CCVal, DAG, DL);
8767 : SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
8768 : SDValue CSel = DAG.getNode(AArch64ISD::CSEL, DL, VT, Add, N0, CCVal, Cmp);
8769 :
8770 : Created.push_back(Cmp.getNode());
8771 : Created.push_back(Add.getNode());
8772 : Created.push_back(CSel.getNode());
8773 64 :
8774 : // Divide by pow2.
8775 : SDValue SRA =
8776 : DAG.getNode(ISD::SRA, DL, VT, CSel, DAG.getConstant(Lg2, DL, MVT::i64));
8777 60 :
8778 : // If we're dividing by a positive value, we're done. Otherwise, we must
8779 60 : // negate the result.
8780 60 : if (Divisor.isNonNegative())
8781 : return SRA;
8782 :
8783 60 : Created.push_back(SRA.getNode());
8784 12 : return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), SRA);
8785 : }
8786 12 :
8787 12 : static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG,
8788 12 : TargetLowering::DAGCombinerInfo &DCI,
8789 12 : const AArch64Subtarget *Subtarget) {
8790 : if (DCI.isBeforeLegalizeOps())
8791 : return SDValue();
8792 :
8793 : // The below optimizations require a constant RHS.
8794 8 : if (!isa<ConstantSDNode>(N->getOperand(1)))
8795 : return SDValue();
8796 :
8797 : ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(1));
8798 8 : const APInt &ConstValue = C->getAPIntValue();
8799 8 :
8800 : // Multiplication of a power of two plus/minus one can be done more
8801 : // cheaply as as shift+add/sub. For now, this is true unilaterally. If
8802 : // future CPUs have a cheaper MADD instruction, this may need to be
8803 8 : // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and
8804 2 : // 64-bit is 5 cycles, so this is always a win.
8805 : // More aggressively, some multiplications N0 * C can be lowered to
8806 6 : // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M,
8807 0 : // e.g. 6=3*2=(2+1)*2.
8808 6 : // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45
8809 2 : // which equals to (1+2)*16-(1+2).
8810 : SDValue N0 = N->getOperand(0);
8811 6 : // TrailingZeroes is used to test if the mul can be lowered to
8812 6 : // shift+add+shift.
8813 : unsigned TrailingZeroes = ConstValue.countTrailingZeros();
8814 6 : if (TrailingZeroes) {
8815 : // Conservatively do not lower to shift+add+shift if the mul might be
8816 : // folded into smul or umul.
8817 14 : if (N0->hasOneUse() && (isSignExtended(N0.getNode(), DAG) ||
8818 : isZeroExtended(N0.getNode(), DAG)))
8819 : return SDValue();
8820 2 : // Conservatively do not lower to shift+add+shift if the mul might be
8821 : // folded into madd or msub.
8822 24 : if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ADD ||
8823 : N->use_begin()->getOpcode() == ISD::SUB))
8824 : return SDValue();
8825 : }
8826 : // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub
8827 : // and shift+add+shift.
8828 : APInt ShiftedConstValue = ConstValue.ashr(TrailingZeroes);
8829 1048 :
8830 : unsigned ShiftAmt, AddSubOpc;
8831 1048 : // Is the shifted value the LHS operand of the add/sub?
8832 2093 : bool ShiftValUseIsN0 = true;
8833 439 : // Do we need to negate the result?
8834 : bool NegateResult = false;
8835 :
8836 : if (ConstValue.isNonNegative()) {
8837 609 : // (mul x, 2^N + 1) => (add (shl x, N), x)
8838 609 : // (mul x, 2^N - 1) => (sub (shl x, N), x)
8839 627 : // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M)
8840 7 : APInt SCVMinus1 = ShiftedConstValue - 1;
8841 602 : APInt CVPlus1 = ConstValue + 1;
8842 : if (SCVMinus1.isPowerOf2()) {
8843 : ShiftAmt = SCVMinus1.logBase2();
8844 : AddSubOpc = ISD::ADD;
8845 7 : } else if (CVPlus1.isPowerOf2()) {
8846 14 : ShiftAmt = CVPlus1.logBase2();
8847 0 : AddSubOpc = ISD::SUB;
8848 : } else
8849 14 : return SDValue();
8850 : } else {
8851 : // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
8852 : // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
8853 1041 : APInt CVNegPlus1 = -ConstValue + 1;
8854 1041 : APInt CVNegMinus1 = -ConstValue - 1;
8855 : if (CVNegPlus1.isPowerOf2()) {
8856 1041 : ShiftAmt = CVNegPlus1.logBase2();
8857 1041 : AddSubOpc = ISD::SUB;
8858 : ShiftValUseIsN0 = false;
8859 : } else if (CVNegMinus1.isPowerOf2()) {
8860 : ShiftAmt = CVNegMinus1.logBase2();
8861 : AddSubOpc = ISD::ADD;
8862 1041 : NegateResult = true;
8863 22 : } else
8864 1048 : return SDValue();
8865 : }
8866 10 :
8867 : SDLoc DL(N);
8868 5 : EVT VT = N->getValueType(0);
8869 : SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0,
8870 : DAG.getConstant(ShiftAmt, DL, MVT::i64));
8871 :
8872 5 : SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0;
8873 : SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal;
8874 : SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1);
8875 10 : assert(!(NegateResult && TrailingZeroes) &&
8876 : "NegateResult and TrailingZeroes cannot both be true for now.");
8877 1036 : // Negate the result.
8878 : if (NegateResult)
8879 : return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res);
8880 1896 : // Shift the result.
8881 : if (TrailingZeroes)
8882 : return DAG.getNode(ISD::SHL, DL, VT, Res,
8883 3792 : DAG.getConstant(TrailingZeroes, DL, MVT::i64));
8884 848 : return Res;
8885 : }
8886 1048 :
8887 7 : static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N,
8888 : SelectionDAG &DAG) {
8889 1041 : // Take advantage of vector comparisons producing 0 or -1 in each lane to
8890 : // optimize away operation when it's from a constant.
8891 : //
8892 : // The general transformation is:
8893 16 : // UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
8894 : // AND(VECTOR_CMP(x,y), constant2)
8895 : // constant2 = UNARYOP(constant)
8896 16 :
8897 32 : // Early exit if this isn't a vector operation, the operand of the
8898 2 : // unary operation isn't a bitwise AND, or if the sizes of the operations
8899 : // aren't the same.
8900 : EVT VT = N->getValueType(0);
8901 28 : if (!VT.isVector() || N->getOperand(0)->getOpcode() != ISD::AND ||
8902 27 : N->getOperand(0)->getOperand(0)->getOpcode() != ISD::SETCC ||
8903 26 : VT.getSizeInBits() != N->getOperand(0)->getValueType(0).getSizeInBits())
8904 1 : return SDValue();
8905 :
8906 : // Now check that the other operand of the AND is a constant. We could
8907 13 : // make the transformation for non-constant splats as well, but it's unclear
8908 13 : // that would be a benefit as it would not eliminate any operations, just
8909 13 : // perform one more step in scalar code before moving to the vector unit.
8910 13 : if (BuildVectorSDNode *BV =
8911 : dyn_cast<BuildVectorSDNode>(N->getOperand(0)->getOperand(1))) {
8912 : // Bail out if the vector isn't a constant.
8913 13 : if (!BV->isConstant())
8914 13 : return SDValue();
8915 13 :
8916 13 : // Everything checks out. Build up the new and improved node.
8917 : SDLoc DL(N);
8918 13 : EVT IntVT = BV->getValueType(0);
8919 13 : // Create a new constant of the appropriate type for the transformed
8920 13 : // DAG.
8921 : SDValue SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
8922 : // The AND node needs bitcasts to/from an integer vector type around it.
8923 : SDValue MaskConst = DAG.getNode(ISD::BITCAST, DL, IntVT, SourceConst);
8924 13 : SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT,
8925 : N->getOperand(0)->getOperand(0), MaskConst);
8926 : SDValue Res = DAG.getNode(ISD::BITCAST, DL, VT, NewAnd);
8927 : return Res;
8928 13 : }
8929 9 :
8930 : return SDValue();
8931 4 : }
8932 4 :
8933 : static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG,
8934 : const AArch64Subtarget *Subtarget) {
8935 0 : // First try to optimize away the conversion when it's conditionally from
8936 : // a constant. Vectors only.
8937 : if (SDValue Res = performVectorCompareAndMaskUnaryOpCombine(N, DAG))
8938 0 : return Res;
8939 0 :
8940 : EVT VT = N->getValueType(0);
8941 : if (VT != MVT::f32 && VT != MVT::f64)
8942 0 : return SDValue();
8943 0 :
8944 : // Only optimize when the source and destination types have the same width.
8945 : if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())
8946 0 : return SDValue();
8947 :
8948 : // If the result of an integer load is only used by an integer-to-float
8949 : // conversion, use a fp load instead and a AdvSIMD scalar {S|U}CVTF instead.
8950 : // This eliminates an "integer-to-vector-move" UOP and improves throughput.
8951 : SDValue N0 = N->getOperand(0);
8952 : if (Subtarget->hasNEON() && ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse() &&
8953 : // Do not change the width of a volatile load.
8954 : !cast<LoadSDNode>(N0)->isVolatile()) {
8955 : LoadSDNode *LN0 = cast<LoadSDNode>(N0);
8956 : SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(),
8957 : LN0->getPointerInfo(), LN0->getAlignment(),
8958 0 : LN0->getMemOperand()->getFlags());
8959 :
8960 : // Make sure successors of the original load stay after it by updating them
8961 0 : // to use the new Chain.
8962 0 : DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
8963 :
8964 : unsigned Opcode =
8965 0 : (N->getOpcode() == ISD::SINT_TO_FP) ? AArch64ISD::SITOF : AArch64ISD::UITOF;
8966 : return DAG.getNode(Opcode, SDLoc(N), VT, Load);
8967 0 : }
8968 :
8969 : return SDValue();
8970 0 : }
8971 :
8972 0 : /// Fold a floating-point multiply by power of two into floating-point to
8973 : /// fixed-point conversion.
8974 : static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
8975 : TargetLowering::DAGCombinerInfo &DCI,
8976 0 : const AArch64Subtarget *Subtarget) {
8977 : if (!Subtarget->hasNEON())
8978 : return SDValue();
8979 :
8980 : SDValue Op = N->getOperand(0);
8981 : if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
8982 : Op.getOpcode() != ISD::FMUL)
8983 : return SDValue();
8984 0 :
8985 : SDValue ConstVec = Op->getOperand(1);
8986 : if (!isa<BuildVectorSDNode>(ConstVec))
8987 : return SDValue();
8988 0 :
8989 0 : MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
8990 0 : uint32_t FloatBits = FloatTy.getSizeInBits();
8991 : if (FloatBits != 32 && FloatBits != 64)
8992 : return SDValue();
8993 0 :
8994 : MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
8995 : uint32_t IntBits = IntTy.getSizeInBits();
8996 : if (IntBits != 16 && IntBits != 32 && IntBits != 64)
8997 0 : return SDValue();
8998 :
8999 : // Avoid conversions where iN is larger than the float (e.g., float -> i64).
9000 : if (IntBits > FloatBits)
9001 0 : return SDValue();
9002 0 :
9003 0 : BitVector UndefElements;
9004 : BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
9005 : int32_t Bits = IntBits == 64 ? 64 : 32;
9006 : int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, Bits + 1);
9007 0 : if (C == -1 || C == 0 || C > Bits)
9008 : return SDValue();
9009 :
9010 : MVT ResTy;
9011 : unsigned NumLanes = Op.getValueType().getVectorNumElements();
9012 0 : switch (NumLanes) {
9013 : default:
9014 : return SDValue();
9015 : case 2:
9016 0 : ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
9017 : break;
9018 0 : case 4:
9019 : ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
9020 0 : break;
9021 0 : }
9022 0 :
9023 : if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
9024 : return SDValue();
9025 :
9026 0 : assert((ResTy != MVT::v4i64 || DCI.isBeforeLegalizeOps()) &&
9027 0 : "Illegal vector type after legalization");
9028 :
9029 0 : SDLoc DL(N);
9030 : bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
9031 0 : unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
9032 0 : : Intrinsic::aarch64_neon_vcvtfp2fxu;
9033 : SDValue FixConv =
9034 : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
9035 759 : DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
9036 : Op->getOperand(0), DAG.getConstant(C, DL, MVT::i32));
9037 : // We can handle smaller integers by generating an extra trunc.
9038 : if (IntBits < FloatBits)
9039 : FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
9040 :
9041 : return FixConv;
9042 : }
9043 :
9044 : /// Fold a floating-point divide by power of two into fixed-point to
9045 : /// floating-point conversion.
9046 : static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
9047 : TargetLowering::DAGCombinerInfo &DCI,
9048 1518 : const AArch64Subtarget *Subtarget) {
9049 381 : if (!Subtarget->hasNEON())
9050 772 : return SDValue();
9051 6 :
9052 757 : SDValue Op = N->getOperand(0);
9053 : unsigned Opc = Op->getOpcode();
9054 : if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
9055 : !Op.getOperand(0).getValueType().isSimple() ||
9056 : (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP))
9057 : return SDValue();
9058 :
9059 : SDValue ConstVec = N->getOperand(1);
9060 : if (!isa<BuildVectorSDNode>(ConstVec))
9061 2 : return SDValue();
9062 0 :
9063 : MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
9064 : int32_t IntBits = IntTy.getSizeInBits();
9065 : if (IntBits != 16 && IntBits != 32 && IntBits != 64)
9066 4 : return SDValue();
9067 :
9068 : MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
9069 4 : int32_t FloatBits = FloatTy.getSizeInBits();
9070 : if (FloatBits != 32 && FloatBits != 64)
9071 2 : return SDValue();
9072 :
9073 4 : // Avoid conversions where iN is larger than the float (e.g., i64 -> float).
9074 2 : if (IntBits > FloatBits)
9075 2 : return SDValue();
9076 :
9077 : BitVector UndefElements;
9078 0 : BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
9079 : int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, FloatBits + 1);
9080 : if (C == -1 || C == 0 || C > FloatBits)
9081 759 : return SDValue();
9082 :
9083 : MVT ResTy;
9084 : unsigned NumLanes = Op.getValueType().getVectorNumElements();
9085 759 : switch (NumLanes) {
9086 2 : default:
9087 : return SDValue();
9088 1514 : case 2:
9089 : ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
9090 431 : break;
9091 : case 4:
9092 : ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64;
9093 652 : break;
9094 188 : }
9095 :
9096 : if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps())
9097 : return SDValue();
9098 :
9099 138 : SDLoc DL(N);
9100 171 : SDValue ConvInput = Op.getOperand(0);
9101 : bool IsSigned = Opc == ISD::SINT_TO_FP;
9102 33 : if (IntBits < FloatBits)
9103 : ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
9104 33 : ResTy, ConvInput);
9105 33 :
9106 66 : unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
9107 : : Intrinsic::aarch64_neon_vcvtfxu2fp;
9108 : return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
9109 : DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
9110 66 : DAG.getConstant(C, DL, MVT::i32));
9111 : }
9112 :
9113 33 : /// An EXTR instruction is made up of two shifts, ORed together. This helper
9114 66 : /// searches for and classifies those shifts.
9115 : static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,
9116 : bool &FromHi) {
9117 105 : if (N.getOpcode() == ISD::SHL)
9118 : FromHi = false;
9119 : else if (N.getOpcode() == ISD::SRL)
9120 : FromHi = true;
9121 : else
9122 446 : return false;
9123 :
9124 : if (!isa<ConstantSDNode>(N.getOperand(1)))
9125 446 : return false;
9126 0 :
9127 : ShiftAmount = N->getConstantOperandVal(1);
9128 446 : Src = N->getOperand(0);
9129 892 : return true;
9130 : }
9131 427 :
9132 : /// EXTR instruction extracts a contiguous chunk of bits from two existing
9133 19 : /// registers viewed as a high/low pair. This function looks for the pattern:
9134 19 : /// <tt>(or (shl VAL1, \#N), (srl VAL2, \#RegWidth-N))</tt> and replaces it
9135 4 : /// with an EXTR. Can't quite be done in TableGen because the two immediates
9136 : /// aren't independent.
9137 15 : static SDValue tryCombineToEXTR(SDNode *N,
9138 15 : TargetLowering::DAGCombinerInfo &DCI) {
9139 15 : SelectionDAG &DAG = DCI.DAG;
9140 0 : SDLoc DL(N);
9141 : EVT VT = N->getValueType(0);
9142 15 :
9143 15 : assert(N->getOpcode() == ISD::OR && "Unexpected root");
9144 15 :
9145 1 : if (VT != MVT::i32 && VT != MVT::i64)
9146 : return SDValue();
9147 :
9148 14 : SDValue LHS;
9149 1 : uint32_t ShiftLHS = 0;
9150 : bool LHSFromHi = false;
9151 : if (!findEXTRHalf(N->getOperand(0), LHS, ShiftLHS, LHSFromHi))
9152 : return SDValue();
9153 13 :
9154 13 : SDValue RHS;
9155 13 : uint32_t ShiftRHS = 0;
9156 4 : bool RHSFromHi = false;
9157 : if (!findEXTRHalf(N->getOperand(1), RHS, ShiftRHS, RHSFromHi))
9158 : return SDValue();
9159 9 :
9160 9 : // If they're both trying to come from the high part of the register, they're
9161 0 : // not really an EXTR.
9162 0 : if (LHSFromHi == RHSFromHi)
9163 7 : return SDValue();
9164 7 :
9165 7 : if (ShiftLHS + ShiftRHS != VT.getSizeInBits())
9166 2 : return SDValue();
9167 2 :
9168 2 : if (LHSFromHi) {
9169 : std::swap(LHS, RHS);
9170 : std::swap(ShiftLHS, ShiftRHS);
9171 9 : }
9172 0 :
9173 : return DAG.getNode(AArch64ISD::EXTR, DL, VT, LHS, RHS,
9174 : DAG.getConstant(ShiftRHS, DL, MVT::i64));
9175 : }
9176 :
9177 : static SDValue tryCombineToBSL(SDNode *N,
9178 9 : TargetLowering::DAGCombinerInfo &DCI) {
9179 9 : EVT VT = N->getValueType(0);
9180 : SelectionDAG &DAG = DCI.DAG;
9181 : SDLoc DL(N);
9182 :
9183 : if (!VT.isVector())
9184 9 : return SDValue();
9185 :
9186 9 : SDValue N0 = N->getOperand(0);
9187 4 : if (N0.getOpcode() != ISD::AND)
9188 : return SDValue();
9189 9 :
9190 : SDValue N1 = N->getOperand(1);
9191 : if (N1.getOpcode() != ISD::AND)
9192 : return SDValue();
9193 :
9194 189 : // We only have to look for constant vectors here since the general, variable
9195 : // case can be handled in TableGen.
9196 : unsigned Bits = VT.getScalarSizeInBits();
9197 189 : uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1);
9198 0 : for (int i = 1; i >= 0; --i)
9199 : for (int j = 1; j >= 0; --j) {
9200 189 : BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(i));
9201 189 : BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(j));
9202 275 : if (!BVN0 || !BVN1)
9203 274 : continue;
9204 85 :
9205 166 : bool FoundMatch = true;
9206 : for (unsigned k = 0; k < VT.getVectorNumElements(); ++k) {
9207 23 : ConstantSDNode *CN0 = dyn_cast<ConstantSDNode>(BVN0->getOperand(k));
9208 23 : ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(BVN1->getOperand(k));
9209 2 : if (!CN0 || !CN1 ||
9210 : CN0->getZExtValue() != (BitMask & ~CN1->getZExtValue())) {
9211 21 : FoundMatch = false;
9212 21 : break;
9213 21 : }
9214 0 : }
9215 :
9216 21 : if (FoundMatch)
9217 21 : return DAG.getNode(AArch64ISD::BSL, DL, VT, SDValue(BVN0, 0),
9218 21 : N0->getOperand(1 - i), N1->getOperand(1 - j));
9219 0 : }
9220 :
9221 : return SDValue();
9222 21 : }
9223 1 :
9224 : static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
9225 : const AArch64Subtarget *Subtarget) {
9226 : // Attempt to form an EXTR from (or (shl VAL1, #N), (srl VAL2, #RegWidth-N))
9227 20 : SelectionDAG &DAG = DCI.DAG;
9228 20 : EVT VT = N->getValueType(0);
9229 2 :
9230 : if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9231 : return SDValue();
9232 18 :
9233 18 : if (SDValue Res = tryCombineToEXTR(N, DCI))
9234 0 : return Res;
9235 0 :
9236 14 : if (SDValue Res = tryCombineToBSL(N, DCI))
9237 14 : return Res;
9238 14 :
9239 4 : return SDValue();
9240 4 : }
9241 4 :
9242 : static SDValue performSRLCombine(SDNode *N,
9243 : TargetLowering::DAGCombinerInfo &DCI) {
9244 18 : SelectionDAG &DAG = DCI.DAG;
9245 1 : EVT VT = N->getValueType(0);
9246 : if (VT != MVT::i32 && VT != MVT::i64)
9247 : return SDValue();
9248 17 :
9249 : // Canonicalize (srl (bswap i32 x), 16) to (rotr (bswap i32 x), 16), if the
9250 17 : // high 16-bits of x are zero. Similarly, canonicalize (srl (bswap i64 x), 32)
9251 10 : // to (rotr (bswap i64 x), 32), if the high 32-bits of x are zero.
9252 11 : SDValue N0 = N->getOperand(0);
9253 : if (N0.getOpcode() == ISD::BSWAP) {
9254 17 : SDLoc DL(N);
9255 : SDValue N1 = N->getOperand(1);
9256 : SDValue N00 = N0.getOperand(0);
9257 : if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
9258 17 : uint64_t ShiftAmt = C->getZExtValue();
9259 : if (VT == MVT::i32 && ShiftAmt == 16 &&
9260 : DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(32, 16)))
9261 : return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
9262 : if (VT == MVT::i64 && ShiftAmt == 32 &&
9263 0 : DAG.MaskedValueIsZero(N00, APInt::getHighBitsSet(64, 32)))
9264 : return DAG.getNode(ISD::ROTR, DL, VT, N0, N1);
9265 0 : }
9266 0 : }
9267 0 : return SDValue();
9268 0 : }
9269 :
9270 0 : static SDValue performBitcastCombine(SDNode *N,
9271 : TargetLowering::DAGCombinerInfo &DCI,
9272 : SelectionDAG &DAG) {
9273 0 : // Wait 'til after everything is legalized to try this. That way we have
9274 : // legal vector types and such.
9275 0 : if (DCI.isBeforeLegalizeOps())
9276 0 : return SDValue();
9277 0 :
9278 : // Remove extraneous bitcasts around an extract_subvector.
9279 : // For example,
9280 : // (v4i16 (bitconvert
9281 : // (extract_subvector (v2i64 (bitconvert (v8i16 ...)), (i64 1)))))
9282 : // becomes
9283 : // (extract_subvector ((v8i16 ...), (i64 4)))
9284 :
9285 0 : // Only interested in 64-bit vectors as the ultimate result.
9286 : EVT VT = N->getValueType(0);
9287 0 : if (!VT.isVector())
9288 : return SDValue();
9289 0 : if (VT.getSimpleVT().getSizeInBits() != 64)
9290 : return SDValue();
9291 : // Is the operand an extract_subvector starting at the beginning or halfway
9292 : // point of the vector? A low half may also come through as an
9293 0 : // EXTRACT_SUBREG, so look for that, too.
9294 0 : SDValue Op0 = N->getOperand(0);
9295 : if (Op0->getOpcode() != ISD::EXTRACT_SUBVECTOR &&
9296 0 : !(Op0->isMachineOpcode() &&
9297 0 : Op0->getMachineOpcode() == AArch64::EXTRACT_SUBREG))
9298 0 : return SDValue();
9299 0 : uint64_t idx = cast<ConstantSDNode>(Op0->getOperand(1))->getZExtValue();
9300 0 : if (Op0->getOpcode() == ISD::EXTRACT_SUBVECTOR) {
9301 : if (Op0->getValueType(0).getVectorNumElements() != idx && idx != 0)
9302 0 : return SDValue();
9303 0 : } else if (Op0->getMachineOpcode() == AArch64::EXTRACT_SUBREG) {
9304 0 : if (idx != AArch64::dsub)
9305 0 : return SDValue();
9306 0 : // The dsub reference is equivalent to a lane zero subvector reference.
9307 : idx = 0;
9308 : }
9309 : // Look through the bitcast of the input to the extract.
9310 0 : if (Op0->getOperand(0)->getOpcode() != ISD::BITCAST)
9311 0 : return SDValue();
9312 : SDValue Source = Op0->getOperand(0)->getOperand(0);
9313 0 : // If the source type has twice the number of elements as our destination
9314 0 : // type, we know this is an extract of the high or low half of the vector.
9315 : EVT SVT = Source->getValueType(0);
9316 0 : if (!SVT.isVector() ||
9317 : SVT.getVectorNumElements() != VT.getVectorNumElements() * 2)
9318 : return SDValue();
9319 :
9320 : LLVM_DEBUG(
9321 : dbgs() << "aarch64-lower: bitcast extract_subvector simplification\n");
9322 0 :
9323 : // Create the simplified form to just extract the low or high half of the
9324 : // vector directly rather than bothering with the bitcasts.
9325 0 : SDLoc dl(N);
9326 : unsigned NumElements = VT.getVectorNumElements();
9327 0 : if (idx) {
9328 0 : SDValue HalfIdx = DAG.getConstant(NumElements, dl, MVT::i64);
9329 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Source, HalfIdx);
9330 : } else {
9331 0 : SDValue SubReg = DAG.getTargetConstant(AArch64::dsub, dl, MVT::i32);
9332 0 : return SDValue(DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl, VT,
9333 : Source, SubReg),
9334 0 : 0);
9335 0 : }
9336 0 : }
9337 :
9338 0 : static SDValue performConcatVectorsCombine(SDNode *N,
9339 0 : TargetLowering::DAGCombinerInfo &DCI,
9340 0 : SelectionDAG &DAG) {
9341 : SDLoc dl(N);
9342 : EVT VT = N->getValueType(0);
9343 : SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
9344 :
9345 0 : // Optimize concat_vectors of truncated vectors, where the intermediate
9346 0 : // type is illegal, to avoid said illegality, e.g.,
9347 0 : // (v4i16 (concat_vectors (v2i16 (truncate (v2i64))),
9348 0 : // (v2i16 (truncate (v2i64)))))
9349 0 : // ->
9350 0 : // (v4i16 (truncate (vector_shuffle (v4i32 (bitcast (v2i64))),
9351 0 : // (v4i32 (bitcast (v2i64))),
9352 : // <0, 2, 4, 6>)))
9353 : // This isn't really target-specific, but ISD::TRUNCATE legality isn't keyed
9354 0 : // on both input and result type, so we might generate worse code.
9355 0 : // On AArch64 we know it's fine for v2i64->v4i16 and v4i32->v8i8.
9356 0 : if (N->getNumOperands() == 2 &&
9357 0 : N0->getOpcode() == ISD::TRUNCATE &&
9358 0 : N1->getOpcode() == ISD::TRUNCATE) {
9359 : SDValue N00 = N0->getOperand(0);
9360 : SDValue N10 = N1->getOperand(0);
9361 : EVT N00VT = N00.getValueType();
9362 :
9363 : if (N00VT == N10.getValueType() &&
9364 0 : (N00VT == MVT::v2i64 || N00VT == MVT::v4i32) &&
9365 : N00VT.getScalarSizeInBits() == 4 * VT.getScalarSizeInBits()) {
9366 0 : MVT MidVT = (N00VT == MVT::v2i64 ? MVT::v4i32 : MVT::v8i16);
9367 : SmallVector<int, 8> Mask(MidVT.getVectorNumElements());
9368 : for (size_t i = 0; i < Mask.size(); ++i)
9369 0 : Mask[i] = i * 2;
9370 : return DAG.getNode(ISD::TRUNCATE, dl, VT,
9371 : DAG.getVectorShuffle(
9372 0 : MidVT, dl,
9373 : DAG.getNode(ISD::BITCAST, dl, MidVT, N00),
9374 : DAG.getNode(ISD::BITCAST, dl, MidVT, N10), Mask));
9375 0 : }
9376 0 : }
9377 :
9378 0 : // Wait 'til after everything is legalized to try this. That way we have
9379 0 : // legal vector types and such.
9380 : if (DCI.isBeforeLegalizeOps())
9381 0 : return SDValue();
9382 0 :
9383 : // If we see a (concat_vectors (v1x64 A), (v1x64 A)) it's really a vector
9384 0 : // splat. The indexed instructions are going to be expecting a DUPLANE64, so
9385 0 : // canonicalise to that.
9386 : if (N0 == N1 && VT.getVectorNumElements() == 2) {
9387 0 : assert(VT.getScalarSizeInBits() == 64);
9388 : return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG),
9389 : DAG.getConstant(0, dl, MVT::i64));
9390 0 : }
9391 :
9392 0 : // Canonicalise concat_vectors so that the right-hand vector has as few
9393 0 : // bit-casts as possible before its real operation. The primary matching
9394 0 : // destination for these operations will be the narrowing "2" instructions,
9395 0 : // which depend on the operation being performed on this right-hand vector.
9396 : // For example,
9397 : // (concat_vectors LHS, (v1i64 (bitconvert (v4i16 RHS))))
9398 : // becomes
9399 : // (bitconvert (concat_vectors (v4i16 (bitconvert LHS)), RHS))
9400 0 :
9401 0 : if (N1->getOpcode() != ISD::BITCAST)
9402 : return SDValue();
9403 0 : SDValue RHS = N1->getOperand(0);
9404 0 : MVT RHSTy = RHS.getValueType().getSimpleVT();
9405 : // If the RHS is not a vector, this is not the pattern we're looking for.
9406 0 : if (!RHSTy.isVector())
9407 0 : return SDValue();
9408 0 :
9409 0 : LLVM_DEBUG(
9410 0 : dbgs() << "aarch64-lower: concat_vectors bitcast simplification\n");
9411 0 :
9412 0 : MVT ConcatTy = MVT::getVectorVT(RHSTy.getVectorElementType(),
9413 : RHSTy.getVectorNumElements() * 2);
9414 : return DAG.getNode(ISD::BITCAST, dl, VT,
9415 0 : DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatTy,
9416 : DAG.getNode(ISD::BITCAST, dl, RHSTy, N0),
9417 : RHS));
9418 10049 : }
9419 :
9420 : static SDValue tryCombineFixedPointConvert(SDNode *N,
9421 : TargetLowering::DAGCombinerInfo &DCI,
9422 : SelectionDAG &DAG) {
9423 20098 : // Wait until after everything is legalized to try this. That way we have
9424 4715 : // legal vector types and such.
9425 : if (DCI.isBeforeLegalizeOps())
9426 : return SDValue();
9427 : // Transform a scalar conversion of a value from a lane extract into a
9428 : // lane extract of a vector conversion. E.g., from foo1 to foo2:
9429 : // double foo1(int64x2_t a) { return vcvtd_n_f64_s64(a[1], 9); }
9430 : // double foo2(int64x2_t a) { return vcvtq_n_f64_s64(a, 9)[1]; }
9431 : //
9432 : // The second form interacts better with instruction selection and the
9433 : // register allocator to avoid cross-class register copies that aren't
9434 10668 : // coalescable due to a lane reference.
9435 5334 :
9436 528 : // Check the operand and see if it originates from a lane extract.
9437 4806 : SDValue Op1 = N->getOperand(1);
9438 3144 : if (Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
9439 : // Yep, no additional predication needed. Perform the transform.
9440 : SDValue IID = N->getOperand(0);
9441 : SDValue Shift = N->getOperand(2);
9442 1662 : SDValue Vec = Op1.getOperand(0);
9443 3324 : SDValue Lane = Op1.getOperand(1);
9444 40 : EVT ResTy = N->getValueType(0);
9445 : EVT VecResTy;
9446 1560 : SDLoc DL(N);
9447 204 :
9448 102 : // The vector width should be 128 bits by the time we get here, even
9449 186 : // if it started as 64 bits (the extract_vector handling will have
9450 0 : // done so).
9451 40 : assert(Vec.getValueSizeInBits() == 128 &&
9452 40 : "unexpected vector size on extract_vector_elt!");
9453 0 : if (Vec.getValueType() == MVT::v4i32)
9454 : VecResTy = MVT::v4f32;
9455 : else if (Vec.getValueType() == MVT::v2i64)
9456 : VecResTy = MVT::v2f64;
9457 : else
9458 102 : llvm_unreachable("unexpected vector type!");
9459 41 :
9460 61 : SDValue Convert =
9461 : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VecResTy, IID, Vec, Shift);
9462 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ResTy, Convert, Lane);
9463 122 : }
9464 61 : return SDValue();
9465 59 : }
9466 7 :
9467 : // AArch64 high-vector "long" operations are formed by performing the non-high
9468 : // version on an extract_subvector of each operand which gets the high half:
9469 : //
9470 : // (longop2 LHS, RHS) == (longop (extract_high LHS), (extract_high RHS))
9471 : //
9472 : // However, there are cases which don't have an extract_high explicitly, but
9473 : // have another operation that can be made compatible with one for free. For
9474 : // example:
9475 54 : //
9476 52 : // (dupv64 scalar) --> (extract_high (dup128 scalar))
9477 52 : //
9478 : // This routine does the actual conversion of such DUPs, once outer routines
9479 2 : // have determined that everything else is in order.
9480 2 : // It also supports immediate DUP-like nodes (MOVI/MVNi), which we can fold
9481 : // similarly here.
9482 2 : static SDValue tryExtendDUPToExtractHigh(SDValue N, SelectionDAG &DAG) {
9483 : switch (N.getOpcode()) {
9484 : case AArch64ISD::DUP:
9485 : case AArch64ISD::DUPLANE8:
9486 824 : case AArch64ISD::DUPLANE16:
9487 : case AArch64ISD::DUPLANE32:
9488 : case AArch64ISD::DUPLANE64:
9489 : case AArch64ISD::MOVI:
9490 824 : case AArch64ISD::MOVIshift:
9491 824 : case AArch64ISD::MOVIedit:
9492 : case AArch64ISD::MOVImsl:
9493 : case AArch64ISD::MVNIshift:
9494 : case AArch64ISD::MVNImsl:
9495 : break;
9496 : default:
9497 : // FMOV could be supported, but isn't very useful, as it would only occur
9498 : // if you passed a bitcast' floating point immediate to an eligible long
9499 : // integer op (addl, smull, ...).
9500 : return SDValue();
9501 : }
9502 :
9503 : MVT NarrowTy = N.getSimpleValueType();
9504 1612 : if (!NarrowTy.is64BitVector())
9505 824 : return SDValue();
9506 44 :
9507 41 : MVT ElementTy = NarrowTy.getVectorElementType();
9508 41 : unsigned NumElems = NarrowTy.getVectorNumElements();
9509 41 : MVT NewVT = MVT::getVectorVT(ElementTy, NumElems * 2);
9510 :
9511 : SDLoc dl(N);
9512 0 : return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NarrowTy,
9513 39 : DAG.getNode(N->getOpcode(), dl, NewVT, N->ops()),
9514 : DAG.getConstant(NumElems, dl, MVT::i64));
9515 2 : }
9516 28 :
9517 24 : static bool isEssentiallyExtractSubvector(SDValue N) {
9518 : if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
9519 : return true;
9520 :
9521 : return N.getOpcode() == ISD::BITCAST &&
9522 2 : N.getOperand(0).getOpcode() == ISD::EXTRACT_SUBVECTOR;
9523 : }
9524 :
9525 : /// Helper structure to keep track of ISD::SET_CC operands.
9526 : struct GenericSetCCInfo {
9527 : const SDValue *Opnd0;
9528 1644 : const SDValue *Opnd1;
9529 441 : ISD::CondCode CC;
9530 : };
9531 :
9532 : /// Helper structure to keep track of a SET_CC lowered into AArch64 code.
9533 : struct AArch64SetCCInfo {
9534 25 : const SDValue *Cmp;
9535 : AArch64CC::CondCode CC;
9536 : };
9537 13 :
9538 : /// Helper structure to keep track of SetCC information.
9539 : union SetCCInfo {
9540 : GenericSetCCInfo Generic;
9541 : AArch64SetCCInfo AArch64;
9542 : };
9543 :
9544 : /// Helper structure to be able to read SetCC information. If set to
9545 : /// true, IsAArch64 field, Info is a AArch64SetCCInfo, otherwise Info is a
9546 : /// GenericSetCCInfo.
9547 : struct SetCCInfoAndKind {
9548 : SetCCInfo Info;
9549 368 : bool IsAArch64;
9550 304 : };
9551 64 :
9552 64 : /// Check whether or not \p Op is a SET_CC operation, either a generic or
9553 : /// an
9554 64 : /// AArch64 lowered one.
9555 0 : /// \p SetCCInfo is filled accordingly.
9556 : /// \post SetCCInfo is meanginfull only when this function returns true.
9557 : /// \return True when Op is a kind of SET_CC operation.
9558 : static bool isSetCC(SDValue Op, SetCCInfoAndKind &SetCCInfo) {
9559 : // If this is a setcc, this is straight forward.
9560 : if (Op.getOpcode() == ISD::SETCC) {
9561 64 : SetCCInfo.Info.Generic.Opnd0 = &Op.getOperand(0);
9562 : SetCCInfo.Info.Generic.Opnd1 = &Op.getOperand(1);
9563 : SetCCInfo.Info.Generic.CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
9564 : SetCCInfo.IsAArch64 = false;
9565 64 : return true;
9566 : }
9567 : // Otherwise, check if this is a matching csel instruction.
9568 85 : // In other words:
9569 : // - csel 1, 0, cc
9570 : // - csel 0, 1, !cc
9571 : if (Op.getOpcode() != AArch64ISD::CSEL)
9572 : return false;
9573 170 : // Set the information about the operands.
9574 44 : // TODO: we want the operands of the Cmp not the csel
9575 : SetCCInfo.Info.AArch64.Cmp = &Op.getOperand(3);
9576 : SetCCInfo.IsAArch64 = true;
9577 : SetCCInfo.Info.AArch64.CC = static_cast<AArch64CC::CondCode>(
9578 : cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue());
9579 :
9580 : // Check that the operands matches the constraints:
9581 : // (1) Both operands must be constants.
9582 : // (2) One must be 1 and the other must be 0.
9583 : ConstantSDNode *TValue = dyn_cast<ConstantSDNode>(Op.getOperand(0));
9584 : ConstantSDNode *FValue = dyn_cast<ConstantSDNode>(Op.getOperand(1));
9585 41 :
9586 41 : // Check (1).
9587 : if (!TValue || !FValue)
9588 1 : return false;
9589 1 :
9590 1 : // Check (2).
9591 1 : if (!TValue->isOne()) {
9592 1 : // Update the comparison when we are interested in !cc.
9593 1 : std::swap(TValue, FValue);
9594 : SetCCInfo.Info.AArch64.CC =
9595 : AArch64CC::getInvertedCondCode(SetCCInfo.Info.AArch64.CC);
9596 : }
9597 : return TValue->isOne() && FValue->isNullValue();
9598 : }
9599 :
9600 : // Returns true if Op is setcc or zext of setcc.
9601 : static bool isSetCCOrZExtSetCC(const SDValue& Op, SetCCInfoAndKind &Info) {
9602 0 : if (isSetCC(Op, Info))
9603 : return true;
9604 1 : return ((Op.getOpcode() == ISD::ZERO_EXTEND) &&
9605 : isSetCC(Op->getOperand(0), Info));
9606 0 : }
9607 :
9608 : // The folding we want to perform is:
9609 1 : // (add x, [zext] (setcc cc ...) )
9610 1 : // -->
9611 : // (csel x, (add x, 1), !cc ...)
9612 40 : //
9613 : // The latter will get matched to a CSINC instruction.
9614 : static SDValue performSetccAddFolding(SDNode *Op, SelectionDAG &DAG) {
9615 : assert(Op && Op->getOpcode() == ISD::ADD && "Unexpected operation!");
9616 : SDValue LHS = Op->getOperand(0);
9617 : SDValue RHS = Op->getOperand(1);
9618 : SetCCInfoAndKind InfoAndKind;
9619 :
9620 : // If neither operand is a SET_CC, give up.
9621 : if (!isSetCCOrZExtSetCC(LHS, InfoAndKind)) {
9622 : std::swap(LHS, RHS);
9623 : if (!isSetCCOrZExtSetCC(LHS, InfoAndKind))
9624 : return SDValue();
9625 : }
9626 :
9627 : // FIXME: This could be generatized to work for FP comparisons.
9628 : EVT CmpVT = InfoAndKind.IsAArch64
9629 : ? InfoAndKind.Info.AArch64.Cmp->getOperand(0).getValueType()
9630 0 : : InfoAndKind.Info.Generic.Opnd0->getValueType();
9631 0 : if (CmpVT != MVT::i32 && CmpVT != MVT::i64)
9632 : return SDValue();
9633 :
9634 : SDValue CCVal;
9635 : SDValue Cmp;
9636 : SDLoc dl(Op);
9637 : if (InfoAndKind.IsAArch64) {
9638 : CCVal = DAG.getConstant(
9639 : AArch64CC::getInvertedCondCode(InfoAndKind.Info.AArch64.CC), dl,
9640 : MVT::i32);
9641 : Cmp = *InfoAndKind.Info.AArch64.Cmp;
9642 : } else
9643 : Cmp = getAArch64Cmp(*InfoAndKind.Info.Generic.Opnd0,
9644 0 : *InfoAndKind.Info.Generic.Opnd1,
9645 : ISD::getSetCCInverse(InfoAndKind.Info.Generic.CC, true),
9646 : CCVal, DAG, dl);
9647 :
9648 0 : EVT VT = Op->getValueType(0);
9649 : LHS = DAG.getNode(ISD::ADD, dl, VT, RHS, DAG.getConstant(1, dl, VT));
9650 : return DAG.getNode(AArch64ISD::CSEL, dl, VT, RHS, LHS, CCVal, Cmp);
9651 0 : }
9652 :
9653 0 : // The basic add/sub long vector instructions have variants with "2" on the end
9654 : // which act on the high-half of their inputs. They are normally matched by
9655 0 : // patterns like:
9656 : //
9657 0 : // (add (zeroext (extract_high LHS)),
9658 : // (zeroext (extract_high RHS)))
9659 0 : // -> uaddl2 vD, vN, vM
9660 : //
9661 : // However, if one of the extracts is something like a duplicate, this
9662 0 : // instruction can still be used profitably. This function puts the DAG into a
9663 : // more appropriate form for those patterns to trigger.
9664 : static SDValue performAddSubLongCombine(SDNode *N,
9665 0 : TargetLowering::DAGCombinerInfo &DCI,
9666 152 : SelectionDAG &DAG) {
9667 0 : if (DCI.isBeforeLegalizeOps())
9668 : return SDValue();
9669 98 :
9670 8 : MVT VT = N->getSimpleValueType(0);
9671 : if (!VT.is128BitVector()) {
9672 : if (N->getOpcode() == ISD::ADD)
9673 : return performSetccAddFolding(N, DAG);
9674 : return SDValue();
9675 : }
9676 :
9677 : // Make sure both branches are extended in the same way.
9678 : SDValue LHS = N->getOperand(0);
9679 : SDValue RHS = N->getOperand(1);
9680 : if ((LHS.getOpcode() != ISD::ZERO_EXTEND &&
9681 : LHS.getOpcode() != ISD::SIGN_EXTEND) ||
9682 : LHS.getOpcode() != RHS.getOpcode())
9683 : return SDValue();
9684 :
9685 : unsigned ExtType = LHS.getOpcode();
9686 :
9687 : // It's not worth doing if at least one of the inputs isn't already an
9688 : // extract, but we don't know which it'll be so we have to try both.
9689 : if (isEssentiallyExtractSubvector(LHS.getOperand(0))) {
9690 : RHS = tryExtendDUPToExtractHigh(RHS.getOperand(0), DAG);
9691 : if (!RHS.getNode())
9692 : return SDValue();
9693 :
9694 : RHS = DAG.getNode(ExtType, SDLoc(N), VT, RHS);
9695 : } else if (isEssentiallyExtractSubvector(RHS.getOperand(0))) {
9696 : LHS = tryExtendDUPToExtractHigh(LHS.getOperand(0), DAG);
9697 : if (!LHS.getNode())
9698 : return SDValue();
9699 :
9700 : LHS = DAG.getNode(ExtType, SDLoc(N), VT, LHS);
9701 : }
9702 :
9703 : return DAG.getNode(N->getOpcode(), SDLoc(N), VT, LHS, RHS);
9704 : }
9705 :
9706 0 : // Massage DAGs which we can use the high-half "long" operations on into
9707 : // something isel will recognize better. E.g.
9708 0 : //
9709 0 : // (aarch64_neon_umull (extract_high vec) (dupv64 scalar)) -->
9710 0 : // (aarch64_neon_umull (extract_high (v2i64 vec)))
9711 0 : // (extract_high (v2i64 (dup128 scalar)))))
9712 0 : //
9713 0 : static SDValue tryCombineLongOpWithDup(unsigned IID, SDNode *N,
9714 : TargetLowering::DAGCombinerInfo &DCI,
9715 : SelectionDAG &DAG) {
9716 : if (DCI.isBeforeLegalizeOps())
9717 : return SDValue();
9718 :
9719 0 : SDValue LHS = N->getOperand(1);
9720 0 : SDValue RHS = N->getOperand(2);
9721 : assert(LHS.getValueType().is64BitVector() &&
9722 : RHS.getValueType().is64BitVector() &&
9723 0 : "unexpected shape for long operation");
9724 0 :
9725 0 : // Either node could be a DUP, but it's not worth doing both of them (you'd
9726 0 : // just as well use the non-high version) so look for a corresponding extract
9727 : // operation on the other "wing".
9728 : if (isEssentiallyExtractSubvector(LHS)) {
9729 : RHS = tryExtendDUPToExtractHigh(RHS, DAG);
9730 : if (!RHS.getNode())
9731 : return SDValue();
9732 : } else if (isEssentiallyExtractSubvector(RHS)) {
9733 : LHS = tryExtendDUPToExtractHigh(LHS, DAG);
9734 : if (!LHS.getNode())
9735 0 : return SDValue();
9736 0 : }
9737 :
9738 : return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), N->getValueType(0),
9739 0 : N->getOperand(0), LHS, RHS);
9740 : }
9741 :
9742 0 : static SDValue tryCombineShiftImm(unsigned IID, SDNode *N, SelectionDAG &DAG) {
9743 : MVT ElemTy = N->getSimpleValueType(0).getScalarType();
9744 : unsigned ElemBits = ElemTy.getSizeInBits();
9745 0 :
9746 : int64_t ShiftAmount;
9747 : if (BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(2))) {
9748 : APInt SplatValue, SplatUndef;
9749 12222 : unsigned SplatBitSize;
9750 12222 : bool HasAnyUndefs;
9751 : if (!BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
9752 24477 : HasAnyUndefs, ElemBits) ||
9753 37 : SplatBitSize != ElemBits)
9754 : return SDValue();
9755 :
9756 : ShiftAmount = SplatValue.getSExtValue();
9757 : } else if (ConstantSDNode *CVN = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
9758 : ShiftAmount = CVN->getSExtValue();
9759 : } else
9760 : return SDValue();
9761 :
9762 6112 : unsigned Opcode;
9763 : bool IsRightShift;
9764 6112 : switch (IID) {
9765 6112 : default:
9766 : llvm_unreachable("Unknown shift intrinsic");
9767 : case Intrinsic::aarch64_neon_sqshl:
9768 : Opcode = AArch64ISD::SQSHL_I;
9769 6112 : IsRightShift = false;
9770 : break;
9771 6110 : case Intrinsic::aarch64_neon_uqshl:
9772 6108 : Opcode = AArch64ISD::UQSHL_I;
9773 : IsRightShift = false;
9774 : break;
9775 : case Intrinsic::aarch64_neon_srshl:
9776 4 : Opcode = AArch64ISD::SRSHR_I;
9777 8 : IsRightShift = true;
9778 4 : break;
9779 : case Intrinsic::aarch64_neon_urshl:
9780 0 : Opcode = AArch64ISD::URSHR_I;
9781 : IsRightShift = true;
9782 4 : break;
9783 4 : case Intrinsic::aarch64_neon_sqshlu:
9784 : Opcode = AArch64ISD::SQSHLU_I;
9785 4 : IsRightShift = false;
9786 4 : break;
9787 4 : }
9788 4 :
9789 4 : if (IsRightShift && ShiftAmount <= -1 && ShiftAmount >= -(int)ElemBits) {
9790 : SDLoc dl(N);
9791 0 : return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
9792 0 : DAG.getConstant(-ShiftAmount, dl, MVT::i32));
9793 : } else if (!IsRightShift && ShiftAmount >= 0 && ShiftAmount < ElemBits) {
9794 0 : SDLoc dl(N);
9795 : return DAG.getNode(Opcode, dl, N->getValueType(0), N->getOperand(1),
9796 8 : DAG.getConstant(ShiftAmount, dl, MVT::i32));
9797 4 : }
9798 4 :
9799 : return SDValue();
9800 : }
9801 :
9802 : // The CRC32[BH] instructions ignore the high bits of their data operand. Since
9803 : // the intrinsics must be legal and take an i32, this means there's almost
9804 : // certainly going to be a zext in the DAG which we can eliminate.
9805 : static SDValue tryCombineCRC32(unsigned Mask, SDNode *N, SelectionDAG &DAG) {
9806 : SDValue AndN = N->getOperand(2);
9807 : if (AndN.getOpcode() != ISD::AND)
9808 : return SDValue();
9809 :
9810 : ConstantSDNode *CMask = dyn_cast<ConstantSDNode>(AndN.getOperand(1));
9811 : if (!CMask || CMask->getZExtValue() != Mask)
9812 17086 : return SDValue();
9813 :
9814 : return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, SDLoc(N), MVT::i32,
9815 34172 : N->getOperand(0), N->getOperand(1), AndN.getOperand(0));
9816 9042 : }
9817 :
9818 8044 : static SDValue combineAcrossLanesIntrinsic(unsigned Opc, SDNode *N,
9819 8044 : SelectionDAG &DAG) {
9820 6751 : SDLoc dl(N);
9821 6112 : return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, N->getValueType(0),
9822 639 : DAG.getNode(Opc, dl,
9823 : N->getOperand(1).getSimpleValueType(),
9824 : N->getOperand(1)),
9825 : DAG.getConstant(0, dl, MVT::i64));
9826 1293 : }
9827 1293 :
9828 1215 : static SDValue performIntrinsicCombine(SDNode *N,
9829 1346 : TargetLowering::DAGCombinerInfo &DCI,
9830 : const AArch64Subtarget *Subtarget) {
9831 1188 : SelectionDAG &DAG = DCI.DAG;
9832 : unsigned IID = getIntrinsicID(N);
9833 : switch (IID) {
9834 : default:
9835 : break;
9836 : case Intrinsic::aarch64_neon_vcvtfxs2fp:
9837 105 : case Intrinsic::aarch64_neon_vcvtfxu2fp:
9838 58 : return tryCombineFixedPointConvert(N, DCI, DAG);
9839 58 : case Intrinsic::aarch64_neon_saddv:
9840 56 : return combineAcrossLanesIntrinsic(AArch64ISD::SADDV, N, DAG);
9841 : case Intrinsic::aarch64_neon_uaddv:
9842 2 : return combineAcrossLanesIntrinsic(AArch64ISD::UADDV, N, DAG);
9843 47 : case Intrinsic::aarch64_neon_sminv:
9844 2 : return combineAcrossLanesIntrinsic(AArch64ISD::SMINV, N, DAG);
9845 2 : case Intrinsic::aarch64_neon_uminv:
9846 0 : return combineAcrossLanesIntrinsic(AArch64ISD::UMINV, N, DAG);
9847 : case Intrinsic::aarch64_neon_smaxv:
9848 2 : return combineAcrossLanesIntrinsic(AArch64ISD::SMAXV, N, DAG);
9849 : case Intrinsic::aarch64_neon_umaxv:
9850 : return combineAcrossLanesIntrinsic(AArch64ISD::UMAXV, N, DAG);
9851 147 : case Intrinsic::aarch64_neon_fmax:
9852 : return DAG.getNode(ISD::FMAXNAN, SDLoc(N), N->getValueType(0),
9853 : N->getOperand(1), N->getOperand(2));
9854 : case Intrinsic::aarch64_neon_fmin:
9855 : return DAG.getNode(ISD::FMINNAN, SDLoc(N), N->getValueType(0),
9856 : N->getOperand(1), N->getOperand(2));
9857 : case Intrinsic::aarch64_neon_fmaxnm:
9858 : return DAG.getNode(ISD::FMAXNUM, SDLoc(N), N->getValueType(0),
9859 : N->getOperand(1), N->getOperand(2));
9860 : case Intrinsic::aarch64_neon_fminnm:
9861 0 : return DAG.getNode(ISD::FMINNUM, SDLoc(N), N->getValueType(0),
9862 : N->getOperand(1), N->getOperand(2));
9863 : case Intrinsic::aarch64_neon_smull:
9864 0 : case Intrinsic::aarch64_neon_umull:
9865 0 : case Intrinsic::aarch64_neon_pmull:
9866 : case Intrinsic::aarch64_neon_sqdmull:
9867 0 : return tryCombineLongOpWithDup(IID, N, DCI, DAG);
9868 0 : case Intrinsic::aarch64_neon_sqshl:
9869 : case Intrinsic::aarch64_neon_uqshl:
9870 : case Intrinsic::aarch64_neon_sqshlu:
9871 : case Intrinsic::aarch64_neon_srshl:
9872 : case Intrinsic::aarch64_neon_urshl:
9873 : return tryCombineShiftImm(IID, N, DAG);
9874 : case Intrinsic::aarch64_crc32b:
9875 : case Intrinsic::aarch64_crc32cb:
9876 : return tryCombineCRC32(0xff, N, DAG);
9877 0 : case Intrinsic::aarch64_crc32h:
9878 0 : case Intrinsic::aarch64_crc32ch:
9879 0 : return tryCombineCRC32(0xffff, N, DAG);
9880 : }
9881 0 : return SDValue();
9882 0 : }
9883 0 :
9884 : static SDValue performExtendCombine(SDNode *N,
9885 : TargetLowering::DAGCombinerInfo &DCI,
9886 0 : SelectionDAG &DAG) {
9887 0 : // If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
9888 : // we can convert that DUP into another extract_high (of a bigger DUP), which
9889 : // helps the backend to decide that an sabdl2 would be useful, saving a real
9890 110 : // extract_high operation.
9891 110 : if (!DCI.isBeforeLegalizeOps() && N->getOpcode() == ISD::ZERO_EXTEND &&
9892 110 : N->getOperand(0).getOpcode() == ISD::INTRINSIC_WO_CHAIN) {
9893 : SDNode *ABDNode = N->getOperand(0).getNode();
9894 : unsigned IID = getIntrinsicID(ABDNode);
9895 110 : if (IID == Intrinsic::aarch64_neon_sabd ||
9896 : IID == Intrinsic::aarch64_neon_uabd) {
9897 : SDValue NewABD = tryCombineLongOpWithDup(IID, ABDNode, DCI, DAG);
9898 : if (!NewABD.getNode())
9899 51 : return SDValue();
9900 51 :
9901 51 : return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0),
9902 0 : NewABD);
9903 : }
9904 : }
9905 :
9906 2 : // This is effectively a custom type legalization for AArch64.
9907 : //
9908 57 : // Type legalization will split an extend of a small, legal, type to a larger
9909 : // illegal type by first splitting the destination type, often creating
9910 : // illegal source types, which then get legalized in isel-confusing ways,
9911 : // leading to really terrible codegen. E.g.,
9912 53 : // %result = v8i32 sext v8i8 %value
9913 0 : // becomes
9914 0 : // %losrc = extract_subreg %value, ...
9915 : // %hisrc = extract_subreg %value, ...
9916 : // %lo = v4i32 sext v4i8 %losrc
9917 : // %hi = v4i32 sext v4i8 %hisrc
9918 : // Things go rapidly downhill from there.
9919 10 : //
9920 : // For AArch64, the [sz]ext vector instructions can only go up one element
9921 : // size, so we can, e.g., extend from i8 to i16, but to go from i8 to i32
9922 10 : // take two instructions.
9923 14 : //
9924 : // This implies that the most efficient way to do the extend from v8i8
9925 : // to two v4i32 values is to first extend the v8i8 to v8i16, then do
9926 14 : // the normal splitting to happen for the v8i16->v8i32.
9927 14 :
9928 : // This is pre-legalization to catch some cases where the default
9929 : // type legalization will create ill-tempered code.
9930 14 : if (!DCI.isBeforeLegalizeOps())
9931 7 : return SDValue();
9932 :
9933 : // We're only interested in cleaning things up for non-legal vector types
9934 7 : // here. If both the source and destination are legal, things will just
9935 : // work naturally without any fiddling.
9936 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9937 53 : EVT ResVT = N->getValueType(0);
9938 : if (!ResVT.isVector() || TLI.isTypeLegal(ResVT))
9939 28 : return SDValue();
9940 28 : // If the vector type isn't a simple VT, it's beyond the scope of what
9941 25 : // we're worried about here. Let legalization do its thing and hope for
9942 : // the best.
9943 23 : SDValue Src = N->getOperand(0);
9944 23 : EVT SrcVT = Src->getValueType(0);
9945 : if (!ResVT.isSimple() || !SrcVT.isSimple())
9946 : return SDValue();
9947 2 :
9948 : // If the source VT is a 64-bit vector, we can play games and get the
9949 : // better results we want.
9950 : if (SrcVT.getSizeInBits() != 64)
9951 : return SDValue();
9952 :
9953 32 : unsigned SrcEltSize = SrcVT.getScalarSizeInBits();
9954 32 : unsigned ElementCount = SrcVT.getVectorNumElements();
9955 32 : SrcVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize * 2), ElementCount);
9956 24 : SDLoc DL(N);
9957 : Src = DAG.getNode(N->getOpcode(), DL, SrcVT, Src);
9958 :
9959 16 : // Now split the rest of the operation into two halves, each with a 64
9960 0 : // bit source.
9961 : EVT LoVT, HiVT;
9962 8 : SDValue Lo, Hi;
9963 16 : unsigned NumElements = ResVT.getVectorNumElements();
9964 : assert(!(NumElements & 1) && "Splitting vector, but not in half!");
9965 : LoVT = HiVT = EVT::getVectorVT(*DAG.getContext(),
9966 115 : ResVT.getVectorElementType(), NumElements / 2);
9967 :
9968 : EVT InNVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getVectorElementType(),
9969 : LoVT.getVectorNumElements());
9970 : Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
9971 : DAG.getConstant(0, DL, MVT::i64));
9972 115 : Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InNVT, Src,
9973 115 : DAG.getConstant(InNVT.getVectorNumElements(), DL, MVT::i64));
9974 : Lo = DAG.getNode(N->getOpcode(), DL, LoVT, Lo);
9975 : Hi = DAG.getNode(N->getOpcode(), DL, HiVT, Hi);
9976 0 :
9977 : // Now combine the parts back together so we still have a single result
9978 : // like the combiner expects.
9979 0 : return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
9980 : }
9981 0 :
9982 : static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St,
9983 : SDValue SplatVal, unsigned NumVecElts) {
9984 0 : unsigned OrigAlignment = St.getAlignment();
9985 : unsigned EltOffset = SplatVal.getValueType().getSizeInBits() / 8;
9986 0 :
9987 0 : // Create scalar stores. This is at least as good as the code sequence for a
9988 0 : // split unaligned store which is a dup.s, ext.b, and two stores.
9989 0 : // Most of the time the three stores should be replaced by store pair
9990 0 : // instructions (stp).
9991 0 : SDLoc DL(&St);
9992 0 : SDValue BasePtr = St.getBasePtr();
9993 0 : uint64_t BaseOffset = 0;
9994 0 :
9995 0 : const MachinePointerInfo &PtrInfo = St.getPointerInfo();
9996 0 : SDValue NewST1 =
9997 0 : DAG.getStore(St.getChain(), DL, SplatVal, BasePtr, PtrInfo,
9998 0 : OrigAlignment, St.getMemOperand()->getFlags());
9999 :
10000 0 : // As this in ISel, we will not merge this add which may degrade results.
10001 0 : if (BasePtr->getOpcode() == ISD::ADD &&
10002 : isa<ConstantSDNode>(BasePtr->getOperand(1))) {
10003 0 : BaseOffset = cast<ConstantSDNode>(BasePtr->getOperand(1))->getSExtValue();
10004 0 : BasePtr = BasePtr->getOperand(0);
10005 : }
10006 0 :
10007 0 : unsigned Offset = EltOffset;
10008 : while (--NumVecElts) {
10009 0 : unsigned Alignment = MinAlign(OrigAlignment, Offset);
10010 0 : SDValue OffsetPtr =
10011 0 : DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
10012 : DAG.getConstant(BaseOffset + Offset, DL, MVT::i64));
10013 : NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr,
10014 : PtrInfo.getWithOffset(Offset), Alignment,
10015 0 : St.getMemOperand()->getFlags());
10016 0 : Offset += EltOffset;
10017 : }
10018 : return NewST1;
10019 : }
10020 :
10021 0 : /// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR. The
10022 0 : /// load store optimizer pass will merge them to store pair stores. This should
10023 : /// be better than a movi to create the vector zero followed by a vector store
10024 0 : /// if the zero constant is not re-used, since one instructions and one register
10025 0 : /// live range will be removed.
10026 : ///
10027 0 : /// For example, the final generated code should be:
10028 : ///
10029 0 : /// stp xzr, xzr, [x0]
10030 : ///
10031 : /// instead of:
10032 3872 : ///
10033 : /// movi v0.2d, #0
10034 : /// str q0, [x0]
10035 : ///
10036 : static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
10037 : SDValue StVal = St.getValue();
10038 : EVT VT = StVal.getValueType();
10039 7744 :
10040 629 : // It is beneficial to scalarize a zero splat store for 2 or 3 i64 elements or
10041 : // 2, 3 or 4 i32 elements.
10042 : int NumVecElts = VT.getVectorNumElements();
10043 108 : if (!(((NumVecElts == 2 || NumVecElts == 3) &&
10044 54 : VT.getVectorElementType().getSizeInBits() == 64) ||
10045 52 : ((NumVecElts == 2 || NumVecElts == 3 || NumVecElts == 4) &&
10046 52 : VT.getVectorElementType().getSizeInBits() == 32)))
10047 14 : return SDValue();
10048 :
10049 38 : if (StVal.getOpcode() != ISD::BUILD_VECTOR)
10050 76 : return SDValue();
10051 :
10052 : // If the zero constant has more than one use then the vector store could be
10053 : // better since the constant mov will be amortized and stp q instructions
10054 : // should be able to be formed.
10055 : if (!StVal.hasOneUse())
10056 : return SDValue();
10057 :
10058 : // If the immediate offset of the address operand is too large for the stp
10059 : // instruction, then bail out.
10060 : if (DAG.isBaseWithConstantOffset(St.getBasePtr())) {
10061 : int64_t Offset = St.getBasePtr()->getConstantOperandVal(1);
10062 : if (Offset < -512 || Offset > 504)
10063 : return SDValue();
10064 : }
10065 :
10066 : for (int I = 0; I < NumVecElts; ++I) {
10067 : SDValue EltVal = StVal.getOperand(I);
10068 : if (!isNullConstant(EltVal) && !isNullFPConstant(EltVal))
10069 : return SDValue();
10070 : }
10071 :
10072 : // Use a CopyFromReg WZR/XZR here to prevent
10073 : // DAGCombiner::MergeConsecutiveStores from undoing this transformation.
10074 : SDLoc DL(&St);
10075 : unsigned ZeroReg;
10076 : EVT ZeroVT;
10077 : if (VT.getVectorElementType().getSizeInBits() == 32) {
10078 3820 : ZeroReg = AArch64::WZR;
10079 1127 : ZeroVT = MVT::i32;
10080 : } else {
10081 : ZeroReg = AArch64::XZR;
10082 : ZeroVT = MVT::i64;
10083 : }
10084 2693 : SDValue SplatVal =
10085 5386 : DAG.getCopyFromReg(DAG.getEntryNode(), DL, ZeroReg, ZeroVT);
10086 2693 : return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
10087 2635 : }
10088 :
10089 : /// Replace a splat of a scalar to a vector store by scalar stores of the scalar
10090 : /// value. The load store optimizer pass will merge them to store pair stores.
10091 58 : /// This has better performance than a splat of the scalar followed by a split
10092 58 : /// vector store. Even if the stores are not merged it is four stores vs a dup,
10093 58 : /// followed by an ext.b and two stores.
10094 1 : static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) {
10095 : SDValue StVal = St.getValue();
10096 : EVT VT = StVal.getValueType();
10097 :
10098 57 : // Don't replace floating point stores, they possibly won't be transformed to
10099 50 : // stp because of the store pair suppress pass.
10100 : if (VT.isFloatingPoint())
10101 : return SDValue();
10102 :
10103 7 : // We can express a splat as store pair(s) for 2 or 4 elements.
10104 : unsigned NumVecElts = VT.getVectorNumElements();
10105 14 : if (NumVecElts != 4 && NumVecElts != 2)
10106 : return SDValue();
10107 :
10108 : // Check that this is a splat.
10109 7 : // Make sure that each of the relevant vector element locations are inserted
10110 : // to, i.e. 0 and 1 for v2i64 and 0, 1, 2, 3 for v4i32.
10111 : std::bitset<4> IndexNotInserted((1 << NumVecElts) - 1);
10112 : SDValue SplatVal;
10113 14 : for (unsigned I = 0; I < NumVecElts; ++I) {
10114 7 : // Check for insert vector elements.
10115 : if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT)
10116 7 : return SDValue();
10117 7 :
10118 7 : // Check that same value is inserted at each vector element.
10119 7 : if (I == 0)
10120 7 : SplatVal = StVal.getOperand(1);
10121 7 : else if (StVal.getOperand(1) != SplatVal)
10122 14 : return SDValue();
10123 14 :
10124 : // Check insert element index.
10125 : ConstantSDNode *CIndex = dyn_cast<ConstantSDNode>(StVal.getOperand(2));
10126 : if (!CIndex)
10127 7 : return SDValue();
10128 : uint64_t IndexVal = CIndex->getZExtValue();
10129 : if (IndexVal >= NumVecElts)
10130 43 : return SDValue();
10131 : IndexNotInserted.reset(IndexVal);
10132 43 :
10133 86 : StVal = StVal.getOperand(0);
10134 : }
10135 : // Check that all vector element locations were inserted to.
10136 : if (IndexNotInserted.any())
10137 : return SDValue();
10138 :
10139 : return splitStoreSplat(DAG, St, SplatVal, NumVecElts);
10140 43 : }
10141 :
10142 : static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
10143 43 : SelectionDAG &DAG,
10144 : const AArch64Subtarget *Subtarget) {
10145 :
10146 86 : StoreSDNode *S = cast<StoreSDNode>(N);
10147 : if (S->isVolatile() || S->isIndexed())
10148 : return SDValue();
10149 43 :
10150 6 : SDValue StVal = S->getValue();
10151 5 : EVT VT = StVal.getValueType();
10152 5 : if (!VT.isVector())
10153 : return SDValue();
10154 :
10155 : // If we get a splat of zeros, convert this vector store to a store of
10156 108 : // scalars. They will be merged into store pairs of xzr thereby removing one
10157 130 : // instruction and one register.
10158 : if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S))
10159 : return ReplacedZeroSplat;
10160 65 :
10161 65 : // FIXME: The logic for deciding if an unaligned store should be split should
10162 : // be included in TLI.allowsMisalignedMemoryAccesses(), and there should be
10163 65 : // a call to that function here.
10164 65 :
10165 : if (!Subtarget->isMisaligned128StoreSlow())
10166 43 : return SDValue();
10167 :
10168 : // Don't split at -Oz.
10169 : if (DAG.getMachineFunction().getFunction().optForMinSize())
10170 : return SDValue();
10171 :
10172 : // Don't split v2i64 vectors. Memcpy lowering produces those and splitting
10173 : // those up regresses performance on micro-benchmarks and olden/bh.
10174 : if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64)
10175 : return SDValue();
10176 :
10177 : // Split unaligned 16B stores. They are terrible for performance.
10178 : // Don't split stores with alignment of 1 or 2. Code that uses clang vector
10179 : // extensions can use this to mark that it does not want splitting to happen
10180 : // (by underspecifying alignment to be 1 or 2). Furthermore, the chance of
10181 : // eliminating alignment hazards is only 1 in 8 for alignment of 2.
10182 : if (VT.getSizeInBits() != 128 || S->getAlignment() >= 16 ||
10183 : S->getAlignment() <= 2)
10184 3041 : return SDValue();
10185 3041 :
10186 3041 : // If we get a splat of a scalar convert this vector store to a store of
10187 : // scalars. They will be merged into store pairs thereby removing two
10188 : // instructions.
10189 : if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S))
10190 3041 : return ReplacedSplat;
10191 3321 :
10192 1458 : SDLoc DL(S);
10193 789 : unsigned NumElts = VT.getVectorNumElements() / 2;
10194 1967 : // Split VT into two.
10195 1280 : EVT HalfVT =
10196 : EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), NumElts);
10197 1761 : SDValue SubVector0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
10198 1417 : DAG.getConstant(0, DL, MVT::i64));
10199 : SDValue SubVector1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, StVal,
10200 : DAG.getConstant(NumElts, DL, MVT::i64));
10201 : SDValue BasePtr = S->getBasePtr();
10202 : SDValue NewST1 =
10203 344 : DAG.getStore(S->getChain(), DL, SubVector0, BasePtr, S->getPointerInfo(),
10204 281 : S->getAlignment(), S->getMemOperand()->getFlags());
10205 : SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr,
10206 : DAG.getConstant(8, DL, MVT::i64));
10207 : return DAG.getStore(NewST1.getValue(0), DL, SubVector1, OffsetPtr,
10208 63 : S->getPointerInfo(), S->getAlignment(),
10209 7 : S->getMemOperand()->getFlags());
10210 7 : }
10211 2 :
10212 : /// Target-specific DAG combine function for post-increment LD1 (lane) and
10213 : /// post-increment LD1R.
10214 169 : static SDValue performPostLD1Combine(SDNode *N,
10215 126 : TargetLowering::DAGCombinerInfo &DCI,
10216 126 : bool IsLaneOp) {
10217 18 : if (DCI.isBeforeLegalizeOps())
10218 : return SDValue();
10219 :
10220 : SelectionDAG &DAG = DCI.DAG;
10221 : EVT VT = N->getValueType(0);
10222 :
10223 : unsigned LoadIdx = IsLaneOp ? 1 : 0;
10224 43 : SDNode *LD = N->getOperand(LoadIdx).getNode();
10225 43 : // If it is not LOAD, can not do such combine.
10226 : if (LD->getOpcode() != ISD::LOAD)
10227 14 : return SDValue();
10228 :
10229 : // The vector lane must be a constant in the LD1LANE opcode.
10230 29 : SDValue Lane;
10231 : if (IsLaneOp) {
10232 : Lane = N->getOperand(2);
10233 43 : auto *LaneC = dyn_cast<ConstantSDNode>(Lane);
10234 43 : if (!LaneC || LaneC->getZExtValue() >= VT.getVectorNumElements())
10235 : return SDValue();
10236 : }
10237 :
10238 : LoadSDNode *LoadSDN = cast<LoadSDNode>(LD);
10239 : EVT MemVT = LoadSDN->getMemoryVT();
10240 : // Check if memory operand is the same type as the vector element.
10241 : if (MemVT != VT.getVectorElementType())
10242 5 : return SDValue();
10243 5 :
10244 5 : // Check if there are other uses. If so, do not combine as it will introduce
10245 : // an extra load.
10246 : for (SDNode::use_iterator UI = LD->use_begin(), UE = LD->use_end(); UI != UE;
10247 : ++UI) {
10248 5 : if (UI.getUse().getResNo() == 1) // Ignore uses of the chain result.
10249 4 : continue;
10250 : if (*UI != N)
10251 : return SDValue();
10252 : }
10253 1 :
10254 0 : SDValue Addr = LD->getOperand(1);
10255 : SDValue Vector = N->getOperand(0);
10256 : // Search for a use of the address operand that is an increment.
10257 : for (SDNode::use_iterator UI = Addr.getNode()->use_begin(), UE =
10258 : Addr.getNode()->use_end(); UI != UE; ++UI) {
10259 1 : SDNode *User = *UI;
10260 : if (User->getOpcode() != ISD::ADD
10261 1 : || UI.getUse().getResNo() != Addr.getResNo())
10262 : continue;
10263 1 :
10264 1 : // If the increment is a constant, it must match the memory ref size.
10265 : SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
10266 : if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
10267 0 : uint32_t IncVal = CInc->getZExtValue();
10268 0 : unsigned NumBytes = VT.getScalarSizeInBits() / 8;
10269 : if (IncVal != NumBytes)
10270 0 : continue;
10271 : Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
10272 : }
10273 :
10274 : // To avoid cycle construction make sure that neither the load nor the add
10275 0 : // are predecessors to each other or the Vector.
10276 0 : SmallPtrSet<const SDNode *, 32> Visited;
10277 0 : SmallVector<const SDNode *, 16> Worklist;
10278 0 : Visited.insert(N);
10279 0 : Worklist.push_back(User);
10280 : Worklist.push_back(LD);
10281 0 : Worklist.push_back(Vector.getNode());
10282 : if (SDNode::hasPredecessorHelper(LD, Visited, Worklist) ||
10283 : SDNode::hasPredecessorHelper(User, Visited, Worklist))
10284 0 : continue;
10285 0 :
10286 : SmallVector<SDValue, 8> Ops;
10287 0 : Ops.push_back(LD->getOperand(0)); // Chain
10288 : if (IsLaneOp) {
10289 : Ops.push_back(Vector); // The vector to be inserted
10290 0 : Ops.push_back(Lane); // The lane to be inserted in the vector
10291 : }
10292 : Ops.push_back(Addr);
10293 : Ops.push_back(Inc);
10294 :
10295 0 : EVT Tys[3] = { VT, MVT::i64, MVT::Other };
10296 0 : SDVTList SDTys = DAG.getVTList(Tys);
10297 : unsigned NewOp = IsLaneOp ? AArch64ISD::LD1LANEpost : AArch64ISD::LD1DUPpost;
10298 0 : SDValue UpdN = DAG.getMemIntrinsicNode(NewOp, SDLoc(N), SDTys, Ops,
10299 0 : MemVT,
10300 0 : LoadSDN->getMemOperand());
10301 0 :
10302 : // Update the uses.
10303 : SDValue NewResults[] = {
10304 : SDValue(LD, 0), // The result of load
10305 : SDValue(UpdN.getNode(), 2) // Chain
10306 0 : };
10307 0 : DCI.CombineTo(LD, NewResults);
10308 : DCI.CombineTo(N, SDValue(UpdN.getNode(), 0)); // Dup/Inserted Result
10309 : DCI.CombineTo(User, SDValue(UpdN.getNode(), 1)); // Write back register
10310 :
10311 : break;
10312 : }
10313 0 : return SDValue();
10314 0 : }
10315 :
10316 : /// Simplify ``Addr`` given that the top byte of it is ignored by HW during
10317 0 : /// address translation.
10318 0 : static bool performTBISimplification(SDValue Addr,
10319 : TargetLowering::DAGCombinerInfo &DCI,
10320 : SelectionDAG &DAG) {
10321 : APInt DemandedMask = APInt::getLowBitsSet(64, 56);
10322 0 : KnownBits Known;
10323 0 : TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
10324 : !DCI.isBeforeLegalizeOps());
10325 : const TargetLowering &TLI = DAG.getTargetLoweringInfo();
10326 : if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) {
10327 : DCI.CommitTargetLoweringOpt(TLO);
10328 : return true;
10329 : }
10330 0 : return false;
10331 0 : }
10332 0 :
10333 : static SDValue performSTORECombine(SDNode *N,
10334 : TargetLowering::DAGCombinerInfo &DCI,
10335 : SelectionDAG &DAG,
10336 : const AArch64Subtarget *Subtarget) {
10337 0 : if (SDValue Split = splitStores(N, DCI, DAG, Subtarget))
10338 0 : return Split;
10339 :
10340 : if (Subtarget->supportsAddressTopByteIgnored() &&
10341 0 : performTBISimplification(N->getOperand(2), DCI, DAG))
10342 : return SDValue(N, 0);
10343 :
10344 0 : return SDValue();
10345 : }
10346 0 :
10347 :
10348 0 : /// Target-specific DAG combine function for NEON load/store intrinsics
10349 0 : /// to merge base address updates.
10350 : static SDValue performNEONPostLDSTCombine(SDNode *N,
10351 0 : TargetLowering::DAGCombinerInfo &DCI,
10352 0 : SelectionDAG &DAG) {
10353 : if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
10354 0 : return SDValue();
10355 :
10356 0 : unsigned AddrOpIdx = N->getNumOperands() - 1;
10357 0 : SDValue Addr = N->getOperand(AddrOpIdx);
10358 :
10359 : // Search for a use of the address operand that is an increment.
10360 : for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
10361 : UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
10362 1853 : SDNode *User = *UI;
10363 : if (User->getOpcode() != ISD::ADD ||
10364 : UI.getUse().getResNo() != Addr.getResNo())
10365 3706 : continue;
10366 690 :
10367 : // Check that the add is independent of the load/store. Otherwise, folding
10368 1163 : // it would create a cycle.
10369 1163 : SmallPtrSet<const SDNode *, 32> Visited;
10370 : SmallVector<const SDNode *, 16> Worklist;
10371 1163 : Visited.insert(Addr.getNode());
10372 1163 : Worklist.push_back(N);
10373 : Worklist.push_back(User);
10374 1163 : if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
10375 1048 : SDNode::hasPredecessorHelper(User, Visited, Worklist))
10376 : continue;
10377 :
10378 115 : // Find the new opcode for the updating load/store.
10379 115 : bool IsStore = false;
10380 68 : bool IsLaneOp = false;
10381 : bool IsDupOp = false;
10382 132 : unsigned NewOpc = 0;
10383 2 : unsigned NumVecs = 0;
10384 : unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
10385 : switch (IntNo) {
10386 : default: llvm_unreachable("unexpected intrinsic for Neon base update");
10387 113 : case Intrinsic::aarch64_neon_ld2: NewOpc = AArch64ISD::LD2post;
10388 : NumVecs = 2; break;
10389 113 : case Intrinsic::aarch64_neon_ld3: NewOpc = AArch64ISD::LD3post;
10390 6 : NumVecs = 3; break;
10391 : case Intrinsic::aarch64_neon_ld4: NewOpc = AArch64ISD::LD4post;
10392 : NumVecs = 4; break;
10393 : case Intrinsic::aarch64_neon_st2: NewOpc = AArch64ISD::ST2post;
10394 274 : NumVecs = 2; IsStore = true; break;
10395 : case Intrinsic::aarch64_neon_st3: NewOpc = AArch64ISD::ST3post;
10396 167 : NumVecs = 3; IsStore = true; break;
10397 : case Intrinsic::aarch64_neon_st4: NewOpc = AArch64ISD::ST4post;
10398 107 : NumVecs = 4; IsStore = true; break;
10399 0 : case Intrinsic::aarch64_neon_ld1x2: NewOpc = AArch64ISD::LD1x2post;
10400 : NumVecs = 2; break;
10401 : case Intrinsic::aarch64_neon_ld1x3: NewOpc = AArch64ISD::LD1x3post;
10402 107 : NumVecs = 3; break;
10403 107 : case Intrinsic::aarch64_neon_ld1x4: NewOpc = AArch64ISD::LD1x4post;
10404 : NumVecs = 4; break;
10405 107 : case Intrinsic::aarch64_neon_st1x2: NewOpc = AArch64ISD::ST1x2post;
10406 192 : NumVecs = 2; IsStore = true; break;
10407 : case Intrinsic::aarch64_neon_st1x3: NewOpc = AArch64ISD::ST1x3post;
10408 127 : NumVecs = 3; IsStore = true; break;
10409 127 : case Intrinsic::aarch64_neon_st1x4: NewOpc = AArch64ISD::ST1x4post;
10410 85 : NumVecs = 4; IsStore = true; break;
10411 : case Intrinsic::aarch64_neon_ld2r: NewOpc = AArch64ISD::LD2DUPpost;
10412 : NumVecs = 2; IsDupOp = true; break;
10413 44 : case Intrinsic::aarch64_neon_ld3r: NewOpc = AArch64ISD::LD3DUPpost;
10414 44 : NumVecs = 3; IsDupOp = true; break;
10415 44 : case Intrinsic::aarch64_neon_ld4r: NewOpc = AArch64ISD::LD4DUPpost;
10416 22 : NumVecs = 4; IsDupOp = true; break;
10417 22 : case Intrinsic::aarch64_neon_ld2lane: NewOpc = AArch64ISD::LD2LANEpost;
10418 : NumVecs = 2; IsLaneOp = true; break;
10419 21 : case Intrinsic::aarch64_neon_ld3lane: NewOpc = AArch64ISD::LD3LANEpost;
10420 : NumVecs = 3; IsLaneOp = true; break;
10421 : case Intrinsic::aarch64_neon_ld4lane: NewOpc = AArch64ISD::LD4LANEpost;
10422 : NumVecs = 4; IsLaneOp = true; break;
10423 : case Intrinsic::aarch64_neon_st2lane: NewOpc = AArch64ISD::ST2LANEpost;
10424 : NumVecs = 2; IsStore = true; IsLaneOp = true; break;
10425 : case Intrinsic::aarch64_neon_st3lane: NewOpc = AArch64ISD::ST3LANEpost;
10426 43 : NumVecs = 3; IsStore = true; IsLaneOp = true; break;
10427 43 : case Intrinsic::aarch64_neon_st4lane: NewOpc = AArch64ISD::ST4LANEpost;
10428 43 : NumVecs = 4; IsStore = true; IsLaneOp = true; break;
10429 43 : }
10430 85 :
10431 42 : EVT VecTy;
10432 : if (IsStore)
10433 : VecTy = N->getOperand(2).getValueType();
10434 : else
10435 84 : VecTy = N->getValueType(0);
10436 42 :
10437 22 : // If the increment is a constant, it must match the memory ref size.
10438 22 : SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
10439 : if (ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode())) {
10440 42 : uint32_t IncVal = CInc->getZExtValue();
10441 42 : unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
10442 : if (IsLaneOp || IsDupOp)
10443 42 : NumBytes /= VecTy.getVectorNumElements();
10444 42 : if (IncVal != NumBytes)
10445 42 : continue;
10446 42 : Inc = DAG.getRegister(AArch64::XZR, MVT::i64);
10447 : }
10448 42 : SmallVector<SDValue, 8> Ops;
10449 : Ops.push_back(N->getOperand(0)); // Incoming chain
10450 : // Load lane and store have vector list as input.
10451 : if (IsLaneOp || IsStore)
10452 : for (unsigned i = 2; i < AddrOpIdx; ++i)
10453 : Ops.push_back(N->getOperand(i));
10454 : Ops.push_back(Addr); // Base register
10455 42 : Ops.push_back(Inc);
10456 42 :
10457 42 : // Return Types.
10458 : EVT Tys[6];
10459 : unsigned NumResultVecs = (IsStore ? 0 : NumVecs);
10460 : unsigned n;
10461 107 : for (n = 0; n < NumResultVecs; ++n)
10462 : Tys[n] = VecTy;
10463 : Tys[n++] = MVT::i64; // Type of write back register
10464 : Tys[n] = MVT::Other; // Type of the chain
10465 : SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs + 2));
10466 25 :
10467 : MemIntrinsicSDNode *MemInt = cast<MemIntrinsicSDNode>(N);
10468 : SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, SDLoc(N), SDTys, Ops,
10469 : MemInt->getMemoryVT(),
10470 25 : MemInt->getMemOperand());
10471 25 :
10472 50 : // Update the uses.
10473 25 : std::vector<SDValue> NewResults;
10474 25 : for (unsigned i = 0; i < NumResultVecs; ++i) {
10475 9 : NewResults.push_back(SDValue(UpdN.getNode(), i));
10476 9 : }
10477 : NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1));
10478 : DCI.CombineTo(N, NewResults);
10479 : DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
10480 :
10481 16665 : break;
10482 : }
10483 : return SDValue();
10484 : }
10485 16665 :
10486 48 : // Checks to see if the value is the prescribed width and returns information
10487 : // about its extension mode.
10488 16620 : static
10489 6 : bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType) {
10490 1 : ExtType = ISD::NON_EXTLOAD;
10491 : switch(V.getNode()->getOpcode()) {
10492 16616 : default:
10493 : return false;
10494 : case ISD::LOAD: {
10495 : LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
10496 : if ((LoadNode->getMemoryVT() == MVT::i8 && width == 8)
10497 : || (LoadNode->getMemoryVT() == MVT::i16 && width == 16)) {
10498 1689 : ExtType = LoadNode->getExtensionType();
10499 : return true;
10500 : }
10501 1689 : return false;
10502 842 : }
10503 : case ISD::AssertSext: {
10504 847 : VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
10505 847 : if ((TypeNode->getVT() == MVT::i8 && width == 8)
10506 : || (TypeNode->getVT() == MVT::i16 && width == 16)) {
10507 : ExtType = ISD::SEXTLOAD;
10508 847 : return true;
10509 1247 : }
10510 : return false;
10511 904 : }
10512 504 : case ISD::AssertZext: {
10513 400 : VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
10514 : if ((TypeNode->getVT() == MVT::i8 && width == 8)
10515 : || (TypeNode->getVT() == MVT::i16 && width == 16)) {
10516 : ExtType = ISD::ZEXTLOAD;
10517 : return true;
10518 : }
10519 504 : return false;
10520 504 : }
10521 504 : case ISD::Constant:
10522 1008 : case ISD::TargetConstant: {
10523 504 : return std::abs(cast<ConstantSDNode>(V.getNode())->getSExtValue()) <
10524 0 : 1LL << (width - 1);
10525 : }
10526 : }
10527 :
10528 : return true;
10529 : }
10530 :
10531 : // This function does a whole lot of voodoo to determine if the tests are
10532 1008 : // equivalent without and with a mask. Essentially what happens is that given a
10533 504 : // DAG resembling:
10534 0 : //
10535 : // +-------------+ +-------------+ +-------------+ +-------------+
10536 : // | Input | | AddConstant | | CompConstant| | CC |
10537 24 : // +-------------+ +-------------+ +-------------+ +-------------+
10538 24 : // | | | |
10539 24 : // V V | +----------+
10540 24 : // +-------------+ +----+ | |
10541 24 : // | ADD | |0xff| | |
10542 24 : // +-------------+ +----+ | |
10543 24 : // | | | |
10544 24 : // V V | |
10545 24 : // +-------------+ | |
10546 24 : // | AND | | |
10547 24 : // +-------------+ | |
10548 24 : // | | |
10549 24 : // +-----+ | |
10550 24 : // | | |
10551 24 : // V V V
10552 24 : // +-------------+
10553 24 : // | CMP |
10554 24 : // +-------------+
10555 24 : //
10556 24 : // The AND node may be safely removed for some combinations of inputs. In
10557 24 : // particular we need to take into account the extension type of the Input,
10558 24 : // the exact values of AddConstant, CompConstant, and CC, along with the nominal
10559 24 : // width of the input (this can work for any width inputs, the above graph is
10560 24 : // specific to 8 bits.
10561 24 : //
10562 24 : // The specific equations were worked out by generating output tables for each
10563 24 : // AArch64CC value in terms of and AddConstant (w1), CompConstant(w2). The
10564 24 : // problem was simplified by working with 4 bit inputs, which means we only
10565 24 : // needed to reason about 24 distinct bit patterns: 8 patterns unique to zero
10566 24 : // extension (8,15), 8 patterns unique to sign extensions (-8,-1), and 8
10567 24 : // patterns present in both extensions (0,7). For every distinct set of
10568 24 : // AddConstant and CompConstants bit patterns we can consider the masked and
10569 24 : // unmasked versions to be equivalent if the result of this function is true for
10570 24 : // all 16 distinct bit patterns of for the current extension type of Input (w0).
10571 24 : //
10572 24 : // sub w8, w0, w1
10573 24 : // and w10, w8, #0x0f
10574 24 : // cmp w8, w2
10575 24 : // cset w9, AArch64CC
10576 24 : // cmp w10, w2
10577 : // cset w11, AArch64CC
10578 : // cmp w9, w11
10579 504 : // cset w0, eq
10580 504 : // ret
10581 432 : //
10582 : // Since the above function shows when the outputs are equivalent it defines
10583 576 : // when it is safe to remove the AND. Unfortunately it only runs on AArch64 and
10584 : // would be expensive to run during compiles. The equations below were written
10585 : // in a test harness that confirmed they gave equivalent outputs to the above
10586 504 : // for all inputs function, so they can be used determine if the removal is
10587 504 : // legal instead.
10588 252 : //
10589 252 : // isEquivalentMaskless() is the code for testing if the AND can be removed
10590 252 : // factored out of the DAG recognition as the DAG can take several forms.
10591 108 :
10592 252 : static bool isEquivalentMaskless(unsigned CC, unsigned width,
10593 : ISD::LoadExtType ExtType, int AddConstant,
10594 252 : int CompConstant) {
10595 : // By being careful about our equations and only writing the in term
10596 : // symbolic values and well known constants (0, 1, -1, MaxUInt) we can
10597 1008 : // make them generally applicable to all bit widths.
10598 : int MaxUInt = (1 << width);
10599 504 :
10600 1296 : // For the purposes of these comparisons sign extending the type is
10601 2016 : // equivalent to zero extending the add and displacing it by half the integer
10602 504 : // width. Provided we are careful and make sure our equations are valid over
10603 504 : // the whole range we can just adjust the input and avoid writing equations
10604 : // for sign extended inputs.
10605 : if (ExtType == ISD::SEXTLOAD)
10606 504 : AddConstant -= (1 << (width-1));
10607 504 :
10608 : switch(CC) {
10609 1368 : case AArch64CC::LE:
10610 864 : case AArch64CC::GT:
10611 504 : if ((AddConstant == 0) ||
10612 504 : (CompConstant == MaxUInt - 1 && AddConstant < 0) ||
10613 504 : (AddConstant >= 0 && CompConstant < 0) ||
10614 : (AddConstant <= 0 && CompConstant <= 0 && CompConstant < AddConstant))
10615 : return true;
10616 504 : break;
10617 : case AArch64CC::LT:
10618 504 : case AArch64CC::GE:
10619 : if ((AddConstant == 0) ||
10620 : (AddConstant >= 0 && CompConstant <= 0) ||
10621 : (AddConstant <= 0 && CompConstant <= 0 && CompConstant <= AddConstant))
10622 1368 : return true;
10623 864 : break;
10624 : case AArch64CC::HI:
10625 504 : case AArch64CC::LS:
10626 504 : if ((AddConstant >= 0 && CompConstant < 0) ||
10627 504 : (AddConstant <= 0 && CompConstant >= -1 &&
10628 : CompConstant < AddConstant + MaxUInt))
10629 : return true;
10630 : break;
10631 847 : case AArch64CC::PL:
10632 : case AArch64CC::MI:
10633 : if ((AddConstant == 0) ||
10634 : (AddConstant > 0 && CompConstant <= 0) ||
10635 : (AddConstant < 0 && CompConstant <= AddConstant))
10636 : return true;
10637 0 : break;
10638 0 : case AArch64CC::LO:
10639 0 : case AArch64CC::HS:
10640 : if ((AddConstant >= 0 && CompConstant <= 0) ||
10641 : (AddConstant <= 0 && CompConstant >= 0 &&
10642 0 : CompConstant <= AddConstant + MaxUInt))
10643 : return true;
10644 0 : break;
10645 0 : case AArch64CC::EQ:
10646 0 : case AArch64CC::NE:
10647 0 : if ((AddConstant > 0 && CompConstant < 0) ||
10648 : (AddConstant < 0 && CompConstant >= 0 &&
10649 : CompConstant < AddConstant + MaxUInt) ||
10650 : (AddConstant >= 0 && CompConstant >= 0 &&
10651 0 : CompConstant >= AddConstant) ||
10652 0 : (AddConstant <= 0 && CompConstant < 0 && CompConstant < AddConstant))
10653 0 : return true;
10654 0 : break;
10655 0 : case AArch64CC::VS:
10656 0 : case AArch64CC::VC:
10657 : case AArch64CC::AL:
10658 : case AArch64CC::NV:
10659 : return true;
10660 0 : case AArch64CC::Invalid:
10661 0 : break;
10662 0 : }
10663 0 :
10664 0 : return false;
10665 0 : }
10666 :
10667 : static
10668 : SDValue performCONDCombine(SDNode *N,
10669 0 : TargetLowering::DAGCombinerInfo &DCI,
10670 : SelectionDAG &DAG, unsigned CCIndex,
10671 0 : unsigned CmpIndex) {
10672 0 : unsigned CC = cast<ConstantSDNode>(N->getOperand(CCIndex))->getSExtValue();
10673 : SDNode *SubsNode = N->getOperand(CmpIndex).getNode();
10674 : unsigned CondOpcode = SubsNode->getOpcode();
10675 :
10676 : if (CondOpcode != AArch64ISD::SUBS)
10677 : return SDValue();
10678 :
10679 : // There is a SUBS feeding this condition. Is it fed by a mask we can
10680 : // use?
10681 :
10682 : SDNode *AndNode = SubsNode->getOperand(0).getNode();
10683 : unsigned MaskBits = 0;
10684 :
10685 : if (AndNode->getOpcode() != ISD::AND)
10686 : return SDValue();
10687 :
10688 : if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndNode->getOperand(1))) {
10689 : uint32_t CNV = CN->getZExtValue();
10690 : if (CNV == 255)
10691 : MaskBits = 8;
10692 : else if (CNV == 65535)
10693 : MaskBits = 16;
10694 : }
10695 :
10696 : if (!MaskBits)
10697 : return SDValue();
10698 :
10699 : SDValue AddValue = AndNode->getOperand(0);
10700 :
10701 : if (AddValue.getOpcode() != ISD::ADD)
10702 : return SDValue();
10703 :
10704 : // The basic dag structure is correct, grab the inputs and validate them.
10705 :
10706 : SDValue AddInputValue1 = AddValue.getNode()->getOperand(0);
10707 : SDValue AddInputValue2 = AddValue.getNode()->getOperand(1);
10708 : SDValue SubsInputValue = SubsNode->getOperand(1);
10709 :
10710 : // The mask is present and the provenance of all the values is a smaller type,
10711 : // lets see if the mask is superfluous.
10712 :
10713 : if (!isa<ConstantSDNode>(AddInputValue2.getNode()) ||
10714 : !isa<ConstantSDNode>(SubsInputValue.getNode()))
10715 : return SDValue();
10716 :
10717 : ISD::LoadExtType ExtType;
10718 :
10719 : if (!checkValueWidth(SubsInputValue, MaskBits, ExtType) ||
10720 : !checkValueWidth(AddInputValue2, MaskBits, ExtType) ||
10721 : !checkValueWidth(AddInputValue1, MaskBits, ExtType) )
10722 : return SDValue();
10723 :
10724 : if(!isEquivalentMaskless(CC, MaskBits, ExtType,
10725 : cast<ConstantSDNode>(AddInputValue2.getNode())->getSExtValue(),
10726 : cast<ConstantSDNode>(SubsInputValue.getNode())->getSExtValue()))
10727 : return SDValue();
10728 :
10729 : // The AND is not necessary, remove it.
10730 :
10731 : SDVTList VTs = DAG.getVTList(SubsNode->getValueType(0),
10732 : SubsNode->getValueType(1));
10733 : SDValue Ops[] = { AddValue, SubsNode->getOperand(1) };
10734 :
10735 : SDValue NewValue = DAG.getNode(CondOpcode, SDLoc(SubsNode), VTs, Ops);
10736 : DAG.ReplaceAllUsesWith(SubsNode, NewValue.getNode());
10737 :
10738 : return SDValue(N, 0);
10739 : }
10740 3 :
10741 : // Optimize compare with zero and branch.
10742 : static SDValue performBRCONDCombine(SDNode *N,
10743 : TargetLowering::DAGCombinerInfo &DCI,
10744 : SelectionDAG &DAG) {
10745 : if (SDValue NV = performCONDCombine(N, DCI, DAG, 2, 3))
10746 3 : N = NV.getNode();
10747 : SDValue Chain = N->getOperand(0);
10748 : SDValue Dest = N->getOperand(1);
10749 : SDValue CCVal = N->getOperand(2);
10750 : SDValue Cmp = N->getOperand(3);
10751 :
10752 : assert(isa<ConstantSDNode>(CCVal) && "Expected a ConstantSDNode here!");
10753 3 : unsigned CC = cast<ConstantSDNode>(CCVal)->getZExtValue();
10754 0 : if (CC != AArch64CC::EQ && CC != AArch64CC::NE)
10755 : return SDValue();
10756 3 :
10757 0 : unsigned CmpOpc = Cmp.getOpcode();
10758 : if (CmpOpc != AArch64ISD::ADDS && CmpOpc != AArch64ISD::SUBS)
10759 0 : return SDValue();
10760 0 :
10761 0 : // Only attempt folding if there is only one use of the flag and no use of the
10762 0 : // value.
10763 : if (!Cmp->hasNUsesOfValue(0, 0) || !Cmp->hasNUsesOfValue(1, 1))
10764 : return SDValue();
10765 0 :
10766 : SDValue LHS = Cmp.getOperand(0);
10767 0 : SDValue RHS = Cmp.getOperand(1);
10768 0 :
10769 0 : assert(LHS.getValueType() == RHS.getValueType() &&
10770 : "Expected the value type to be the same for both operands!");
10771 : if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
10772 2 : return SDValue();
10773 :
10774 2 : if (isNullConstant(LHS))
10775 2 : std::swap(LHS, RHS);
10776 2 :
10777 : if (!isNullConstant(RHS))
10778 : return SDValue();
10779 0 :
10780 : if (LHS.getOpcode() == ISD::SHL || LHS.getOpcode() == ISD::SRA ||
10781 0 : LHS.getOpcode() == ISD::SRL)
10782 0 : return SDValue();
10783 0 :
10784 : // Fold the compare into the branch instruction.
10785 : SDValue BR;
10786 1 : if (CC == AArch64CC::EQ)
10787 : BR = DAG.getNode(AArch64ISD::CBZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
10788 1 : else
10789 1 : BR = DAG.getNode(AArch64ISD::CBNZ, SDLoc(N), MVT::Other, Chain, LHS, Dest);
10790 1 :
10791 : // Do not add new nodes to DAG combiner worklist.
10792 : DCI.CombineTo(N, BR, false);
10793 0 :
10794 : return SDValue();
10795 0 : }
10796 0 :
10797 0 : // Optimize some simple tbz/tbnz cases. Returns the new operand and bit to test
10798 0 : // as well as whether the test should be inverted. This code is required to
10799 0 : // catch these cases (as opposed to standard dag combines) because
10800 0 : // AArch64ISD::TBZ is matched during legalization.
10801 : static SDValue getTestBitOperand(SDValue Op, unsigned &Bit, bool &Invert,
10802 : SelectionDAG &DAG) {
10803 :
10804 : if (!Op->hasOneUse())
10805 : return Op;
10806 :
10807 : // We don't handle undef/constant-fold cases below, as they should have
10808 : // already been taken care of (e.g. and of 0, test of undefined shifted bits,
10809 : // etc.)
10810 :
10811 : // (tbz (trunc x), b) -> (tbz x, b)
10812 0 : // This case is just here to enable more of the below cases to be caught.
10813 : if (Op->getOpcode() == ISD::TRUNCATE &&
10814 : Bit < Op->getValueType(0).getSizeInBits()) {
10815 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10816 0 : }
10817 :
10818 : if (Op->getNumOperands() != 2)
10819 : return Op;
10820 0 :
10821 0 : auto *C = dyn_cast<ConstantSDNode>(Op->getOperand(1));
10822 0 : if (!C)
10823 : return Op;
10824 0 :
10825 0 : switch (Op->getOpcode()) {
10826 : default:
10827 : return Op;
10828 :
10829 : // (tbz (and x, m), b) -> (tbz x, b)
10830 0 : case ISD::AND:
10831 : if ((C->getZExtValue() >> Bit) & 1)
10832 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10833 0 : return Op;
10834 0 :
10835 : // (tbz (shl x, c), b) -> (tbz x, b-c)
10836 0 : case ISD::SHL:
10837 0 : if (C->getZExtValue() <= Bit &&
10838 0 : (Bit - C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
10839 : Bit = Bit - C->getZExtValue();
10840 0 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10841 : }
10842 : return Op;
10843 :
10844 : // (tbz (sra x, c), b) -> (tbz x, b+c) or (tbz x, msb) if b+c is > # bits in x
10845 0 : case ISD::SRA:
10846 : Bit = Bit + C->getZExtValue();
10847 0 : if (Bit >= Op->getValueType(0).getSizeInBits())
10848 : Bit = Op->getValueType(0).getSizeInBits() - 1;
10849 0 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10850 0 :
10851 : // (tbz (srl x, c), b) -> (tbz x, b+c)
10852 : case ISD::SRL:
10853 : if ((Bit + C->getZExtValue()) < Op->getValueType(0).getSizeInBits()) {
10854 0 : Bit = Bit + C->getZExtValue();
10855 0 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10856 0 : }
10857 : return Op;
10858 :
10859 : // (tbz (xor x, -1), b) -> (tbnz x, b)
10860 : case ISD::XOR:
10861 : if ((C->getZExtValue() >> Bit) & 1)
10862 : Invert = !Invert;
10863 0 : return getTestBitOperand(Op->getOperand(0), Bit, Invert, DAG);
10864 : }
10865 : }
10866 :
10867 0 : // Optimize test single bit zero/non-zero and branch.
10868 0 : static SDValue performTBZCombine(SDNode *N,
10869 0 : TargetLowering::DAGCombinerInfo &DCI,
10870 0 : SelectionDAG &DAG) {
10871 : unsigned Bit = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
10872 0 : bool Invert = false;
10873 0 : SDValue TestSrc = N->getOperand(1);
10874 0 : SDValue NewTestSrc = getTestBitOperand(TestSrc, Bit, Invert, DAG);
10875 0 :
10876 : if (TestSrc == NewTestSrc)
10877 : return SDValue();
10878 :
10879 : unsigned NewOpc = N->getOpcode();
10880 0 : if (Invert) {
10881 0 : if (NewOpc == AArch64ISD::TBZ)
10882 : NewOpc = AArch64ISD::TBNZ;
10883 0 : else {
10884 0 : assert(NewOpc == AArch64ISD::TBNZ);
10885 : NewOpc = AArch64ISD::TBZ;
10886 0 : }
10887 : }
10888 :
10889 : SDLoc DL(N);
10890 493 : return DAG.getNode(NewOpc, DL, MVT::Other, N->getOperand(0), NewTestSrc,
10891 : DAG.getConstant(Bit, DL, MVT::i64), N->getOperand(3));
10892 : }
10893 493 :
10894 : // vselect (v1i1 setcc) ->
10895 493 : // vselect (v1iXX setcc) (XX is the size of the compared operand type)
10896 493 : // FIXME: Currently the type legalizer can't handle VSELECT having v1i1 as
10897 493 : // condition. If it can legalize "VSELECT v1i1" correctly, no need to combine
10898 493 : // such VSELECT.
10899 : static SDValue performVSelectCombine(SDNode *N, SelectionDAG &DAG) {
10900 : SDValue N0 = N->getOperand(0);
10901 493 : EVT CCVT = N0.getValueType();
10902 493 :
10903 269 : if (N0.getOpcode() != ISD::SETCC || CCVT.getVectorNumElements() != 1 ||
10904 : CCVT.getVectorElementType() != MVT::i1)
10905 : return SDValue();
10906 224 :
10907 23 : EVT ResVT = N->getValueType(0);
10908 : EVT CmpVT = N0.getOperand(0).getValueType();
10909 : // Only combine when the result type is of the same size as the compared
10910 : // operands.
10911 201 : if (ResVT.getSizeInBits() != CmpVT.getSizeInBits())
10912 1 : return SDValue();
10913 :
10914 200 : SDValue IfTrue = N->getOperand(1);
10915 200 : SDValue IfFalse = N->getOperand(2);
10916 : SDValue SetCC =
10917 : DAG.getSetCC(SDLoc(N), CmpVT.changeVectorElementTypeToInteger(),
10918 : N0.getOperand(0), N0.getOperand(1),
10919 : cast<CondCodeSDNode>(N0.getOperand(2))->get());
10920 0 : return DAG.getNode(ISD::VSELECT, SDLoc(N), ResVT, SetCC,
10921 : IfTrue, IfFalse);
10922 200 : }
10923 :
10924 : /// A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with
10925 200 : /// the compare-mask instructions rather than going via NZCV, even if LHS and
10926 198 : /// RHS are really scalar. This replaces any scalar setcc in the above pattern
10927 : /// with a vector one followed by a DUP shuffle on the result.
10928 2 : static SDValue performSelectCombine(SDNode *N,
10929 : TargetLowering::DAGCombinerInfo &DCI) {
10930 1 : SelectionDAG &DAG = DCI.DAG;
10931 : SDValue N0 = N->getOperand(0);
10932 : EVT ResVT = N->getValueType(0);
10933 1 :
10934 1 : if (N0.getOpcode() != ISD::SETCC)
10935 1 : return SDValue();
10936 :
10937 0 : // Make sure the SETCC result is either i1 (initial DAG), or i32, the lowered
10938 : // scalar SetCCResultType. We also don't expect vectors, because we assume
10939 : // that selects fed by vector SETCCs are canonicalized to VSELECT.
10940 1 : assert((N0.getValueType() == MVT::i1 || N0.getValueType() == MVT::i32) &&
10941 : "Scalar-SETCC feeding SELECT has unexpected result type!");
10942 1 :
10943 : // If NumMaskElts == 0, the comparison is larger than select result. The
10944 : // largest real NEON comparison is 64-bits per lane, which means the result is
10945 : // at most 32-bits and an illegal vector. Just bail out for now.
10946 : EVT SrcVT = N0.getOperand(0).getValueType();
10947 :
10948 : // Don't try to do this optimization when the setcc itself has i1 operands.
10949 186 : // There are no legal vectors of i1, so this would be pointless.
10950 : if (SrcVT == MVT::i1)
10951 : return SDValue();
10952 :
10953 43 : int NumMaskElts = ResVT.getSizeInBits() / SrcVT.getSizeInBits();
10954 : if (!ResVT.isVector() || NumMaskElts == 0)
10955 : return SDValue();
10956 :
10957 : SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumMaskElts);
10958 : EVT CCVT = SrcVT.changeVectorElementTypeToInteger();
10959 :
10960 : // Also bail out if the vector CCVT isn't the same size as ResVT.
10961 286 : // This can happen if the SETCC operand size doesn't divide the ResVT size
10962 4 : // (e.g., f64 vs v3f32).
10963 4 : if (CCVT.getSizeInBits() != ResVT.getSizeInBits())
10964 : return SDValue();
10965 :
10966 141 : // Make sure we didn't create illegal types, if we're not supposed to.
10967 5 : assert(DCI.isBeforeLegalize() ||
10968 : DAG.getTargetLoweringInfo().isTypeLegal(SrcVT));
10969 136 :
10970 : // First perform a vector comparison, where lane 0 is the one we're interested
10971 125 : // in.
10972 : SDLoc DL(N0);
10973 11 : SDValue LHS =
10974 7 : DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(0));
10975 7 : SDValue RHS =
10976 : DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, SrcVT, N0.getOperand(1));
10977 : SDValue SetCC = DAG.getNode(ISD::SETCC, DL, CCVT, LHS, RHS, N0.getOperand(2));
10978 0 :
10979 0 : // Now duplicate the comparison mask we want across all other lanes.
10980 0 : SmallVector<int, 8> DUPMask(CCVT.getVectorNumElements(), 0);
10981 0 : SDValue Mask = DAG.getVectorShuffle(CCVT, DL, SetCC, SetCC, DUPMask);
10982 : Mask = DAG.getNode(ISD::BITCAST, DL,
10983 : ResVT.changeVectorElementTypeToInteger(), Mask);
10984 1 :
10985 2 : return DAG.getSelect(DL, ResVT, Mask, N->getOperand(1), N->getOperand(2));
10986 2 : }
10987 1 :
10988 2 : /// Get rid of unnecessary NVCASTs (that don't change the type).
10989 : static SDValue performNVCASTCombine(SDNode *N) {
10990 0 : if (N->getValueType(0) == N->getOperand(0).getValueType())
10991 : return N->getOperand(0);
10992 :
10993 1 : return SDValue();
10994 1 : }
10995 2 :
10996 1 : // If all users of the globaladdr are of the form (globaladdr + constant), find
10997 2 : // the smallest constant, fold it into the globaladdr's offset and rewrite the
10998 : // globaladdr as (globaladdr + constant) - constant.
10999 : static SDValue performGlobalAddressCombine(SDNode *N, SelectionDAG &DAG,
11000 2 : const AArch64Subtarget *Subtarget,
11001 6 : const TargetMachine &TM) {
11002 2 : auto *GN = dyn_cast<GlobalAddressSDNode>(N);
11003 4 : if (!GN || Subtarget->ClassifyGlobalReference(GN->getGlobal(), TM) !=
11004 : AArch64II::MO_NO_FLAG)
11005 0 : return SDValue();
11006 :
11007 : uint64_t MinOffset = -1ull;
11008 0 : for (SDNode *N : GN->uses()) {
11009 0 : if (N->getOpcode() != ISD::ADD)
11010 0 : return SDValue();
11011 0 : auto *C = dyn_cast<ConstantSDNode>(N->getOperand(0));
11012 : if (!C)
11013 : C = dyn_cast<ConstantSDNode>(N->getOperand(1));
11014 : if (!C)
11015 : return SDValue();
11016 0 : MinOffset = std::min(MinOffset, C->getZExtValue());
11017 : }
11018 : uint64_t Offset = MinOffset + GN->getOffset();
11019 0 :
11020 0 : // Require that the new offset is larger than the existing one. Otherwise, we
11021 0 : // can end up oscillating between two possible DAGs, for example,
11022 0 : // (add (add globaladdr + 10, -1), 1) and (add globaladdr + 9, 1).
11023 : if (Offset <= uint64_t(GN->getOffset()))
11024 0 : return SDValue();
11025 0 :
11026 : // Check whether folding this offset is legal. It must not go out of bounds of
11027 0 : // the referenced object to avoid violating the code model, and must be
11028 0 : // smaller than 2^21 because this is the largest offset expressible in all
11029 0 : // object formats.
11030 : //
11031 : // This check also prevents us from folding negative offsets, which will end
11032 : // up being treated in the same way as large positive ones. They could also
11033 : // cause code model violations, and aren't really common enough to matter.
11034 : if (Offset >= (1 << 21))
11035 : return SDValue();
11036 :
11037 : const GlobalValue *GV = GN->getGlobal();
11038 0 : Type *T = GV->getValueType();
11039 0 : if (!T->isSized() ||
11040 : Offset > GV->getParent()->getDataLayout().getTypeAllocSize(T))
11041 : return SDValue();
11042 :
11043 : SDLoc DL(GN);
11044 : SDValue Result = DAG.getGlobalAddress(GV, DL, MVT::i64, Offset);
11045 : return DAG.getNode(ISD::SUB, DL, MVT::i64, Result,
11046 : DAG.getConstant(MinOffset, DL, MVT::i64));
11047 122 : }
11048 122 :
11049 122 : SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
11050 : DAGCombinerInfo &DCI) const {
11051 196 : SelectionDAG &DAG = DCI.DAG;
11052 17 : switch (N->getOpcode()) {
11053 116 : default:
11054 : LLVM_DEBUG(dbgs() << "Custom combining: skipping\n");
11055 12 : break;
11056 6 : case ISD::ADD:
11057 : case ISD::SUB:
11058 : return performAddSubLongCombine(N, DCI, DAG);
11059 6 : case ISD::XOR:
11060 0 : return performXorCombine(N, DAG, DCI, Subtarget);
11061 : case ISD::MUL:
11062 6 : return performMulCombine(N, DAG, DCI, Subtarget);
11063 6 : case ISD::SINT_TO_FP:
11064 : case ISD::UINT_TO_FP:
11065 6 : return performIntToFpCombine(N, DAG, Subtarget);
11066 : case ISD::FP_TO_SINT:
11067 18 : case ISD::FP_TO_UINT:
11068 6 : return performFpToIntCombine(N, DAG, DCI, Subtarget);
11069 12 : case ISD::FDIV:
11070 : return performFDivCombine(N, DAG, DCI, Subtarget);
11071 : case ISD::OR:
11072 : return performORCombine(N, DCI, Subtarget);
11073 : case ISD::SRL:
11074 : return performSRLCombine(N, DCI);
11075 : case ISD::INTRINSIC_WO_CHAIN:
11076 0 : return performIntrinsicCombine(N, DCI, Subtarget);
11077 : case ISD::ANY_EXTEND:
11078 0 : case ISD::ZERO_EXTEND:
11079 0 : case ISD::SIGN_EXTEND:
11080 0 : return performExtendCombine(N, DCI, DAG);
11081 : case ISD::BITCAST:
11082 0 : return performBitcastCombine(N, DCI, DAG);
11083 0 : case ISD::CONCAT_VECTORS:
11084 : return performConcatVectorsCombine(N, DCI, DAG);
11085 : case ISD::SELECT:
11086 : return performSelectCombine(N, DCI);
11087 : case ISD::VSELECT:
11088 : return performVSelectCombine(N, DCI.DAG);
11089 : case ISD::LOAD:
11090 : if (performTBISimplification(N->getOperand(1), DCI, DAG))
11091 : return SDValue(N, 0);
11092 : break;
11093 : case ISD::STORE:
11094 0 : return performSTORECombine(N, DCI, DAG, Subtarget);
11095 : case AArch64ISD::BRCOND:
11096 : return performBRCONDCombine(N, DCI, DAG);
11097 : case AArch64ISD::TBNZ:
11098 0 : case AArch64ISD::TBZ:
11099 0 : return performTBZCombine(N, DCI, DAG);
11100 : case AArch64ISD::CSEL:
11101 0 : return performCONDCombine(N, DCI, DAG, 2, 3);
11102 0 : case AArch64ISD::DUP:
11103 0 : return performPostLD1Combine(N, DCI, false);
11104 : case AArch64ISD::NVCAST:
11105 0 : return performNVCASTCombine(N);
11106 0 : case ISD::INSERT_VECTOR_ELT:
11107 : return performPostLD1Combine(N, DCI, true);
11108 : case ISD::INTRINSIC_VOID:
11109 : case ISD::INTRINSIC_W_CHAIN:
11110 : switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
11111 0 : case Intrinsic::aarch64_neon_ld2:
11112 0 : case Intrinsic::aarch64_neon_ld3:
11113 : case Intrinsic::aarch64_neon_ld4:
11114 : case Intrinsic::aarch64_neon_ld1x2:
11115 : case Intrinsic::aarch64_neon_ld1x3:
11116 : case Intrinsic::aarch64_neon_ld1x4:
11117 : case Intrinsic::aarch64_neon_ld2lane:
11118 : case Intrinsic::aarch64_neon_ld3lane:
11119 : case Intrinsic::aarch64_neon_ld4lane:
11120 0 : case Intrinsic::aarch64_neon_ld2r:
11121 : case Intrinsic::aarch64_neon_ld3r:
11122 0 : case Intrinsic::aarch64_neon_ld4r:
11123 : case Intrinsic::aarch64_neon_st2:
11124 0 : case Intrinsic::aarch64_neon_st3:
11125 0 : case Intrinsic::aarch64_neon_st4:
11126 : case Intrinsic::aarch64_neon_st1x2:
11127 : case Intrinsic::aarch64_neon_st1x3:
11128 0 : case Intrinsic::aarch64_neon_st1x4:
11129 0 : case Intrinsic::aarch64_neon_st2lane:
11130 0 : case Intrinsic::aarch64_neon_st3lane:
11131 0 : case Intrinsic::aarch64_neon_st4lane:
11132 : return performNEONPostLDSTCombine(N, DCI, DAG);
11133 0 : default:
11134 : break;
11135 : }
11136 : case ISD::GlobalAddress:
11137 656 : return performGlobalAddressCombine(N, DAG, Subtarget, getTargetMachine());
11138 1312 : }
11139 273 : return SDValue();
11140 : }
11141 383 :
11142 : // Check if the return value is used as only a return value, as otherwise
11143 : // we can't perform a tail-call. In particular, we need to check for
11144 : // target ISD nodes that are returns and any other "odd" constructs
11145 : // that the generic analysis code won't necessarily catch.
11146 : bool AArch64TargetLowering::isUsedByReturnOnly(SDNode *N,
11147 4804 : SDValue &Chain) const {
11148 : if (N->getNumValues() != 1)
11149 : return false;
11150 : if (!N->hasNUsesOfValue(1, 0))
11151 4093 : return false;
11152 :
11153 949 : SDValue TCChain = Chain;
11154 : SDNode *Copy = *N->use_begin();
11155 3855 : if (Copy->getOpcode() == ISD::CopyToReg) {
11156 4020 : // If the copy has a glue operand, we conservatively assume it isn't safe to
11157 3979 : // perform a tail call.
11158 3785 : if (Copy->getOperand(Copy->getNumOperands() - 1).getValueType() ==
11159 194 : MVT::Glue)
11160 : return false;
11161 : TCChain = Copy->getOperand(0);
11162 165 : } else if (Copy->getOpcode() != ISD::FP_EXTEND)
11163 29 : return false;
11164 477 :
11165 : bool HasRet = false;
11166 41 : for (SDNode *Node : Copy->uses()) {
11167 : if (Node->getOpcode() != AArch64ISD::RET_FLAG)
11168 : return false;
11169 : HasRet = true;
11170 : }
11171 41 :
11172 1 : if (!HasRet)
11173 : return false;
11174 :
11175 : Chain = TCChain;
11176 : return true;
11177 : }
11178 :
11179 : // Return whether the an instruction can potentially be optimized to a tail
11180 : // call. This will cause the optimizers to attempt to move, or duplicate,
11181 : // return instructions to help enable tail call optimizations for this
11182 40 : // instruction.
11183 1 : bool AArch64TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
11184 : return CI->isTailCall();
11185 39 : }
11186 39 :
11187 78 : bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
11188 39 : SDValue &Offset,
11189 9 : ISD::MemIndexedMode &AM,
11190 : bool &IsInc,
11191 : SelectionDAG &DAG) const {
11192 60 : if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
11193 : return false;
11194 30 :
11195 : Base = Op->getOperand(0);
11196 : // All of the indexed addressing mode instructions take a signed
11197 122678 : // 9 bit immediate offset.
11198 : if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1))) {
11199 122678 : int64_t RHSC = RHS->getSExtValue();
11200 245356 : if (Op->getOpcode() == ISD::SUB)
11201 : RHSC = -(uint64_t)RHSC;
11202 : if (!isInt<9>(RHSC))
11203 : return false;
11204 17086 : IsInc = (Op->getOpcode() == ISD::ADD);
11205 : Offset = Op->getOperand(1);
11206 17086 : return true;
11207 1896 : }
11208 1896 : return false;
11209 1336 : }
11210 1336 :
11211 759 : bool AArch64TargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
11212 : SDValue &Offset,
11213 759 : ISD::MemIndexedMode &AM,
11214 446 : SelectionDAG &DAG) const {
11215 : EVT VT;
11216 446 : SDValue Ptr;
11217 189 : if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
11218 189 : VT = LD->getMemoryVT();
11219 1269 : Ptr = LD->getBasePtr();
11220 1269 : } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
11221 1040 : VT = ST->getMemoryVT();
11222 1040 : Ptr = ST->getBasePtr();
11223 6271 : } else
11224 6271 : return false;
11225 3872 :
11226 : bool IsInc;
11227 : if (!getIndexedAddressParts(Ptr.getNode(), Base, Offset, AM, IsInc, DAG))
11228 3872 : return false;
11229 10049 : AM = IsInc ? ISD::PRE_INC : ISD::PRE_DEC;
11230 10049 : return true;
11231 824 : }
11232 824 :
11233 533 : bool AArch64TargetLowering::getPostIndexedAddressParts(
11234 533 : SDNode *N, SDNode *Op, SDValue &Base, SDValue &Offset,
11235 122 : ISD::MemIndexedMode &AM, SelectionDAG &DAG) const {
11236 122 : EVT VT;
11237 22 : SDValue Ptr;
11238 44 : if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
11239 8 : VT = LD->getMemoryVT();
11240 : Ptr = LD->getBasePtr();
11241 16665 : } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
11242 16665 : VT = ST->getMemoryVT();
11243 493 : Ptr = ST->getBasePtr();
11244 493 : } else
11245 180 : return false;
11246 :
11247 180 : bool IsInc;
11248 999 : if (!getIndexedAddressParts(Op, Base, Offset, AM, IsInc, DAG))
11249 999 : return false;
11250 221 : // Post-indexing updates the base, so it's not a valid transform
11251 221 : // if that's not the same as the load's pointer.
11252 656 : if (Ptr != Base)
11253 656 : return false;
11254 1632 : AM = IsInc ? ISD::POST_INC : ISD::POST_DEC;
11255 1632 : return true;
11256 2400 : }
11257 :
11258 4800 : static void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
11259 1689 : SelectionDAG &DAG) {
11260 : SDLoc DL(N);
11261 : SDValue Op = N->getOperand(0);
11262 :
11263 : if (N->getValueType(0) != MVT::i16 || Op.getValueType() != MVT::f16)
11264 : return;
11265 :
11266 : Op = SDValue(
11267 : DAG.getMachineNode(TargetOpcode::INSERT_SUBREG, DL, MVT::f32,
11268 : DAG.getUNDEF(MVT::i32), Op,
11269 : DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
11270 : 0);
11271 : Op = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Op);
11272 : Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Op));
11273 : }
11274 :
11275 : static void ReplaceReductionResults(SDNode *N,
11276 : SmallVectorImpl<SDValue> &Results,
11277 : SelectionDAG &DAG, unsigned InterOp,
11278 : unsigned AcrossOp) {
11279 : EVT LoVT, HiVT;
11280 1689 : SDValue Lo, Hi;
11281 : SDLoc dl(N);
11282 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
11283 : std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
11284 : SDValue InterVal = DAG.getNode(InterOp, dl, LoVT, Lo, Hi);
11285 4804 : SDValue SplitVal = DAG.getNode(AcrossOp, dl, LoVT, InterVal);
11286 : Results.push_back(SplitVal);
11287 49639 : }
11288 :
11289 : static std::pair<SDValue, SDValue> splitInt128(SDValue N, SelectionDAG &DAG) {
11290 : SDLoc DL(N);
11291 : SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, N);
11292 : SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64,
11293 : DAG.getNode(ISD::SRL, DL, MVT::i128, N,
11294 354 : DAG.getConstant(64, DL, MVT::i64)));
11295 : return std::make_pair(Lo, Hi);
11296 354 : }
11297 :
11298 354 : // Create an even/odd pair of X registers holding integer value V.
11299 : static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
11300 : SDLoc dl(V.getNode());
11301 354 : SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i64);
11302 354 : SDValue VHi = DAG.getAnyExtOrTrunc(
11303 708 : DAG.getNode(ISD::SRL, dl, MVT::i128, V, DAG.getConstant(64, dl, MVT::i64)),
11304 : dl, MVT::i64);
11305 : if (DAG.getDataLayout().isBigEndian())
11306 45 : std::swap (VLo, VHi);
11307 : SDValue RegClass =
11308 9 : DAG.getTargetConstant(AArch64::XSeqPairsClassRegClassID, dl, MVT::i32);
11309 6 : SDValue SubReg0 = DAG.getTargetConstant(AArch64::sube64, dl, MVT::i32);
11310 339 : SDValue SubReg1 = DAG.getTargetConstant(AArch64::subo64, dl, MVT::i32);
11311 : const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
11312 : return SDValue(
11313 : DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
11314 14 : }
11315 10 :
11316 : static void ReplaceCMP_SWAP_128Results(SDNode *N,
11317 : SmallVectorImpl<SDValue> &Results,
11318 : SelectionDAG &DAG,
11319 : const AArch64Subtarget *Subtarget) {
11320 4 : assert(N->getValueType(0) == MVT::i128 &&
11321 : "AtomicCmpSwap on types less than 128 should be legal");
11322 :
11323 4 : if (Subtarget->hasLSE()) {
11324 4 : // LSE has a 128-bit compare and swap (CASP), but i128 is not a legal type,
11325 : // so lower it here, wrapped in REG_SEQUENCE and EXTRACT_SUBREG.
11326 : SDValue Ops[] = {
11327 : createGPRPairNode(DAG, N->getOperand(2)), // Compare value
11328 : createGPRPairNode(DAG, N->getOperand(3)), // Store value
11329 : N->getOperand(1), // Ptr
11330 : N->getOperand(0), // Chain in
11331 86 : };
11332 86 :
11333 : MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
11334 :
11335 6680 : unsigned Opcode;
11336 : switch (MemOp->getOrdering()) {
11337 : case AtomicOrdering::Monotonic:
11338 : Opcode = AArch64::CASPX;
11339 : break;
11340 13360 : case AtomicOrdering::Acquire:
11341 : Opcode = AArch64::CASPAX;
11342 : break;
11343 6680 : case AtomicOrdering::Release:
11344 : Opcode = AArch64::CASPLX;
11345 : break;
11346 6680 : case AtomicOrdering::AcquireRelease:
11347 6588 : case AtomicOrdering::SequentiallyConsistent:
11348 13176 : Opcode = AArch64::CASPALX;
11349 0 : break;
11350 6588 : default:
11351 : llvm_unreachable("Unexpected ordering!");
11352 6562 : }
11353 6562 :
11354 6562 : MachineSDNode *CmpSwap = DAG.getMachineNode(
11355 : Opcode, SDLoc(N), DAG.getVTList(MVT::Untyped, MVT::Other), Ops);
11356 : DAG.setNodeMemRefs(CmpSwap, {MemOp});
11357 :
11358 : unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64;
11359 2578 : if (DAG.getDataLayout().isBigEndian())
11360 : std::swap(SubReg1, SubReg2);
11361 : Results.push_back(DAG.getTargetExtractSubreg(SubReg1, SDLoc(N), MVT::i64,
11362 : SDValue(CmpSwap, 0)));
11363 : Results.push_back(DAG.getTargetExtractSubreg(SubReg2, SDLoc(N), MVT::i64,
11364 : SDValue(CmpSwap, 0)));
11365 : Results.push_back(SDValue(CmpSwap, 1)); // Chain out
11366 : return;
11367 1289 : }
11368 :
11369 : auto Desired = splitInt128(N->getOperand(2), DAG);
11370 1289 : auto New = splitInt128(N->getOperand(3), DAG);
11371 : SDValue Ops[] = {N->getOperand(1), Desired.first, Desired.second,
11372 : New.first, New.second, N->getOperand(0)};
11373 : SDNode *CmpSwap = DAG.getMachineNode(
11374 : AArch64::CMP_SWAP_128, SDLoc(N),
11375 2578 : DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops);
11376 :
11377 2524 : MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
11378 2524 : DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
11379 :
11380 : Results.push_back(SDValue(CmpSwap, 0));
11381 4102 : Results.push_back(SDValue(CmpSwap, 1));
11382 : Results.push_back(SDValue(CmpSwap, 3));
11383 : }
11384 :
11385 : void AArch64TargetLowering::ReplaceNodeResults(
11386 : SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
11387 : switch (N->getOpcode()) {
11388 1973 : default:
11389 : llvm_unreachable("Don't know how to custom expand this");
11390 : case ISD::BITCAST:
11391 2129 : ReplaceBITCASTResults(N, Results, DAG);
11392 : return;
11393 : case ISD::VECREDUCE_ADD:
11394 : case ISD::VECREDUCE_SMAX:
11395 : case ISD::VECREDUCE_SMIN:
11396 4102 : case ISD::VECREDUCE_UMAX:
11397 : case ISD::VECREDUCE_UMIN:
11398 : Results.push_back(LowerVECREDUCE(SDValue(N, 0), DAG));
11399 : return;
11400 :
11401 : case AArch64ISD::SADDV:
11402 4038 : ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::SADDV);
11403 4038 : return;
11404 : case AArch64ISD::UADDV:
11405 : ReplaceReductionResults(N, Results, DAG, ISD::ADD, AArch64ISD::UADDV);
11406 8 : return;
11407 : case AArch64ISD::SMINV:
11408 : ReplaceReductionResults(N, Results, DAG, ISD::SMIN, AArch64ISD::SMINV);
11409 8 : return;
11410 : case AArch64ISD::UMINV:
11411 8 : ReplaceReductionResults(N, Results, DAG, ISD::UMIN, AArch64ISD::UMINV);
11412 : return;
11413 : case AArch64ISD::SMAXV:
11414 7 : ReplaceReductionResults(N, Results, DAG, ISD::SMAX, AArch64ISD::SMAXV);
11415 14 : return;
11416 : case AArch64ISD::UMAXV:
11417 : ReplaceReductionResults(N, Results, DAG, ISD::UMAX, AArch64ISD::UMAXV);
11418 : return;
11419 7 : case ISD::FP_TO_UINT:
11420 14 : case ISD::FP_TO_SINT:
11421 : assert(N->getValueType(0) == MVT::i128 && "unexpected illegal conversion");
11422 : // Let normal code take care of it by not adding anything to Results.
11423 5 : return;
11424 : case ISD::ATOMIC_CMP_SWAP:
11425 : ReplaceCMP_SWAP_128Results(N, Results, DAG, Subtarget);
11426 : return;
11427 5 : }
11428 5 : }
11429 :
11430 10 : bool AArch64TargetLowering::useLoadStackGuardNode() const {
11431 5 : if (Subtarget->isTargetAndroid() || Subtarget->isTargetFuchsia())
11432 5 : return TargetLowering::useLoadStackGuardNode();
11433 5 : return true;
11434 5 : }
11435 5 :
11436 : unsigned AArch64TargetLowering::combineRepeatedFPDivisors() const {
11437 4 : // Combine multiple FDIVs with the same divisor into multiple FMULs by the
11438 : // reciprocal if there are three or more FDIVs.
11439 4 : return 3;
11440 : }
11441 :
11442 4 : TargetLoweringBase::LegalizeTypeAction
11443 4 : AArch64TargetLowering::getPreferredVectorAction(EVT VT) const {
11444 : MVT SVT = VT.getSimpleVT();
11445 : // During type legalization, we prefer to widen v1i8, v1i16, v1i32 to v8i8,
11446 : // v4i16, v2i32 instead of to promote.
11447 32 : if (SVT == MVT::v1i8 || SVT == MVT::v1i16 || SVT == MVT::v1i32
11448 32 : || SVT == MVT::v1f32)
11449 32 : return TypeWidenVector;
11450 :
11451 : return TargetLoweringBase::getPreferredVectorAction(VT);
11452 32 : }
11453 32 :
11454 : // Loads and stores less than 128-bits are already atomic; ones above that
11455 : // are doomed anyway, so defer to the default libcall and blame the OS when
11456 32 : // things go wrong.
11457 32 : bool AArch64TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
11458 32 : unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
11459 32 : return Size == 128;
11460 : }
11461 32 :
11462 : // Loads and stores less than 128-bits are already atomic; ones above that
11463 : // are doomed anyway, so defer to the default libcall and blame the OS when
11464 18 : // things go wrong.
11465 : TargetLowering::AtomicExpansionKind
11466 : AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
11467 : unsigned Size = LI->getType()->getPrimitiveSizeInBits();
11468 : return Size == 128 ? AtomicExpansionKind::LLSC : AtomicExpansionKind::None;
11469 : }
11470 :
11471 18 : // For the real atomic operations, we have ldxr/stxr up to 128 bits,
11472 : TargetLowering::AtomicExpansionKind
11473 : AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
11474 : unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11475 32 : if (Size > 128) return AtomicExpansionKind::None;
11476 32 : // Nand not supported in LSE.
11477 16 : if (AI->getOperation() == AtomicRMWInst::Nand) return AtomicExpansionKind::LLSC;
11478 : // Leave 128 bits to LLSC.
11479 16 : return (Subtarget->hasLSE() && Size < 128) ? AtomicExpansionKind::None : AtomicExpansionKind::LLSC;
11480 : }
11481 16 :
11482 : TargetLowering::AtomicExpansionKind
11483 : AArch64TargetLowering::shouldExpandAtomicCmpXchgInIR(
11484 16 : AtomicCmpXchgInst *AI) const {
11485 : // If subtarget has LSE, leave cmpxchg intact for codegen.
11486 : if (Subtarget->hasLSE())
11487 : return AtomicExpansionKind::None;
11488 8 : // At -O0, fast-regalloc cannot cope with the live vregs necessary to
11489 : // implement cmpxchg without spilling. If the address being exchanged is also
11490 8 : // on the stack and close enough to the spill slot, this can lead to a
11491 0 : // situation where the monitor always gets cleared and the atomic operation
11492 : // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
11493 0 : if (getTargetMachine().getOptLevel() == 0)
11494 4 : return AtomicExpansionKind::None;
11495 : return AtomicExpansionKind::LLSC;
11496 : }
11497 4 :
11498 0 : Value *AArch64TargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
11499 0 : AtomicOrdering Ord) const {
11500 : Module *M = Builder.GetInsertBlock()->getParent()->getParent();
11501 : Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
11502 32 : bool IsAcquire = isAcquireOrStronger(Ord);
11503 16 :
11504 32 : // Since i128 isn't legal and intrinsics don't get type-lowered, the ldrexd
11505 : // intrinsic must return {i64, i64} and we have to recombine them into a
11506 : // single i128 here.
11507 16 : if (ValTy->getPrimitiveSizeInBits() == 128) {
11508 : Intrinsic::ID Int =
11509 32 : IsAcquire ? Intrinsic::aarch64_ldaxp : Intrinsic::aarch64_ldxp;
11510 32 : Function *Ldxr = Intrinsic::getDeclaration(M, Int);
11511 16 :
11512 32 : Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
11513 16 : Value *LoHi = Builder.CreateCall(Ldxr, Addr, "lohi");
11514 :
11515 : Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
11516 : Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
11517 4 : Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
11518 4 : Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
11519 2 : return Builder.CreateOr(
11520 2 : Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 64)), "val64");
11521 4 : }
11522 2 :
11523 : Type *Tys[] = { Addr->getType() };
11524 : Intrinsic::ID Int =
11525 2 : IsAcquire ? Intrinsic::aarch64_ldaxr : Intrinsic::aarch64_ldxr;
11526 4 : Function *Ldxr = Intrinsic::getDeclaration(M, Int, Tys);
11527 :
11528 4 : return Builder.CreateTruncOrBitCast(
11529 2 : Builder.CreateCall(Ldxr, Addr),
11530 2 : cast<PointerType>(Addr->getType())->getElementType());
11531 : }
11532 :
11533 76 : void AArch64TargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
11534 : IRBuilder<> &Builder) const {
11535 152 : Module *M = Builder.GetInsertBlock()->getParent()->getParent();
11536 0 : Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::aarch64_clrex));
11537 0 : }
11538 8 :
11539 8 : Value *AArch64TargetLowering::emitStoreConditional(IRBuilder<> &Builder,
11540 8 : Value *Val, Value *Addr,
11541 35 : AtomicOrdering Ord) const {
11542 : Module *M = Builder.GetInsertBlock()->getParent()->getParent();
11543 : bool IsRelease = isReleaseOrStronger(Ord);
11544 :
11545 : // Since the intrinsics must have legal type, the i128 intrinsics take two
11546 70 : // parameters: "i64, i64". We must marshal Val into the appropriate form
11547 35 : // before the call.
11548 : if (Val->getType()->getPrimitiveSizeInBits() == 128) {
11549 0 : Intrinsic::ID Int =
11550 0 : IsRelease ? Intrinsic::aarch64_stlxp : Intrinsic::aarch64_stxp;
11551 0 : Function *Stxr = Intrinsic::getDeclaration(M, Int);
11552 1 : Type *Int64Ty = Type::getInt64Ty(M->getContext());
11553 1 :
11554 1 : Value *Lo = Builder.CreateTrunc(Val, Int64Ty, "lo");
11555 1 : Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 64), Int64Ty, "hi");
11556 1 : Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
11557 1 : return Builder.CreateCall(Stxr, {Lo, Hi, Addr});
11558 1 : }
11559 1 :
11560 1 : Intrinsic::ID Int =
11561 1 : IsRelease ? Intrinsic::aarch64_stlxr : Intrinsic::aarch64_stxr;
11562 1 : Type *Tys[] = { Addr->getType() };
11563 1 : Function *Stxr = Intrinsic::getDeclaration(M, Int, Tys);
11564 1 :
11565 1 : return Builder.CreateCall(Stxr,
11566 1 : {Builder.CreateZExtOrBitCast(
11567 : Val, Stxr->getFunctionType()->getParamType(0)),
11568 : Addr});
11569 : }
11570 :
11571 : bool AArch64TargetLowering::functionArgumentNeedsConsecutiveRegisters(
11572 18 : Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
11573 18 : return Ty->isArrayTy();
11574 18 : }
11575 :
11576 : bool AArch64TargetLowering::shouldNormalizeToSelectSequence(LLVMContext &,
11577 : EVT) const {
11578 32 : return false;
11579 64 : }
11580 3 :
11581 : static Value *UseTlsOffset(IRBuilder<> &IRB, unsigned Offset) {
11582 : Module *M = IRB.GetInsertBlock()->getParent()->getParent();
11583 : Function *ThreadPointerFunc =
11584 70 : Intrinsic::getDeclaration(M, Intrinsic::thread_pointer);
11585 : return IRB.CreatePointerCast(
11586 : IRB.CreateConstGEP1_32(IRB.CreateCall(ThreadPointerFunc), Offset),
11587 70 : Type::getInt8PtrTy(IRB.getContext())->getPointerTo(0));
11588 : }
11589 :
11590 : Value *AArch64TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
11591 127492 : // Android provides a fixed TLS slot for the stack cookie. See the definition
11592 : // of TLS_SLOT_STACK_GUARD in
11593 : // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
11594 : if (Subtarget->isTargetAndroid())
11595 125922 : return UseTlsOffset(IRB, 0x28);
11596 250274 :
11597 : // Fuchsia is similar.
11598 : // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
11599 121212 : if (Subtarget->isTargetFuchsia())
11600 : return UseTlsOffset(IRB, -0x10);
11601 :
11602 : return TargetLowering::getIRStackGuard(IRB);
11603 : }
11604 :
11605 93 : Value *AArch64TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
11606 93 : // Android provides a fixed TLS slot for the SafeStack pointer. See the
11607 93 : // definition of TLS_SLOT_SAFESTACK in
11608 : // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
11609 : if (Subtarget->isTargetAndroid())
11610 : return UseTlsOffset(IRB, 0x48);
11611 :
11612 : // Fuchsia is similar.
11613 : // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
11614 40 : if (Subtarget->isTargetFuchsia())
11615 40 : return UseTlsOffset(IRB, -0x8);
11616 40 :
11617 : return TargetLowering::getSafeStackPointerLocation(IRB);
11618 : }
11619 :
11620 : bool AArch64TargetLowering::isMaskAndCmp0FoldingBeneficial(
11621 1613 : const Instruction &AndI) const {
11622 1613 : // Only sink 'and' mask to cmp use block if it is masking a single bit, since
11623 1613 : // this is likely to be fold the and/cmp/br into a single tbz instruction. It
11624 : // may be beneficial to sink in other cases, but we would have to check that
11625 1602 : // the cmp would not get folded into the br to form a cbz for these to be
11626 : // beneficial.
11627 1599 : ConstantInt* Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
11628 : if (!Mask)
11629 : return false;
11630 : return Mask->getValue().isPowerOf2();
11631 117 : }
11632 :
11633 : void AArch64TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
11634 117 : // Update IsSplitCSR in AArch64unctionInfo.
11635 : AArch64FunctionInfo *AFI = Entry->getParent()->getInfo<AArch64FunctionInfo>();
11636 : AFI->setIsSplitCSR(true);
11637 : }
11638 :
11639 : void AArch64TargetLowering::insertCopiesSplitCSR(
11640 : MachineBasicBlock *Entry,
11641 29 : const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
11642 12 : const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
11643 : const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
11644 : if (!IStart)
11645 : return;
11646 117 :
11647 : const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11648 117 : MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
11649 117 : MachineBasicBlock::iterator MBBI = Entry->begin();
11650 : for (const MCPhysReg *I = IStart; *I; ++I) {
11651 : const TargetRegisterClass *RC = nullptr;
11652 : if (AArch64::GPR64RegClass.contains(*I))
11653 : RC = &AArch64::GPR64RegClass;
11654 : else if (AArch64::FPR64RegClass.contains(*I))
11655 117 : RC = &AArch64::FPR64RegClass;
11656 : else
11657 14 : llvm_unreachable("Unexpected register class in CSRsViaCopy!");
11658 14 :
11659 : unsigned NewVR = MRI->createVirtualRegister(RC);
11660 14 : // Create copy from CSR to a virtual register.
11661 14 : // FIXME: this currently does not emit CFI pseudo-instructions, it works
11662 : // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
11663 28 : // nounwind. If we want to generalize this later, we may need to emit
11664 28 : // CFI pseudo-instructions.
11665 14 : assert(Entry->getParent()->getFunction().hasFnAttribute(
11666 14 : Attribute::NoUnwind) &&
11667 14 : "Function should be nounwind in insertCopiesSplitCSR!");
11668 14 : Entry->addLiveIn(*I);
11669 : BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
11670 : .addReg(*I);
11671 103 :
11672 : // Insert the copy-back instructions right before the terminator.
11673 103 : for (auto *Exit : Exits)
11674 103 : BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
11675 : TII->get(TargetOpcode::COPY), *I)
11676 103 : .addReg(NewVR);
11677 103 : }
11678 : }
11679 :
11680 : bool AArch64TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
11681 17 : // Integer division on AArch64 is expensive. However, when aggressively
11682 : // optimizing for code size, we prefer to use a div instruction, as it is
11683 17 : // usually smaller than the alternative sequence.
11684 17 : // The exception to this is vector division. Since AArch64 doesn't have vector
11685 17 : // integer division, leaving the division as-is is a loss even in terms of
11686 : // size, because it will have to be scalarized, while the alternative code
11687 117 : // sequence can be performed in vector form.
11688 : bool OptSize =
11689 : Attr.hasAttribute(AttributeList::FunctionIndex, Attribute::MinSize);
11690 117 : return OptSize && !VT.isVector();
11691 : }
11692 :
11693 : bool AArch64TargetLowering::enableAggressiveFMAFusion(EVT VT) const {
11694 : return Subtarget->hasAggressiveFMA() && VT.isFloatingPoint();
11695 : }
11696 117 :
11697 : unsigned
11698 14 : AArch64TargetLowering::getVaListSizeInBits(const DataLayout &DL) const {
11699 14 : if (Subtarget->isTargetDarwin() || Subtarget->isTargetWindows())
11700 14 : return getPointerTy(DL).getSizeInBits();
11701 :
11702 14 : return 3 * getPointerTy(DL).getSizeInBits() + 2 * 32;
11703 28 : }
11704 28 :
11705 14 : void AArch64TargetLowering::finalizeLowering(MachineFunction &MF) const {
11706 : MF.getFrameInfo().computeMaxCallFrameSize(MF);
11707 : TargetLoweringBase::finalizeLowering(MF);
11708 : }
|
