Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1159, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64ISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-09-002921-48462-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

1//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64ExpandImm.h"
14#include "AArch64ISelLowering.h"
15#include "AArch64CallingConvention.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64PerfectShuffle.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ArrayRef.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SmallSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/StringSwitch.h"
31#include "llvm/ADT/Triple.h"
32#include "llvm/ADT/Twine.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/CallingConvLower.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineInstr.h"
39#include "llvm/CodeGen/MachineInstrBuilder.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/MachineRegisterInfo.h"
42#include "llvm/CodeGen/RuntimeLibcalls.h"
43#include "llvm/CodeGen/SelectionDAG.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/TargetCallingConv.h"
46#include "llvm/CodeGen/TargetInstrInfo.h"
47#include "llvm/CodeGen/ValueTypes.h"
48#include "llvm/IR/Attributes.h"
49#include "llvm/IR/Constants.h"
50#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/DebugLoc.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/GetElementPtrTypeIterator.h"
55#include "llvm/IR/GlobalValue.h"
56#include "llvm/IR/IRBuilder.h"
57#include "llvm/IR/Instruction.h"
58#include "llvm/IR/Instructions.h"
59#include "llvm/IR/IntrinsicInst.h"
60#include "llvm/IR/Intrinsics.h"
61#include "llvm/IR/Module.h"
62#include "llvm/IR/OperandTraits.h"
63#include "llvm/IR/PatternMatch.h"
64#include "llvm/IR/Type.h"
65#include "llvm/IR/Use.h"
66#include "llvm/IR/Value.h"
67#include "llvm/MC/MCRegisterInfo.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CodeGen.h"
70#include "llvm/Support/CommandLine.h"
71#include "llvm/Support/Compiler.h"
72#include "llvm/Support/Debug.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/KnownBits.h"
75#include "llvm/Support/MachineValueType.h"
76#include "llvm/Support/MathExtras.h"
77#include "llvm/Support/raw_ostream.h"
78#include "llvm/Target/TargetMachine.h"
79#include "llvm/Target/TargetOptions.h"
80#include <algorithm>
81#include <bitset>
82#include <cassert>
83#include <cctype>
84#include <cstdint>
85#include <cstdlib>
86#include <iterator>
87#include <limits>
88#include <tuple>
89#include <utility>
90#include <vector>
91
92using namespace llvm;
93using namespace llvm::PatternMatch;
94
95#define DEBUG_TYPE"aarch64-lower" "aarch64-lower"
96
97STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"aarch64-lower", "NumTailCalls"
, "Number of tail calls"}
;
98STATISTIC(NumShiftInserts, "Number of vector shift inserts")static llvm::Statistic NumShiftInserts = {"aarch64-lower", "NumShiftInserts"
, "Number of vector shift inserts"}
;
99STATISTIC(NumOptimizedImms, "Number of times immediates were optimized")static llvm::Statistic NumOptimizedImms = {"aarch64-lower", "NumOptimizedImms"
, "Number of times immediates were optimized"}
;
100
101static cl::opt<bool>
102EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden,
103 cl::desc("Allow AArch64 SLI/SRI formation"),
104 cl::init(false));
105
106// FIXME: The necessary dtprel relocations don't seem to be supported
107// well in the GNU bfd and gold linkers at the moment. Therefore, by
108// default, for now, fall back to GeneralDynamic code generation.
109cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
110 "aarch64-elf-ldtls-generation", cl::Hidden,
111 cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
112 cl::init(false));
113
114static cl::opt<bool>
115EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
116 cl::desc("Enable AArch64 logical imm instruction "
117 "optimization"),
118 cl::init(true));
119
120/// Value type used for condition codes.
121static const MVT MVT_CC = MVT::i32;
122
123AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
124 const AArch64Subtarget &STI)
125 : TargetLowering(TM), Subtarget(&STI) {
126 // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
127 // we have to make something up. Arbitrarily, choose ZeroOrOne.
128 setBooleanContents(ZeroOrOneBooleanContent);
129 // When comparing vectors the result sets the different elements in the
130 // vector to all-one or all-zero.
131 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
132
133 // Set up the register classes.
134 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
135 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
136
137 if (Subtarget->hasFPARMv8()) {
138 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
139 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
140 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
141 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
142 }
143
144 if (Subtarget->hasNEON()) {
145 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
146 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
147 // Someone set us up the NEON.
148 addDRTypeForNEON(MVT::v2f32);
149 addDRTypeForNEON(MVT::v8i8);
150 addDRTypeForNEON(MVT::v4i16);
151 addDRTypeForNEON(MVT::v2i32);
152 addDRTypeForNEON(MVT::v1i64);
153 addDRTypeForNEON(MVT::v1f64);
154 addDRTypeForNEON(MVT::v4f16);
155
156 addQRTypeForNEON(MVT::v4f32);
157 addQRTypeForNEON(MVT::v2f64);
158 addQRTypeForNEON(MVT::v16i8);
159 addQRTypeForNEON(MVT::v8i16);
160 addQRTypeForNEON(MVT::v4i32);
161 addQRTypeForNEON(MVT::v2i64);
162 addQRTypeForNEON(MVT::v8f16);
163 }
164
165 if (Subtarget->hasSVE()) {
166 // Add legal sve predicate types
167 addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
168 addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
169 addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
170 addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
171
172 // Add legal sve data types
173 addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
174 addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
175 addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
176 addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
177
178 addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
179 addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
180 addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
181 addRegisterClass(MVT::nxv1f32, &AArch64::ZPRRegClass);
182 addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
183 addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
184 addRegisterClass(MVT::nxv1f64, &AArch64::ZPRRegClass);
185 addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
186
187 for (auto VT : { MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64 }) {
188 setOperationAction(ISD::SADDSAT, VT, Legal);
189 setOperationAction(ISD::UADDSAT, VT, Legal);
190 setOperationAction(ISD::SSUBSAT, VT, Legal);
191 setOperationAction(ISD::USUBSAT, VT, Legal);
192 }
193 }
194
195 // Compute derived properties from the register classes
196 computeRegisterProperties(Subtarget->getRegisterInfo());
197
198 // Provide all sorts of operation actions
199 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
200 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
201 setOperationAction(ISD::SETCC, MVT::i32, Custom);
202 setOperationAction(ISD::SETCC, MVT::i64, Custom);
203 setOperationAction(ISD::SETCC, MVT::f16, Custom);
204 setOperationAction(ISD::SETCC, MVT::f32, Custom);
205 setOperationAction(ISD::SETCC, MVT::f64, Custom);
206 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
207 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
208 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
209 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
210 setOperationAction(ISD::BR_CC, MVT::i64, Custom);
211 setOperationAction(ISD::BR_CC, MVT::f16, Custom);
212 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
213 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
214 setOperationAction(ISD::SELECT, MVT::i32, Custom);
215 setOperationAction(ISD::SELECT, MVT::i64, Custom);
216 setOperationAction(ISD::SELECT, MVT::f16, Custom);
217 setOperationAction(ISD::SELECT, MVT::f32, Custom);
218 setOperationAction(ISD::SELECT, MVT::f64, Custom);
219 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
220 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
221 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
222 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
223 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
224 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
225 setOperationAction(ISD::JumpTable, MVT::i64, Custom);
226
227 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
228 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
229 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
230
231 setOperationAction(ISD::FREM, MVT::f32, Expand);
232 setOperationAction(ISD::FREM, MVT::f64, Expand);
233 setOperationAction(ISD::FREM, MVT::f80, Expand);
234
235 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
236
237 // Custom lowering hooks are needed for XOR
238 // to fold it into CSINC/CSINV.
239 setOperationAction(ISD::XOR, MVT::i32, Custom);
240 setOperationAction(ISD::XOR, MVT::i64, Custom);
241
242 // Virtually no operation on f128 is legal, but LLVM can't expand them when
243 // there's a valid register class, so we need custom operations in most cases.
244 setOperationAction(ISD::FABS, MVT::f128, Expand);
245 setOperationAction(ISD::FADD, MVT::f128, Custom);
246 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
247 setOperationAction(ISD::FCOS, MVT::f128, Expand);
248 setOperationAction(ISD::FDIV, MVT::f128, Custom);
249 setOperationAction(ISD::FMA, MVT::f128, Expand);
250 setOperationAction(ISD::FMUL, MVT::f128, Custom);
251 setOperationAction(ISD::FNEG, MVT::f128, Expand);
252 setOperationAction(ISD::FPOW, MVT::f128, Expand);
253 setOperationAction(ISD::FREM, MVT::f128, Expand);
254 setOperationAction(ISD::FRINT, MVT::f128, Expand);
255 setOperationAction(ISD::FSIN, MVT::f128, Expand);
256 setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
257 setOperationAction(ISD::FSQRT, MVT::f128, Expand);
258 setOperationAction(ISD::FSUB, MVT::f128, Custom);
259 setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
260 setOperationAction(ISD::SETCC, MVT::f128, Custom);
261 setOperationAction(ISD::BR_CC, MVT::f128, Custom);
262 setOperationAction(ISD::SELECT, MVT::f128, Custom);
263 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
264 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
265
266 // Lowering for many of the conversions is actually specified by the non-f128
267 // type. The LowerXXX function will be trivial when f128 isn't involved.
268 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
269 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
270 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
271 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
272 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
273 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
274 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
275 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
276 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
277 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
278 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
279 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
280 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
281 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
282
283 // Variable arguments.
284 setOperationAction(ISD::VASTART, MVT::Other, Custom);
285 setOperationAction(ISD::VAARG, MVT::Other, Custom);
286 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
287 setOperationAction(ISD::VAEND, MVT::Other, Expand);
288
289 // Variable-sized objects.
290 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
291 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
292
293 if (Subtarget->isTargetWindows())
294 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
295 else
296 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
297
298 // Constant pool entries
299 setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
300
301 // BlockAddress
302 setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
303
304 // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
305 setOperationAction(ISD::ADDC, MVT::i32, Custom);
306 setOperationAction(ISD::ADDE, MVT::i32, Custom);
307 setOperationAction(ISD::SUBC, MVT::i32, Custom);
308 setOperationAction(ISD::SUBE, MVT::i32, Custom);
309 setOperationAction(ISD::ADDC, MVT::i64, Custom);
310 setOperationAction(ISD::ADDE, MVT::i64, Custom);
311 setOperationAction(ISD::SUBC, MVT::i64, Custom);
312 setOperationAction(ISD::SUBE, MVT::i64, Custom);
313
314 // AArch64 lacks both left-rotate and popcount instructions.
315 setOperationAction(ISD::ROTL, MVT::i32, Expand);
316 setOperationAction(ISD::ROTL, MVT::i64, Expand);
317 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
318 setOperationAction(ISD::ROTL, VT, Expand);
319 setOperationAction(ISD::ROTR, VT, Expand);
320 }
321
322 // AArch64 doesn't have {U|S}MUL_LOHI.
323 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
324 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
325
326 setOperationAction(ISD::CTPOP, MVT::i32, Custom);
327 setOperationAction(ISD::CTPOP, MVT::i64, Custom);
328
329 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
330 setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
331 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
332 setOperationAction(ISD::SDIVREM, VT, Expand);
333 setOperationAction(ISD::UDIVREM, VT, Expand);
334 }
335 setOperationAction(ISD::SREM, MVT::i32, Expand);
336 setOperationAction(ISD::SREM, MVT::i64, Expand);
337 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
338 setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
339 setOperationAction(ISD::UREM, MVT::i32, Expand);
340 setOperationAction(ISD::UREM, MVT::i64, Expand);
341
342 // Custom lower Add/Sub/Mul with overflow.
343 setOperationAction(ISD::SADDO, MVT::i32, Custom);
344 setOperationAction(ISD::SADDO, MVT::i64, Custom);
345 setOperationAction(ISD::UADDO, MVT::i32, Custom);
346 setOperationAction(ISD::UADDO, MVT::i64, Custom);
347 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
348 setOperationAction(ISD::SSUBO, MVT::i64, Custom);
349 setOperationAction(ISD::USUBO, MVT::i32, Custom);
350 setOperationAction(ISD::USUBO, MVT::i64, Custom);
351 setOperationAction(ISD::SMULO, MVT::i32, Custom);
352 setOperationAction(ISD::SMULO, MVT::i64, Custom);
353 setOperationAction(ISD::UMULO, MVT::i32, Custom);
354 setOperationAction(ISD::UMULO, MVT::i64, Custom);
355
356 setOperationAction(ISD::FSIN, MVT::f32, Expand);
357 setOperationAction(ISD::FSIN, MVT::f64, Expand);
358 setOperationAction(ISD::FCOS, MVT::f32, Expand);
359 setOperationAction(ISD::FCOS, MVT::f64, Expand);
360 setOperationAction(ISD::FPOW, MVT::f32, Expand);
361 setOperationAction(ISD::FPOW, MVT::f64, Expand);
362 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
363 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
364 if (Subtarget->hasFullFP16())
365 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
366 else
367 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
368
369 setOperationAction(ISD::FREM, MVT::f16, Promote);
370 setOperationAction(ISD::FREM, MVT::v4f16, Expand);
371 setOperationAction(ISD::FREM, MVT::v8f16, Expand);
372 setOperationAction(ISD::FPOW, MVT::f16, Promote);
373 setOperationAction(ISD::FPOW, MVT::v4f16, Expand);
374 setOperationAction(ISD::FPOW, MVT::v8f16, Expand);
375 setOperationAction(ISD::FPOWI, MVT::f16, Promote);
376 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand);
377 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand);
378 setOperationAction(ISD::FCOS, MVT::f16, Promote);
379 setOperationAction(ISD::FCOS, MVT::v4f16, Expand);
380 setOperationAction(ISD::FCOS, MVT::v8f16, Expand);
381 setOperationAction(ISD::FSIN, MVT::f16, Promote);
382 setOperationAction(ISD::FSIN, MVT::v4f16, Expand);
383 setOperationAction(ISD::FSIN, MVT::v8f16, Expand);
384 setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
385 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
386 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
387 setOperationAction(ISD::FEXP, MVT::f16, Promote);
388 setOperationAction(ISD::FEXP, MVT::v4f16, Expand);
389 setOperationAction(ISD::FEXP, MVT::v8f16, Expand);
390 setOperationAction(ISD::FEXP2, MVT::f16, Promote);
391 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand);
392 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand);
393 setOperationAction(ISD::FLOG, MVT::f16, Promote);
394 setOperationAction(ISD::FLOG, MVT::v4f16, Expand);
395 setOperationAction(ISD::FLOG, MVT::v8f16, Expand);
396 setOperationAction(ISD::FLOG2, MVT::f16, Promote);
397 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand);
398 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand);
399 setOperationAction(ISD::FLOG10, MVT::f16, Promote);
400 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand);
401 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand);
402
403 if (!Subtarget->hasFullFP16()) {
404 setOperationAction(ISD::SELECT, MVT::f16, Promote);
405 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote);
406 setOperationAction(ISD::SETCC, MVT::f16, Promote);
407 setOperationAction(ISD::BR_CC, MVT::f16, Promote);
408 setOperationAction(ISD::FADD, MVT::f16, Promote);
409 setOperationAction(ISD::FSUB, MVT::f16, Promote);
410 setOperationAction(ISD::FMUL, MVT::f16, Promote);
411 setOperationAction(ISD::FDIV, MVT::f16, Promote);
412 setOperationAction(ISD::FMA, MVT::f16, Promote);
413 setOperationAction(ISD::FNEG, MVT::f16, Promote);
414 setOperationAction(ISD::FABS, MVT::f16, Promote);
415 setOperationAction(ISD::FCEIL, MVT::f16, Promote);
416 setOperationAction(ISD::FSQRT, MVT::f16, Promote);
417 setOperationAction(ISD::FFLOOR, MVT::f16, Promote);
418 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
419 setOperationAction(ISD::FRINT, MVT::f16, Promote);
420 setOperationAction(ISD::FROUND, MVT::f16, Promote);
421 setOperationAction(ISD::FTRUNC, MVT::f16, Promote);
422 setOperationAction(ISD::FMINNUM, MVT::f16, Promote);
423 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote);
424 setOperationAction(ISD::FMINIMUM, MVT::f16, Promote);
425 setOperationAction(ISD::FMAXIMUM, MVT::f16, Promote);
426
427 // promote v4f16 to v4f32 when that is known to be safe.
428 setOperationAction(ISD::FADD, MVT::v4f16, Promote);
429 setOperationAction(ISD::FSUB, MVT::v4f16, Promote);
430 setOperationAction(ISD::FMUL, MVT::v4f16, Promote);
431 setOperationAction(ISD::FDIV, MVT::v4f16, Promote);
432 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote);
433 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote);
434 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32);
435 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32);
436 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32);
437 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32);
438 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32);
439 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32);
440
441 setOperationAction(ISD::FABS, MVT::v4f16, Expand);
442 setOperationAction(ISD::FNEG, MVT::v4f16, Expand);
443 setOperationAction(ISD::FROUND, MVT::v4f16, Expand);
444 setOperationAction(ISD::FMA, MVT::v4f16, Expand);
445 setOperationAction(ISD::SETCC, MVT::v4f16, Expand);
446 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand);
447 setOperationAction(ISD::SELECT, MVT::v4f16, Expand);
448 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand);
449 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand);
450 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand);
451 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand);
452 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand);
453 setOperationAction(ISD::FRINT, MVT::v4f16, Expand);
454 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand);
455 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand);
456
457 setOperationAction(ISD::FABS, MVT::v8f16, Expand);
458 setOperationAction(ISD::FADD, MVT::v8f16, Expand);
459 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand);
460 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand);
461 setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
462 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand);
463 setOperationAction(ISD::FMA, MVT::v8f16, Expand);
464 setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
465 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand);
466 setOperationAction(ISD::FNEG, MVT::v8f16, Expand);
467 setOperationAction(ISD::FROUND, MVT::v8f16, Expand);
468 setOperationAction(ISD::FRINT, MVT::v8f16, Expand);
469 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand);
470 setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
471 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand);
472 setOperationAction(ISD::SETCC, MVT::v8f16, Expand);
473 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand);
474 setOperationAction(ISD::SELECT, MVT::v8f16, Expand);
475 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand);
476 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand);
477 }
478
479 // AArch64 has implementations of a lot of rounding-like FP operations.
480 for (MVT Ty : {MVT::f32, MVT::f64}) {
481 setOperationAction(ISD::FFLOOR, Ty, Legal);
482 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
483 setOperationAction(ISD::FCEIL, Ty, Legal);
484 setOperationAction(ISD::FRINT, Ty, Legal);
485 setOperationAction(ISD::FTRUNC, Ty, Legal);
486 setOperationAction(ISD::FROUND, Ty, Legal);
487 setOperationAction(ISD::FMINNUM, Ty, Legal);
488 setOperationAction(ISD::FMAXNUM, Ty, Legal);
489 setOperationAction(ISD::FMINIMUM, Ty, Legal);
490 setOperationAction(ISD::FMAXIMUM, Ty, Legal);
491 setOperationAction(ISD::LROUND, Ty, Legal);
492 setOperationAction(ISD::LLROUND, Ty, Legal);
493 setOperationAction(ISD::LRINT, Ty, Legal);
494 setOperationAction(ISD::LLRINT, Ty, Legal);
495 }
496
497 if (Subtarget->hasFullFP16()) {
498 setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
499 setOperationAction(ISD::FFLOOR, MVT::f16, Legal);
500 setOperationAction(ISD::FCEIL, MVT::f16, Legal);
501 setOperationAction(ISD::FRINT, MVT::f16, Legal);
502 setOperationAction(ISD::FTRUNC, MVT::f16, Legal);
503 setOperationAction(ISD::FROUND, MVT::f16, Legal);
504 setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
505 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
506 setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
507 setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
508 }
509
510 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
511
512 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
513
514 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
515 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
516 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
517 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
518 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
519
520 // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
521 // This requires the Performance Monitors extension.
522 if (Subtarget->hasPerfMon())
523 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
524
525 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
526 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
527 // Issue __sincos_stret if available.
528 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
529 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
530 } else {
531 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
532 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
533 }
534
535 if (Subtarget->getTargetTriple().isOSMSVCRT()) {
536 // MSVCRT doesn't have powi; fall back to pow
537 setLibcallName(RTLIB::POWI_F32, nullptr);
538 setLibcallName(RTLIB::POWI_F64, nullptr);
539 }
540
541 // Make floating-point constants legal for the large code model, so they don't
542 // become loads from the constant pool.
543 if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
544 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
545 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
546 }
547
548 // AArch64 does not have floating-point extending loads, i1 sign-extending
549 // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
550 for (MVT VT : MVT::fp_valuetypes()) {
551 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
552 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
553 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
554 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
555 }
556 for (MVT VT : MVT::integer_valuetypes())
557 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
558
559 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
560 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
561 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
562 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
563 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
564 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
565 setTruncStoreAction(MVT::f128, MVT::f16, Expand);
566
567 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
568 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
569
570 // Indexed loads and stores are supported.
571 for (unsigned im = (unsigned)ISD::PRE_INC;
572 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
573 setIndexedLoadAction(im, MVT::i8, Legal);
574 setIndexedLoadAction(im, MVT::i16, Legal);
575 setIndexedLoadAction(im, MVT::i32, Legal);
576 setIndexedLoadAction(im, MVT::i64, Legal);
577 setIndexedLoadAction(im, MVT::f64, Legal);
578 setIndexedLoadAction(im, MVT::f32, Legal);
579 setIndexedLoadAction(im, MVT::f16, Legal);
580 setIndexedStoreAction(im, MVT::i8, Legal);
581 setIndexedStoreAction(im, MVT::i16, Legal);
582 setIndexedStoreAction(im, MVT::i32, Legal);
583 setIndexedStoreAction(im, MVT::i64, Legal);
584 setIndexedStoreAction(im, MVT::f64, Legal);
585 setIndexedStoreAction(im, MVT::f32, Legal);
586 setIndexedStoreAction(im, MVT::f16, Legal);
587 }
588
589 // Trap.
590 setOperationAction(ISD::TRAP, MVT::Other, Legal);
591 if (Subtarget->isTargetWindows())
592 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
593
594 // We combine OR nodes for bitfield operations.
595 setTargetDAGCombine(ISD::OR);
596 // Try to create BICs for vector ANDs.
597 setTargetDAGCombine(ISD::AND);
598
599 // Vector add and sub nodes may conceal a high-half opportunity.
600 // Also, try to fold ADD into CSINC/CSINV..
601 setTargetDAGCombine(ISD::ADD);
602 setTargetDAGCombine(ISD::SUB);
603 setTargetDAGCombine(ISD::SRL);
604 setTargetDAGCombine(ISD::XOR);
605 setTargetDAGCombine(ISD::SINT_TO_FP);
606 setTargetDAGCombine(ISD::UINT_TO_FP);
607
608 setTargetDAGCombine(ISD::FP_TO_SINT);
609 setTargetDAGCombine(ISD::FP_TO_UINT);
610 setTargetDAGCombine(ISD::FDIV);
611
612 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
613
614 setTargetDAGCombine(ISD::ANY_EXTEND);
615 setTargetDAGCombine(ISD::ZERO_EXTEND);
616 setTargetDAGCombine(ISD::SIGN_EXTEND);
617 setTargetDAGCombine(ISD::BITCAST);
618 setTargetDAGCombine(ISD::CONCAT_VECTORS);
619 setTargetDAGCombine(ISD::STORE);
620 if (Subtarget->supportsAddressTopByteIgnored())
621 setTargetDAGCombine(ISD::LOAD);
622
623 setTargetDAGCombine(ISD::MUL);
624
625 setTargetDAGCombine(ISD::SELECT);
626 setTargetDAGCombine(ISD::VSELECT);
627
628 setTargetDAGCombine(ISD::INTRINSIC_VOID);
629 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
630 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
631
632 setTargetDAGCombine(ISD::GlobalAddress);
633
634 // In case of strict alignment, avoid an excessive number of byte wide stores.
635 MaxStoresPerMemsetOptSize = 8;
636 MaxStoresPerMemset = Subtarget->requiresStrictAlign()
637 ? MaxStoresPerMemsetOptSize : 32;
638
639 MaxGluedStoresPerMemcpy = 4;
640 MaxStoresPerMemcpyOptSize = 4;
641 MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
642 ? MaxStoresPerMemcpyOptSize : 16;
643
644 MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
645
646 MaxLoadsPerMemcmpOptSize = 4;
647 MaxLoadsPerMemcmp = Subtarget->requiresStrictAlign()
648 ? MaxLoadsPerMemcmpOptSize : 8;
649
650 setStackPointerRegisterToSaveRestore(AArch64::SP);
651
652 setSchedulingPreference(Sched::Hybrid);
653
654 EnableExtLdPromotion = true;
655
656 // Set required alignment.
657 setMinFunctionAlignment(Align(4));
658 // Set preferred alignments.
659 setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
660 setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
661
662 // Only change the limit for entries in a jump table if specified by
663 // the sub target, but not at the command line.
664 unsigned MaxJT = STI.getMaximumJumpTableSize();
665 if (MaxJT && getMaximumJumpTableSize() == UINT_MAX(2147483647 *2U +1U))
666 setMaximumJumpTableSize(MaxJT);
667
668 setHasExtractBitsInsn(true);
669
670 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
671
672 if (Subtarget->hasNEON()) {
673 // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
674 // silliness like this:
675 setOperationAction(ISD::FABS, MVT::v1f64, Expand);
676 setOperationAction(ISD::FADD, MVT::v1f64, Expand);
677 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
678 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
679 setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
680 setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
681 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
682 setOperationAction(ISD::FMA, MVT::v1f64, Expand);
683 setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
684 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
685 setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
686 setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
687 setOperationAction(ISD::FREM, MVT::v1f64, Expand);
688 setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
689 setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
690 setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
691 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
692 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
693 setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
694 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
695 setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
696 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
697 setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
698 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
699 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
700
701 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
702 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
703 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
704 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
705 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
706
707 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
708
709 // AArch64 doesn't have a direct vector ->f32 conversion instructions for
710 // elements smaller than i32, so promote the input to i32 first.
711 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
712 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
713 // i8 vector elements also need promotion to i32 for v8i8
714 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
715 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
716 // Similarly, there is no direct i32 -> f64 vector conversion instruction.
717 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
718 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
719 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
720 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
721 // Or, direct i32 -> f16 vector conversion. Set it so custom, so the
722 // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
723 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
724 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
725
726 if (Subtarget->hasFullFP16()) {
727 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
728 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
729 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
730 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
731 } else {
732 // when AArch64 doesn't have fullfp16 support, promote the input
733 // to i32 first.
734 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
735 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
736 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
737 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
738 }
739
740 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
741 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
742
743 // AArch64 doesn't have MUL.2d:
744 setOperationAction(ISD::MUL, MVT::v2i64, Expand);
745 // Custom handling for some quad-vector types to detect MULL.
746 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
747 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
748 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
749
750 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
751 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
752 // Vector reductions
753 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
754 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
755 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
756 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
757 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
758
759 // Saturates
760 setOperationAction(ISD::SADDSAT, VT, Legal);
761 setOperationAction(ISD::UADDSAT, VT, Legal);
762 setOperationAction(ISD::SSUBSAT, VT, Legal);
763 setOperationAction(ISD::USUBSAT, VT, Legal);
764 }
765 for (MVT VT : { MVT::v4f16, MVT::v2f32,
766 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
767 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
768 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
769 }
770
771 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
772 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
773 // Likewise, narrowing and extending vector loads/stores aren't handled
774 // directly.
775 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
776 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
777
778 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
779 setOperationAction(ISD::MULHS, VT, Legal);
780 setOperationAction(ISD::MULHU, VT, Legal);
781 } else {
782 setOperationAction(ISD::MULHS, VT, Expand);
783 setOperationAction(ISD::MULHU, VT, Expand);
784 }
785 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
786 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
787
788 setOperationAction(ISD::BSWAP, VT, Expand);
789 setOperationAction(ISD::CTTZ, VT, Expand);
790
791 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
792 setTruncStoreAction(VT, InnerVT, Expand);
793 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
794 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
795 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
796 }
797 }
798
799 // AArch64 has implementations of a lot of rounding-like FP operations.
800 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
801 setOperationAction(ISD::FFLOOR, Ty, Legal);
802 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
803 setOperationAction(ISD::FCEIL, Ty, Legal);
804 setOperationAction(ISD::FRINT, Ty, Legal);
805 setOperationAction(ISD::FTRUNC, Ty, Legal);
806 setOperationAction(ISD::FROUND, Ty, Legal);
807 }
808
809 if (Subtarget->hasFullFP16()) {
810 for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
811 setOperationAction(ISD::FFLOOR, Ty, Legal);
812 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
813 setOperationAction(ISD::FCEIL, Ty, Legal);
814 setOperationAction(ISD::FRINT, Ty, Legal);
815 setOperationAction(ISD::FTRUNC, Ty, Legal);
816 setOperationAction(ISD::FROUND, Ty, Legal);
817 }
818 }
819
820 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
821 }
822
823 if (Subtarget->hasSVE()) {
824 // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
825 // splat of 0 or undef) once vector selects supported in SVE codegen. See
826 // D68877 for more details.
827 for (MVT VT : MVT::integer_scalable_vector_valuetypes()) {
828 if (isTypeLegal(VT) && VT.getVectorElementType() != MVT::i1)
829 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
830 }
831 }
832
833 PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
834}
835
836void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
837 assert(VT.isVector() && "VT should be a vector type")((VT.isVector() && "VT should be a vector type") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"VT should be a vector type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 837, __PRETTY_FUNCTION__))
;
838
839 if (VT.isFloatingPoint()) {
840 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
841 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
842 setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
843 }
844
845 // Mark vector float intrinsics as expand.
846 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
847 setOperationAction(ISD::FSIN, VT, Expand);
848 setOperationAction(ISD::FCOS, VT, Expand);
849 setOperationAction(ISD::FPOW, VT, Expand);
850 setOperationAction(ISD::FLOG, VT, Expand);
851 setOperationAction(ISD::FLOG2, VT, Expand);
852 setOperationAction(ISD::FLOG10, VT, Expand);
853 setOperationAction(ISD::FEXP, VT, Expand);
854 setOperationAction(ISD::FEXP2, VT, Expand);
855
856 // But we do support custom-lowering for FCOPYSIGN.
857 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
858 }
859
860 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
861 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
862 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
863 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
864 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
865 setOperationAction(ISD::SRA, VT, Custom);
866 setOperationAction(ISD::SRL, VT, Custom);
867 setOperationAction(ISD::SHL, VT, Custom);
868 setOperationAction(ISD::OR, VT, Custom);
869 setOperationAction(ISD::SETCC, VT, Custom);
870 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
871
872 setOperationAction(ISD::SELECT, VT, Expand);
873 setOperationAction(ISD::SELECT_CC, VT, Expand);
874 setOperationAction(ISD::VSELECT, VT, Expand);
875 for (MVT InnerVT : MVT::all_valuetypes())
876 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
877
878 // CNT supports only B element sizes, then use UADDLP to widen.
879 if (VT != MVT::v8i8 && VT != MVT::v16i8)
880 setOperationAction(ISD::CTPOP, VT, Custom);
881
882 setOperationAction(ISD::UDIV, VT, Expand);
883 setOperationAction(ISD::SDIV, VT, Expand);
884 setOperationAction(ISD::UREM, VT, Expand);
885 setOperationAction(ISD::SREM, VT, Expand);
886 setOperationAction(ISD::FREM, VT, Expand);
887
888 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
889 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
890
891 if (!VT.isFloatingPoint())
892 setOperationAction(ISD::ABS, VT, Legal);
893
894 // [SU][MIN|MAX] are available for all NEON types apart from i64.
895 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
896 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
897 setOperationAction(Opcode, VT, Legal);
898
899 // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
900 if (VT.isFloatingPoint() &&
901 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
902 for (unsigned Opcode :
903 {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM})
904 setOperationAction(Opcode, VT, Legal);
905
906 if (Subtarget->isLittleEndian()) {
907 for (unsigned im = (unsigned)ISD::PRE_INC;
908 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
909 setIndexedLoadAction(im, VT, Legal);
910 setIndexedStoreAction(im, VT, Legal);
911 }
912 }
913}
914
915void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
916 addRegisterClass(VT, &AArch64::FPR64RegClass);
917 addTypeForNEON(VT, MVT::v2i32);
918}
919
920void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
921 addRegisterClass(VT, &AArch64::FPR128RegClass);
922 addTypeForNEON(VT, MVT::v4i32);
923}
924
925EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
926 EVT VT) const {
927 if (!VT.isVector())
928 return MVT::i32;
929 return VT.changeVectorElementTypeToInteger();
930}
931
932static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
933 const APInt &Demanded,
934 TargetLowering::TargetLoweringOpt &TLO,
935 unsigned NewOpc) {
936 uint64_t OldImm = Imm, NewImm, Enc;
937 uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
938
939 // Return if the immediate is already all zeros, all ones, a bimm32 or a
940 // bimm64.
941 if (Imm == 0 || Imm == Mask ||
942 AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
943 return false;
944
945 unsigned EltSize = Size;
946 uint64_t DemandedBits = Demanded.getZExtValue();
947
948 // Clear bits that are not demanded.
949 Imm &= DemandedBits;
950
951 while (true) {
952 // The goal here is to set the non-demanded bits in a way that minimizes
953 // the number of switching between 0 and 1. In order to achieve this goal,
954 // we set the non-demanded bits to the value of the preceding demanded bits.
955 // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
956 // non-demanded bit), we copy bit0 (1) to the least significant 'x',
957 // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
958 // The final result is 0b11000011.
959 uint64_t NonDemandedBits = ~DemandedBits;
960 uint64_t InvertedImm = ~Imm & DemandedBits;
961 uint64_t RotatedImm =
962 ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
963 NonDemandedBits;
964 uint64_t Sum = RotatedImm + NonDemandedBits;
965 bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
966 uint64_t Ones = (Sum + Carry) & NonDemandedBits;
967 NewImm = (Imm | Ones) & Mask;
968
969 // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
970 // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
971 // we halve the element size and continue the search.
972 if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
973 break;
974
975 // We cannot shrink the element size any further if it is 2-bits.
976 if (EltSize == 2)
977 return false;
978
979 EltSize /= 2;
980 Mask >>= EltSize;
981 uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
982
983 // Return if there is mismatch in any of the demanded bits of Imm and Hi.
984 if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
985 return false;
986
987 // Merge the upper and lower halves of Imm and DemandedBits.
988 Imm |= Hi;
989 DemandedBits |= DemandedBitsHi;
990 }
991
992 ++NumOptimizedImms;
993
994 // Replicate the element across the register width.
995 while (EltSize < Size) {
996 NewImm |= NewImm << EltSize;
997 EltSize *= 2;
998 }
999
1000 (void)OldImm;
1001 assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&((((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
"demanded bits should never be altered") ? static_cast<void
> (0) : __assert_fail ("((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 && \"demanded bits should never be altered\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1002, __PRETTY_FUNCTION__))
1002 "demanded bits should never be altered")((((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
"demanded bits should never be altered") ? static_cast<void
> (0) : __assert_fail ("((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 && \"demanded bits should never be altered\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1002, __PRETTY_FUNCTION__))
;
1003 assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm")((OldImm != NewImm && "the new imm shouldn't be equal to the old imm"
) ? static_cast<void> (0) : __assert_fail ("OldImm != NewImm && \"the new imm shouldn't be equal to the old imm\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1003, __PRETTY_FUNCTION__))
;
1004
1005 // Create the new constant immediate node.
1006 EVT VT = Op.getValueType();
1007 SDLoc DL(Op);
1008 SDValue New;
1009
1010 // If the new constant immediate is all-zeros or all-ones, let the target
1011 // independent DAG combine optimize this node.
1012 if (NewImm == 0 || NewImm == OrigMask) {
1013 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
1014 TLO.DAG.getConstant(NewImm, DL, VT));
1015 // Otherwise, create a machine node so that target independent DAG combine
1016 // doesn't undo this optimization.
1017 } else {
1018 Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
1019 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1020 New = SDValue(
1021 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
1022 }
1023
1024 return TLO.CombineTo(Op, New);
1025}
1026
1027bool AArch64TargetLowering::targetShrinkDemandedConstant(
1028 SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const {
1029 // Delay this optimization to as late as possible.
1030 if (!TLO.LegalOps)
1031 return false;
1032
1033 if (!EnableOptimizeLogicalImm)
1034 return false;
1035
1036 EVT VT = Op.getValueType();
1037 if (VT.isVector())
1038 return false;
1039
1040 unsigned Size = VT.getSizeInBits();
1041 assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "i32 or i64 is expected after legalization."
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"i32 or i64 is expected after legalization.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1042, __PRETTY_FUNCTION__))
1042 "i32 or i64 is expected after legalization.")(((Size == 32 || Size == 64) && "i32 or i64 is expected after legalization."
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"i32 or i64 is expected after legalization.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1042, __PRETTY_FUNCTION__))
;
1043
1044 // Exit early if we demand all bits.
1045 if (Demanded.countPopulation() == Size)
1046 return false;
1047
1048 unsigned NewOpc;
1049 switch (Op.getOpcode()) {
1050 default:
1051 return false;
1052 case ISD::AND:
1053 NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1054 break;
1055 case ISD::OR:
1056 NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1057 break;
1058 case ISD::XOR:
1059 NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1060 break;
1061 }
1062 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1063 if (!C)
1064 return false;
1065 uint64_t Imm = C->getZExtValue();
1066 return optimizeLogicalImm(Op, Size, Imm, Demanded, TLO, NewOpc);
1067}
1068
1069/// computeKnownBitsForTargetNode - Determine which of the bits specified in
1070/// Mask are known to be either zero or one and return them Known.
1071void AArch64TargetLowering::computeKnownBitsForTargetNode(
1072 const SDValue Op, KnownBits &Known,
1073 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
1074 switch (Op.getOpcode()) {
1075 default:
1076 break;
1077 case AArch64ISD::CSEL: {
1078 KnownBits Known2;
1079 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
1080 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
1081 Known.Zero &= Known2.Zero;
1082 Known.One &= Known2.One;
1083 break;
1084 }
1085 case AArch64ISD::LOADgot:
1086 case AArch64ISD::ADDlow: {
1087 if (!Subtarget->isTargetILP32())
1088 break;
1089 // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
1090 Known.Zero = APInt::getHighBitsSet(64, 32);
1091 break;
1092 }
1093 case ISD::INTRINSIC_W_CHAIN: {
1094 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1095 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1096 switch (IntID) {
1097 default: return;
1098 case Intrinsic::aarch64_ldaxr:
1099 case Intrinsic::aarch64_ldxr: {
1100 unsigned BitWidth = Known.getBitWidth();
1101 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1102 unsigned MemBits = VT.getScalarSizeInBits();
1103 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1104 return;
1105 }
1106 }
1107 break;
1108 }
1109 case ISD::INTRINSIC_WO_CHAIN:
1110 case ISD::INTRINSIC_VOID: {
1111 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1112 switch (IntNo) {
1113 default:
1114 break;
1115 case Intrinsic::aarch64_neon_umaxv:
1116 case Intrinsic::aarch64_neon_uminv: {
1117 // Figure out the datatype of the vector operand. The UMINV instruction
1118 // will zero extend the result, so we can mark as known zero all the
1119 // bits larger than the element datatype. 32-bit or larget doesn't need
1120 // this as those are legal types and will be handled by isel directly.
1121 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1122 unsigned BitWidth = Known.getBitWidth();
1123 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1124 assert(BitWidth >= 8 && "Unexpected width!")((BitWidth >= 8 && "Unexpected width!") ? static_cast
<void> (0) : __assert_fail ("BitWidth >= 8 && \"Unexpected width!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1124, __PRETTY_FUNCTION__))
;
1125 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1126 Known.Zero |= Mask;
1127 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1128 assert(BitWidth >= 16 && "Unexpected width!")((BitWidth >= 16 && "Unexpected width!") ? static_cast
<void> (0) : __assert_fail ("BitWidth >= 16 && \"Unexpected width!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1128, __PRETTY_FUNCTION__))
;
1129 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1130 Known.Zero |= Mask;
1131 }
1132 break;
1133 } break;
1134 }
1135 }
1136 }
1137}
1138
1139MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1140 EVT) const {
1141 return MVT::i64;
1142}
1143
1144bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1145 EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1146 bool *Fast) const {
1147 if (Subtarget->requiresStrictAlign())
1148 return false;
1149
1150 if (Fast) {
1151 // Some CPUs are fine with unaligned stores except for 128-bit ones.
1152 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1153 // See comments in performSTORECombine() for more details about
1154 // these conditions.
1155
1156 // Code that uses clang vector extensions can mark that it
1157 // wants unaligned accesses to be treated as fast by
1158 // underspecifying alignment to be 1 or 2.
1159 Align <= 2 ||
1160
1161 // Disregard v2i64. Memcpy lowering produces those and splitting
1162 // them regresses performance on micro-benchmarks and olden/bh.
1163 VT == MVT::v2i64;
1164 }
1165 return true;
1166}
1167
1168// Same as above but handling LLTs instead.
1169bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1170 LLT Ty, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1171 bool *Fast) const {
1172 if (Subtarget->requiresStrictAlign())
1173 return false;
1174
1175 if (Fast) {
1176 // Some CPUs are fine with unaligned stores except for 128-bit ones.
1177 *Fast = !Subtarget->isMisaligned128StoreSlow() ||
1178 Ty.getSizeInBytes() != 16 ||
1179 // See comments in performSTORECombine() for more details about
1180 // these conditions.
1181
1182 // Code that uses clang vector extensions can mark that it
1183 // wants unaligned accesses to be treated as fast by
1184 // underspecifying alignment to be 1 or 2.
1185 Align <= 2 ||
1186
1187 // Disregard v2i64. Memcpy lowering produces those and splitting
1188 // them regresses performance on micro-benchmarks and olden/bh.
1189 Ty == LLT::vector(2, 64);
1190 }
1191 return true;
1192}
1193
1194FastISel *
1195AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1196 const TargetLibraryInfo *libInfo) const {
1197 return AArch64::createFastISel(funcInfo, libInfo);
1198}
1199
1200const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1201 switch ((AArch64ISD::NodeType)Opcode) {
1202 case AArch64ISD::FIRST_NUMBER: break;
1203 case AArch64ISD::CALL: return "AArch64ISD::CALL";
1204 case AArch64ISD::ADRP: return "AArch64ISD::ADRP";
1205 case AArch64ISD::ADR: return "AArch64ISD::ADR";
1206 case AArch64ISD::ADDlow: return "AArch64ISD::ADDlow";
1207 case AArch64ISD::LOADgot: return "AArch64ISD::LOADgot";
1208 case AArch64ISD::RET_FLAG: return "AArch64ISD::RET_FLAG";
1209 case AArch64ISD::BRCOND: return "AArch64ISD::BRCOND";
1210 case AArch64ISD::CSEL: return "AArch64ISD::CSEL";
1211 case AArch64ISD::FCSEL: return "AArch64ISD::FCSEL";
1212 case AArch64ISD::CSINV: return "AArch64ISD::CSINV";
1213 case AArch64ISD::CSNEG: return "AArch64ISD::CSNEG";
1214 case AArch64ISD::CSINC: return "AArch64ISD::CSINC";
1215 case AArch64ISD::THREAD_POINTER: return "AArch64ISD::THREAD_POINTER";
1216 case AArch64ISD::TLSDESC_CALLSEQ: return "AArch64ISD::TLSDESC_CALLSEQ";
1217 case AArch64ISD::ADC: return "AArch64ISD::ADC";
1218 case AArch64ISD::SBC: return "AArch64ISD::SBC";
1219 case AArch64ISD::ADDS: return "AArch64ISD::ADDS";
1220 case AArch64ISD::SUBS: return "AArch64ISD::SUBS";
1221 case AArch64ISD::ADCS: return "AArch64ISD::ADCS";
1222 case AArch64ISD::SBCS: return "AArch64ISD::SBCS";
1223 case AArch64ISD::ANDS: return "AArch64ISD::ANDS";
1224 case AArch64ISD::CCMP: return "AArch64ISD::CCMP";
1225 case AArch64ISD::CCMN: return "AArch64ISD::CCMN";
1226 case AArch64ISD::FCCMP: return "AArch64ISD::FCCMP";
1227 case AArch64ISD::FCMP: return "AArch64ISD::FCMP";
1228 case AArch64ISD::DUP: return "AArch64ISD::DUP";
1229 case AArch64ISD::DUPLANE8: return "AArch64ISD::DUPLANE8";
1230 case AArch64ISD::DUPLANE16: return "AArch64ISD::DUPLANE16";
1231 case AArch64ISD::DUPLANE32: return "AArch64ISD::DUPLANE32";
1232 case AArch64ISD::DUPLANE64: return "AArch64ISD::DUPLANE64";
1233 case AArch64ISD::MOVI: return "AArch64ISD::MOVI";
1234 case AArch64ISD::MOVIshift: return "AArch64ISD::MOVIshift";
1235 case AArch64ISD::MOVIedit: return "AArch64ISD::MOVIedit";
1236 case AArch64ISD::MOVImsl: return "AArch64ISD::MOVImsl";
1237 case AArch64ISD::FMOV: return "AArch64ISD::FMOV";
1238 case AArch64ISD::MVNIshift: return "AArch64ISD::MVNIshift";
1239 case AArch64ISD::MVNImsl: return "AArch64ISD::MVNImsl";
1240 case AArch64ISD::BICi: return "AArch64ISD::BICi";
1241 case AArch64ISD::ORRi: return "AArch64ISD::ORRi";
1242 case AArch64ISD::BSL: return "AArch64ISD::BSL";
1243 case AArch64ISD::NEG: return "AArch64ISD::NEG";
1244 case AArch64ISD::EXTR: return "AArch64ISD::EXTR";
1245 case AArch64ISD::ZIP1: return "AArch64ISD::ZIP1";
1246 case AArch64ISD::ZIP2: return "AArch64ISD::ZIP2";
1247 case AArch64ISD::UZP1: return "AArch64ISD::UZP1";
1248 case AArch64ISD::UZP2: return "AArch64ISD::UZP2";
1249 case AArch64ISD::TRN1: return "AArch64ISD::TRN1";
1250 case AArch64ISD::TRN2: return "AArch64ISD::TRN2";
1251 case AArch64ISD::REV16: return "AArch64ISD::REV16";
1252 case AArch64ISD::REV32: return "AArch64ISD::REV32";
1253 case AArch64ISD::REV64: return "AArch64ISD::REV64";
1254 case AArch64ISD::EXT: return "AArch64ISD::EXT";
1255 case AArch64ISD::VSHL: return "AArch64ISD::VSHL";
1256 case AArch64ISD::VLSHR: return "AArch64ISD::VLSHR";
1257 case AArch64ISD::VASHR: return "AArch64ISD::VASHR";
1258 case AArch64ISD::CMEQ: return "AArch64ISD::CMEQ";
1259 case AArch64ISD::CMGE: return "AArch64ISD::CMGE";
1260 case AArch64ISD::CMGT: return "AArch64ISD::CMGT";
1261 case AArch64ISD::CMHI: return "AArch64ISD::CMHI";
1262 case AArch64ISD::CMHS: return "AArch64ISD::CMHS";
1263 case AArch64ISD::FCMEQ: return "AArch64ISD::FCMEQ";
1264 case AArch64ISD::FCMGE: return "AArch64ISD::FCMGE";
1265 case AArch64ISD::FCMGT: return "AArch64ISD::FCMGT";
1266 case AArch64ISD::CMEQz: return "AArch64ISD::CMEQz";
1267 case AArch64ISD::CMGEz: return "AArch64ISD::CMGEz";
1268 case AArch64ISD::CMGTz: return "AArch64ISD::CMGTz";
1269 case AArch64ISD::CMLEz: return "AArch64ISD::CMLEz";
1270 case AArch64ISD::CMLTz: return "AArch64ISD::CMLTz";
1271 case AArch64ISD::FCMEQz: return "AArch64ISD::FCMEQz";
1272 case AArch64ISD::FCMGEz: return "AArch64ISD::FCMGEz";
1273 case AArch64ISD::FCMGTz: return "AArch64ISD::FCMGTz";
1274 case AArch64ISD::FCMLEz: return "AArch64ISD::FCMLEz";
1275 case AArch64ISD::FCMLTz: return "AArch64ISD::FCMLTz";
1276 case AArch64ISD::SADDV: return "AArch64ISD::SADDV";
1277 case AArch64ISD::UADDV: return "AArch64ISD::UADDV";
1278 case AArch64ISD::SMINV: return "AArch64ISD::SMINV";
1279 case AArch64ISD::UMINV: return "AArch64ISD::UMINV";
1280 case AArch64ISD::SMAXV: return "AArch64ISD::SMAXV";
1281 case AArch64ISD::UMAXV: return "AArch64ISD::UMAXV";
1282 case AArch64ISD::NOT: return "AArch64ISD::NOT";
1283 case AArch64ISD::BIT: return "AArch64ISD::BIT";
1284 case AArch64ISD::CBZ: return "AArch64ISD::CBZ";
1285 case AArch64ISD::CBNZ: return "AArch64ISD::CBNZ";
1286 case AArch64ISD::TBZ: return "AArch64ISD::TBZ";
1287 case AArch64ISD::TBNZ: return "AArch64ISD::TBNZ";
1288 case AArch64ISD::TC_RETURN: return "AArch64ISD::TC_RETURN";
1289 case AArch64ISD::PREFETCH: return "AArch64ISD::PREFETCH";
1290 case AArch64ISD::SITOF: return "AArch64ISD::SITOF";
1291 case AArch64ISD::UITOF: return "AArch64ISD::UITOF";
1292 case AArch64ISD::NVCAST: return "AArch64ISD::NVCAST";
1293 case AArch64ISD::SQSHL_I: return "AArch64ISD::SQSHL_I";
1294 case AArch64ISD::UQSHL_I: return "AArch64ISD::UQSHL_I";
1295 case AArch64ISD::SRSHR_I: return "AArch64ISD::SRSHR_I";
1296 case AArch64ISD::URSHR_I: return "AArch64ISD::URSHR_I";
1297 case AArch64ISD::SQSHLU_I: return "AArch64ISD::SQSHLU_I";
1298 case AArch64ISD::WrapperLarge: return "AArch64ISD::WrapperLarge";
1299 case AArch64ISD::LD2post: return "AArch64ISD::LD2post";
1300 case AArch64ISD::LD3post: return "AArch64ISD::LD3post";
1301 case AArch64ISD::LD4post: return "AArch64ISD::LD4post";
1302 case AArch64ISD::ST2post: return "AArch64ISD::ST2post";
1303 case AArch64ISD::ST3post: return "AArch64ISD::ST3post";
1304 case AArch64ISD::ST4post: return "AArch64ISD::ST4post";
1305 case AArch64ISD::LD1x2post: return "AArch64ISD::LD1x2post";
1306 case AArch64ISD::LD1x3post: return "AArch64ISD::LD1x3post";
1307 case AArch64ISD::LD1x4post: return "AArch64ISD::LD1x4post";
1308 case AArch64ISD::ST1x2post: return "AArch64ISD::ST1x2post";
1309 case AArch64ISD::ST1x3post: return "AArch64ISD::ST1x3post";
1310 case AArch64ISD::ST1x4post: return "AArch64ISD::ST1x4post";
1311 case AArch64ISD::LD1DUPpost: return "AArch64ISD::LD1DUPpost";
1312 case AArch64ISD::LD2DUPpost: return "AArch64ISD::LD2DUPpost";
1313 case AArch64ISD::LD3DUPpost: return "AArch64ISD::LD3DUPpost";
1314 case AArch64ISD::LD4DUPpost: return "AArch64ISD::LD4DUPpost";
1315 case AArch64ISD::LD1LANEpost: return "AArch64ISD::LD1LANEpost";
1316 case AArch64ISD::LD2LANEpost: return "AArch64ISD::LD2LANEpost";
1317 case AArch64ISD::LD3LANEpost: return "AArch64ISD::LD3LANEpost";
1318 case AArch64ISD::LD4LANEpost: return "AArch64ISD::LD4LANEpost";
1319 case AArch64ISD::ST2LANEpost: return "AArch64ISD::ST2LANEpost";
1320 case AArch64ISD::ST3LANEpost: return "AArch64ISD::ST3LANEpost";
1321 case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost";
1322 case AArch64ISD::SMULL: return "AArch64ISD::SMULL";
1323 case AArch64ISD::UMULL: return "AArch64ISD::UMULL";
1324 case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE";
1325 case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS";
1326 case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE";
1327 case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS";
1328 case AArch64ISD::STG: return "AArch64ISD::STG";
1329 case AArch64ISD::STZG: return "AArch64ISD::STZG";
1330 case AArch64ISD::ST2G: return "AArch64ISD::ST2G";
1331 case AArch64ISD::STZ2G: return "AArch64ISD::STZ2G";
1332 case AArch64ISD::SUNPKHI: return "AArch64ISD::SUNPKHI";
1333 case AArch64ISD::SUNPKLO: return "AArch64ISD::SUNPKLO";
1334 case AArch64ISD::UUNPKHI: return "AArch64ISD::UUNPKHI";
1335 case AArch64ISD::UUNPKLO: return "AArch64ISD::UUNPKLO";
1336 }
1337 return nullptr;
1338}
1339
1340MachineBasicBlock *
1341AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
1342 MachineBasicBlock *MBB) const {
1343 // We materialise the F128CSEL pseudo-instruction as some control flow and a
1344 // phi node:
1345
1346 // OrigBB:
1347 // [... previous instrs leading to comparison ...]
1348 // b.ne TrueBB
1349 // b EndBB
1350 // TrueBB:
1351 // ; Fallthrough
1352 // EndBB:
1353 // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
1354
1355 MachineFunction *MF = MBB->getParent();
1356 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1357 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
1358 DebugLoc DL = MI.getDebugLoc();
1359 MachineFunction::iterator It = ++MBB->getIterator();
1360
1361 Register DestReg = MI.getOperand(0).getReg();
1362 Register IfTrueReg = MI.getOperand(1).getReg();
1363 Register IfFalseReg = MI.getOperand(2).getReg();
1364 unsigned CondCode = MI.getOperand(3).getImm();
1365 bool NZCVKilled = MI.getOperand(4).isKill();
1366
1367 MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
1368 MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
1369 MF->insert(It, TrueBB);
1370 MF->insert(It, EndBB);
1371
1372 // Transfer rest of current basic-block to EndBB
1373 EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
1374 MBB->end());
1375 EndBB->transferSuccessorsAndUpdatePHIs(MBB);
1376
1377 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
1378 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
1379 MBB->addSuccessor(TrueBB);
1380 MBB->addSuccessor(EndBB);
1381
1382 // TrueBB falls through to the end.
1383 TrueBB->addSuccessor(EndBB);
1384
1385 if (!NZCVKilled) {
1386 TrueBB->addLiveIn(AArch64::NZCV);
1387 EndBB->addLiveIn(AArch64::NZCV);
1388 }
1389
1390 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
1391 .addReg(IfTrueReg)
1392 .addMBB(TrueBB)
1393 .addReg(IfFalseReg)
1394 .addMBB(MBB);
1395
1396 MI.eraseFromParent();
1397 return EndBB;
1398}
1399
1400MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
1401 MachineInstr &MI, MachineBasicBlock *BB) const {
1402 assert(!isAsynchronousEHPersonality(classifyEHPersonality(((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1404, __PRETTY_FUNCTION__))
1403 BB->getParent()->getFunction().getPersonalityFn())) &&((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1404, __PRETTY_FUNCTION__))
1404 "SEH does not use catchret!")((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1404, __PRETTY_FUNCTION__))
;
1405 return BB;
1406}
1407
1408MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchPad(
1409 MachineInstr &MI, MachineBasicBlock *BB) const {
1410 MI.eraseFromParent();
1411 return BB;
1412}
1413
1414MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
1415 MachineInstr &MI, MachineBasicBlock *BB) const {
1416 switch (MI.getOpcode()) {
1417 default:
1418#ifndef NDEBUG
1419 MI.dump();
1420#endif
1421 llvm_unreachable("Unexpected instruction for custom inserter!")::llvm::llvm_unreachable_internal("Unexpected instruction for custom inserter!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1421)
;
1422
1423 case AArch64::F128CSEL:
1424 return EmitF128CSEL(MI, BB);
1425
1426 case TargetOpcode::STACKMAP:
1427 case TargetOpcode::PATCHPOINT:
1428 return emitPatchPoint(MI, BB);
1429
1430 case AArch64::CATCHRET:
1431 return EmitLoweredCatchRet(MI, BB);
1432 case AArch64::CATCHPAD:
1433 return EmitLoweredCatchPad(MI, BB);
1434 }
1435}
1436
1437//===----------------------------------------------------------------------===//
1438// AArch64 Lowering private implementation.
1439//===----------------------------------------------------------------------===//
1440
1441//===----------------------------------------------------------------------===//
1442// Lowering Code
1443//===----------------------------------------------------------------------===//
1444
1445/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
1446/// CC
1447static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
1448 switch (CC) {
1449 default:
1450 llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1450)
;
1451 case ISD::SETNE:
1452 return AArch64CC::NE;
1453 case ISD::SETEQ:
1454 return AArch64CC::EQ;
1455 case ISD::SETGT:
1456 return AArch64CC::GT;
1457 case ISD::SETGE:
1458 return AArch64CC::GE;
1459 case ISD::SETLT:
1460 return AArch64CC::LT;
1461 case ISD::SETLE:
1462 return AArch64CC::LE;
1463 case ISD::SETUGT:
1464 return AArch64CC::HI;
1465 case ISD::SETUGE:
1466 return AArch64CC::HS;
1467 case ISD::SETULT:
1468 return AArch64CC::LO;
1469 case ISD::SETULE:
1470 return AArch64CC::LS;
1471 }
1472}
1473
1474/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
1475static void changeFPCCToAArch64CC(ISD::CondCode CC,
1476 AArch64CC::CondCode &CondCode,
1477 AArch64CC::CondCode &CondCode2) {
1478 CondCode2 = AArch64CC::AL;
1479 switch (CC) {
1480 default:
1481 llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1481)
;
1482 case ISD::SETEQ:
1483 case ISD::SETOEQ:
1484 CondCode = AArch64CC::EQ;
1485 break;
1486 case ISD::SETGT:
1487 case ISD::SETOGT:
1488 CondCode = AArch64CC::GT;
1489 break;
1490 case ISD::SETGE:
1491 case ISD::SETOGE:
1492 CondCode = AArch64CC::GE;
1493 break;
1494 case ISD::SETOLT:
1495 CondCode = AArch64CC::MI;
1496 break;
1497 case ISD::SETOLE:
1498 CondCode = AArch64CC::LS;
1499 break;
1500 case ISD::SETONE:
1501 CondCode = AArch64CC::MI;
1502 CondCode2 = AArch64CC::GT;
1503 break;
1504 case ISD::SETO:
1505 CondCode = AArch64CC::VC;
1506 break;
1507 case ISD::SETUO:
1508 CondCode = AArch64CC::VS;
1509 break;
1510 case ISD::SETUEQ:
1511 CondCode = AArch64CC::EQ;
1512 CondCode2 = AArch64CC::VS;
1513 break;
1514 case ISD::SETUGT:
1515 CondCode = AArch64CC::HI;
1516 break;
1517 case ISD::SETUGE:
1518 CondCode = AArch64CC::PL;
1519 break;
1520 case ISD::SETLT:
1521 case ISD::SETULT:
1522 CondCode = AArch64CC::LT;
1523 break;
1524 case ISD::SETLE:
1525 case ISD::SETULE:
1526 CondCode = AArch64CC::LE;
1527 break;
1528 case ISD::SETNE:
1529 case ISD::SETUNE:
1530 CondCode = AArch64CC::NE;
1531 break;
1532 }
1533}
1534
1535/// Convert a DAG fp condition code to an AArch64 CC.
1536/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1537/// should be AND'ed instead of OR'ed.
1538static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
1539 AArch64CC::CondCode &CondCode,
1540 AArch64CC::CondCode &CondCode2) {
1541 CondCode2 = AArch64CC::AL;
1542 switch (CC) {
1543 default:
1544 changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1545 assert(CondCode2 == AArch64CC::AL)((CondCode2 == AArch64CC::AL) ? static_cast<void> (0) :
__assert_fail ("CondCode2 == AArch64CC::AL", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1545, __PRETTY_FUNCTION__))
;
1546 break;
1547 case ISD::SETONE:
1548 // (a one b)
1549 // == ((a olt b) || (a ogt b))
1550 // == ((a ord b) && (a une b))
1551 CondCode = AArch64CC::VC;
1552 CondCode2 = AArch64CC::NE;
1553 break;
1554 case ISD::SETUEQ:
1555 // (a ueq b)
1556 // == ((a uno b) || (a oeq b))
1557 // == ((a ule b) && (a uge b))
1558 CondCode = AArch64CC::PL;
1559 CondCode2 = AArch64CC::LE;
1560 break;
1561 }
1562}
1563
1564/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
1565/// CC usable with the vector instructions. Fewer operations are available
1566/// without a real NZCV register, so we have to use less efficient combinations
1567/// to get the same effect.
1568static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
1569 AArch64CC::CondCode &CondCode,
1570 AArch64CC::CondCode &CondCode2,
1571 bool &Invert) {
1572 Invert = false;
1573 switch (CC) {
1574 default:
1575 // Mostly the scalar mappings work fine.
1576 changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1577 break;
1578 case ISD::SETUO:
1579 Invert = true;
1580 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1581 case ISD::SETO:
1582 CondCode = AArch64CC::MI;
1583 CondCode2 = AArch64CC::GE;
1584 break;
1585 case ISD::SETUEQ:
1586 case ISD::SETULT:
1587 case ISD::SETULE:
1588 case ISD::SETUGT:
1589 case ISD::SETUGE:
1590 // All of the compare-mask comparisons are ordered, but we can switch
1591 // between the two by a double inversion. E.g. ULE == !OGT.
1592 Invert = true;
1593 changeFPCCToAArch64CC(getSetCCInverse(CC, false), CondCode, CondCode2);
1594 break;
1595 }
1596}
1597
1598static bool isLegalArithImmed(uint64_t C) {
1599 // Matches AArch64DAGToDAGISel::SelectArithImmed().
1600 bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
1601 LLVM_DEBUG(dbgs() << "Is imm " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Is imm " << C <<
" legal: " << (IsLegal ? "yes\n" : "no\n"); } } while (
false)
1602 << " legal: " << (IsLegal ? "yes\n" : "no\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Is imm " << C <<
" legal: " << (IsLegal ? "yes\n" : "no\n"); } } while (
false)
;
1603 return IsLegal;
1604}
1605
1606// Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
1607// the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
1608// can be set differently by this operation. It comes down to whether
1609// "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
1610// everything is fine. If not then the optimization is wrong. Thus general
1611// comparisons are only valid if op2 != 0.
1612//
1613// So, finally, the only LLVM-native comparisons that don't mention C and V
1614// are SETEQ and SETNE. They're the only ones we can safely use CMN for in
1615// the absence of information about op2.
1616static bool isCMN(SDValue Op, ISD::CondCode CC) {
1617 return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
1618 (CC == ISD::SETEQ || CC == ISD::SETNE);
1619}
1620
1621static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1622 const SDLoc &dl, SelectionDAG &DAG) {
1623 EVT VT = LHS.getValueType();
1624 const bool FullFP16 =
1625 static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1626
1627 if (VT.isFloatingPoint()) {
1628 assert(VT != MVT::f128)((VT != MVT::f128) ? static_cast<void> (0) : __assert_fail
("VT != MVT::f128", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1628, __PRETTY_FUNCTION__))
;
1629 if (VT == MVT::f16 && !FullFP16) {
1630 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
1631 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
1632 VT = MVT::f32;
1633 }
1634 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
1635 }
1636
1637 // The CMP instruction is just an alias for SUBS, and representing it as
1638 // SUBS means that it's possible to get CSE with subtract operations.
1639 // A later phase can perform the optimization of setting the destination
1640 // register to WZR/XZR if it ends up being unused.
1641 unsigned Opcode = AArch64ISD::SUBS;
1642
1643 if (isCMN(RHS, CC)) {
1644 // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
1645 Opcode = AArch64ISD::ADDS;
1646 RHS = RHS.getOperand(1);
1647 } else if (isCMN(LHS, CC)) {
1648 // As we are looking for EQ/NE compares, the operands can be commuted ; can
1649 // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
1650 Opcode = AArch64ISD::ADDS;
1651 LHS = LHS.getOperand(1);
1652 } else if (LHS.getOpcode() == ISD::AND && isNullConstant(RHS) &&
1653 !isUnsignedIntSetCC(CC)) {
1654 // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
1655 // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
1656 // of the signed comparisons.
1657 Opcode = AArch64ISD::ANDS;
1658 RHS = LHS.getOperand(1);
1659 LHS = LHS.getOperand(0);
1660 }
1661
1662 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
1663 .getValue(1);
1664}
1665
1666/// \defgroup AArch64CCMP CMP;CCMP matching
1667///
1668/// These functions deal with the formation of CMP;CCMP;... sequences.
1669/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
1670/// a comparison. They set the NZCV flags to a predefined value if their
1671/// predicate is false. This allows to express arbitrary conjunctions, for
1672/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
1673/// expressed as:
1674/// cmp A
1675/// ccmp B, inv(CB), CA
1676/// check for CB flags
1677///
1678/// This naturally lets us implement chains of AND operations with SETCC
1679/// operands. And we can even implement some other situations by transforming
1680/// them:
1681/// - We can implement (NEG SETCC) i.e. negating a single comparison by
1682/// negating the flags used in a CCMP/FCCMP operations.
1683/// - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
1684/// by negating the flags we test for afterwards. i.e.
1685/// NEG (CMP CCMP CCCMP ...) can be implemented.
1686/// - Note that we can only ever negate all previously processed results.
1687/// What we can not implement by flipping the flags to test is a negation
1688/// of two sub-trees (because the negation affects all sub-trees emitted so
1689/// far, so the 2nd sub-tree we emit would also affect the first).
1690/// With those tools we can implement some OR operations:
1691/// - (OR (SETCC A) (SETCC B)) can be implemented via:
1692/// NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
1693/// - After transforming OR to NEG/AND combinations we may be able to use NEG
1694/// elimination rules from earlier to implement the whole thing as a
1695/// CCMP/FCCMP chain.
1696///
1697/// As complete example:
1698/// or (or (setCA (cmp A)) (setCB (cmp B)))
1699/// (and (setCC (cmp C)) (setCD (cmp D)))"
1700/// can be reassociated to:
1701/// or (and (setCC (cmp C)) setCD (cmp D))
1702// (or (setCA (cmp A)) (setCB (cmp B)))
1703/// can be transformed to:
1704/// not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
1705/// (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
1706/// which can be implemented as:
1707/// cmp C
1708/// ccmp D, inv(CD), CC
1709/// ccmp A, CA, inv(CD)
1710/// ccmp B, CB, inv(CA)
1711/// check for CB flags
1712///
1713/// A counterexample is "or (and A B) (and C D)" which translates to
1714/// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
1715/// can only implement 1 of the inner (not) operations, but not both!
1716/// @{
1717
1718/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
1719static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1720 ISD::CondCode CC, SDValue CCOp,
1721 AArch64CC::CondCode Predicate,
1722 AArch64CC::CondCode OutCC,
1723 const SDLoc &DL, SelectionDAG &DAG) {
1724 unsigned Opcode = 0;
1725 const bool FullFP16 =
1726 static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1727
1728 if (LHS.getValueType().isFloatingPoint()) {
1729 assert(LHS.getValueType() != MVT::f128)((LHS.getValueType() != MVT::f128) ? static_cast<void> (
0) : __assert_fail ("LHS.getValueType() != MVT::f128", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1729, __PRETTY_FUNCTION__))
;
1730 if (LHS.getValueType() == MVT::f16 && !FullFP16) {
1731 LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
1732 RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
1733 }
1734 Opcode = AArch64ISD::FCCMP;
1735 } else if (RHS.getOpcode() == ISD::SUB) {
1736 SDValue SubOp0 = RHS.getOperand(0);
1737 if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
1738 // See emitComparison() on why we can only do this for SETEQ and SETNE.
1739 Opcode = AArch64ISD::CCMN;
1740 RHS = RHS.getOperand(1);
1741 }
1742 }
1743 if (Opcode == 0)
1744 Opcode = AArch64ISD::CCMP;
1745
1746 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
1747 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
1748 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
1749 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
1750 return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
1751}
1752
1753/// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
1754/// expressed as a conjunction. See \ref AArch64CCMP.
1755/// \param CanNegate Set to true if we can negate the whole sub-tree just by
1756/// changing the conditions on the SETCC tests.
1757/// (this means we can call emitConjunctionRec() with
1758/// Negate==true on this sub-tree)
1759/// \param MustBeFirst Set to true if this subtree needs to be negated and we
1760/// cannot do the negation naturally. We are required to
1761/// emit the subtree first in this case.
1762/// \param WillNegate Is true if are called when the result of this
1763/// subexpression must be negated. This happens when the
1764/// outer expression is an OR. We can use this fact to know
1765/// that we have a double negation (or (or ...) ...) that
1766/// can be implemented for free.
1767static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
1768 bool &MustBeFirst, bool WillNegate,
1769 unsigned Depth = 0) {
1770 if (!Val.hasOneUse())
1771 return false;
1772 unsigned Opcode = Val->getOpcode();
1773 if (Opcode == ISD::SETCC) {
1774 if (Val->getOperand(0).getValueType() == MVT::f128)
1775 return false;
1776 CanNegate = true;
1777 MustBeFirst = false;
1778 return true;
1779 }
1780 // Protect against exponential runtime and stack overflow.
1781 if (Depth > 6)
1782 return false;
1783 if (Opcode == ISD::AND || Opcode == ISD::OR) {
1784 bool IsOR = Opcode == ISD::OR;
1785 SDValue O0 = Val->getOperand(0);
1786 SDValue O1 = Val->getOperand(1);
1787 bool CanNegateL;
1788 bool MustBeFirstL;
1789 if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
1790 return false;
1791 bool CanNegateR;
1792 bool MustBeFirstR;
1793 if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
1794 return false;
1795
1796 if (MustBeFirstL && MustBeFirstR)
1797 return false;
1798
1799 if (IsOR) {
1800 // For an OR expression we need to be able to naturally negate at least
1801 // one side or we cannot do the transformation at all.
1802 if (!CanNegateL && !CanNegateR)
1803 return false;
1804 // If we the result of the OR will be negated and we can naturally negate
1805 // the leafs, then this sub-tree as a whole negates naturally.
1806 CanNegate = WillNegate && CanNegateL && CanNegateR;
1807 // If we cannot naturally negate the whole sub-tree, then this must be
1808 // emitted first.
1809 MustBeFirst = !CanNegate;
1810 } else {
1811 assert(Opcode == ISD::AND && "Must be OR or AND")((Opcode == ISD::AND && "Must be OR or AND") ? static_cast
<void> (0) : __assert_fail ("Opcode == ISD::AND && \"Must be OR or AND\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1811, __PRETTY_FUNCTION__))
;
1812 // We cannot naturally negate an AND operation.
1813 CanNegate = false;
1814 MustBeFirst = MustBeFirstL || MustBeFirstR;
1815 }
1816 return true;
1817 }
1818 return false;
1819}
1820
1821/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
1822/// of CCMP/CFCMP ops. See @ref AArch64CCMP.
1823/// Tries to transform the given i1 producing node @p Val to a series compare
1824/// and conditional compare operations. @returns an NZCV flags producing node
1825/// and sets @p OutCC to the flags that should be tested or returns SDValue() if
1826/// transformation was not possible.
1827/// \p Negate is true if we want this sub-tree being negated just by changing
1828/// SETCC conditions.
1829static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
1830 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
1831 AArch64CC::CondCode Predicate) {
1832 // We're at a tree leaf, produce a conditional comparison operation.
1833 unsigned Opcode = Val->getOpcode();
1834 if (Opcode == ISD::SETCC) {
1835 SDValue LHS = Val->getOperand(0);
1836 SDValue RHS = Val->getOperand(1);
1837 ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
1838 bool isInteger = LHS.getValueType().isInteger();
1839 if (Negate)
1840 CC = getSetCCInverse(CC, isInteger);
1841 SDLoc DL(Val);
1842 // Determine OutCC and handle FP special case.
1843 if (isInteger) {
1844 OutCC = changeIntCCToAArch64CC(CC);
1845 } else {
1846 assert(LHS.getValueType().isFloatingPoint())((LHS.getValueType().isFloatingPoint()) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType().isFloatingPoint()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1846, __PRETTY_FUNCTION__))
;
1847 AArch64CC::CondCode ExtraCC;
1848 changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
1849 // Some floating point conditions can't be tested with a single condition
1850 // code. Construct an additional comparison in this case.
1851 if (ExtraCC != AArch64CC::AL) {
1852 SDValue ExtraCmp;
1853 if (!CCOp.getNode())
1854 ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
1855 else
1856 ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
1857 ExtraCC, DL, DAG);
1858 CCOp = ExtraCmp;
1859 Predicate = ExtraCC;
1860 }
1861 }
1862
1863 // Produce a normal comparison if we are first in the chain
1864 if (!CCOp)
1865 return emitComparison(LHS, RHS, CC, DL, DAG);
1866 // Otherwise produce a ccmp.
1867 return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
1868 DAG);
1869 }
1870 assert(Val->hasOneUse() && "Valid conjunction/disjunction tree")((Val->hasOneUse() && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("Val->hasOneUse() && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1870, __PRETTY_FUNCTION__))
;
1871
1872 bool IsOR = Opcode == ISD::OR;
1873
1874 SDValue LHS = Val->getOperand(0);
1875 bool CanNegateL;
1876 bool MustBeFirstL;
1877 bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
1878 assert(ValidL && "Valid conjunction/disjunction tree")((ValidL && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("ValidL && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1878, __PRETTY_FUNCTION__))
;
1879 (void)ValidL;
1880
1881 SDValue RHS = Val->getOperand(1);
1882 bool CanNegateR;
1883 bool MustBeFirstR;
1884 bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
1885 assert(ValidR && "Valid conjunction/disjunction tree")((ValidR && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("ValidR && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1885, __PRETTY_FUNCTION__))
;
1886 (void)ValidR;
1887
1888 // Swap sub-tree that must come first to the right side.
1889 if (MustBeFirstL) {
1890 assert(!MustBeFirstR && "Valid conjunction/disjunction tree")((!MustBeFirstR && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("!MustBeFirstR && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1890, __PRETTY_FUNCTION__))
;
1891 std::swap(LHS, RHS);
1892 std::swap(CanNegateL, CanNegateR);
1893 std::swap(MustBeFirstL, MustBeFirstR);
1894 }
1895
1896 bool NegateR;
1897 bool NegateAfterR;
1898 bool NegateL;
1899 bool NegateAfterAll;
1900 if (Opcode == ISD::OR) {
1901 // Swap the sub-tree that we can negate naturally to the left.
1902 if (!CanNegateL) {
1903 assert(CanNegateR && "at least one side must be negatable")((CanNegateR && "at least one side must be negatable"
) ? static_cast<void> (0) : __assert_fail ("CanNegateR && \"at least one side must be negatable\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1903, __PRETTY_FUNCTION__))
;
1904 assert(!MustBeFirstR && "invalid conjunction/disjunction tree")((!MustBeFirstR && "invalid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("!MustBeFirstR && \"invalid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1904, __PRETTY_FUNCTION__))
;
1905 assert(!Negate)((!Negate) ? static_cast<void> (0) : __assert_fail ("!Negate"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1905, __PRETTY_FUNCTION__))
;
1906 std::swap(LHS, RHS);
1907 NegateR = false;
1908 NegateAfterR = true;
1909 } else {
1910 // Negate the left sub-tree if possible, otherwise negate the result.
1911 NegateR = CanNegateR;
1912 NegateAfterR = !CanNegateR;
1913 }
1914 NegateL = true;
1915 NegateAfterAll = !Negate;
1916 } else {
1917 assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree")((Opcode == ISD::AND && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("Opcode == ISD::AND && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1917, __PRETTY_FUNCTION__))
;
1918 assert(!Negate && "Valid conjunction/disjunction tree")((!Negate && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("!Negate && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1918, __PRETTY_FUNCTION__))
;
1919
1920 NegateL = false;
1921 NegateR = false;
1922 NegateAfterR = false;
1923 NegateAfterAll = false;
1924 }
1925
1926 // Emit sub-trees.
1927 AArch64CC::CondCode RHSCC;
1928 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
1929 if (NegateAfterR)
1930 RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
1931 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
1932 if (NegateAfterAll)
1933 OutCC = AArch64CC::getInvertedCondCode(OutCC);
1934 return CmpL;
1935}
1936
1937/// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
1938/// In some cases this is even possible with OR operations in the expression.
1939/// See \ref AArch64CCMP.
1940/// \see emitConjunctionRec().
1941static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
1942 AArch64CC::CondCode &OutCC) {
1943 bool DummyCanNegate;
1944 bool DummyMustBeFirst;
1945 if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
1946 return SDValue();
1947
1948 return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
1949}
1950
1951/// @}
1952
1953/// Returns how profitable it is to fold a comparison's operand's shift and/or
1954/// extension operations.
1955static unsigned getCmpOperandFoldingProfit(SDValue Op) {
1956 auto isSupportedExtend = [&](SDValue V) {
1957 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
1958 return true;
1959
1960 if (V.getOpcode() == ISD::AND)
1961 if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
1962 uint64_t Mask = MaskCst->getZExtValue();
1963 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
1964 }
1965
1966 return false;
1967 };
1968
1969 if (!Op.hasOneUse())
1970 return 0;
1971
1972 if (isSupportedExtend(Op))
1973 return 1;
1974
1975 unsigned Opc = Op.getOpcode();
1976 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
1977 if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1978 uint64_t Shift = ShiftCst->getZExtValue();
1979 if (isSupportedExtend(Op.getOperand(0)))
1980 return (Shift <= 4) ? 2 : 1;
1981 EVT VT = Op.getValueType();
1982 if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
1983 return 1;
1984 }
1985
1986 return 0;
1987}
1988
1989static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1990 SDValue &AArch64cc, SelectionDAG &DAG,
1991 const SDLoc &dl) {
1992 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
1993 EVT VT = RHS.getValueType();
1994 uint64_t C = RHSC->getZExtValue();
1995 if (!isLegalArithImmed(C)) {
1996 // Constant does not fit, try adjusting it by one?
1997 switch (CC) {
1998 default:
1999 break;
2000 case ISD::SETLT:
2001 case ISD::SETGE:
2002 if ((VT == MVT::i32 && C != 0x80000000 &&
2003 isLegalArithImmed((uint32_t)(C - 1))) ||
2004 (VT == MVT::i64 && C != 0x80000000ULL &&
2005 isLegalArithImmed(C - 1ULL))) {
2006 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
2007 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2008 RHS = DAG.getConstant(C, dl, VT);
2009 }
2010 break;
2011 case ISD::SETULT:
2012 case ISD::SETUGE:
2013 if ((VT == MVT::i32 && C != 0 &&
2014 isLegalArithImmed((uint32_t)(C - 1))) ||
2015 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
2016 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
2017 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2018 RHS = DAG.getConstant(C, dl, VT);
2019 }
2020 break;
2021 case ISD::SETLE:
2022 case ISD::SETGT:
2023 if ((VT == MVT::i32 && C != INT32_MAX(2147483647) &&
2024 isLegalArithImmed((uint32_t)(C + 1))) ||
2025 (VT == MVT::i64 && C != INT64_MAX(9223372036854775807L) &&
2026 isLegalArithImmed(C + 1ULL))) {
2027 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
2028 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2029 RHS = DAG.getConstant(C, dl, VT);
2030 }
2031 break;
2032 case ISD::SETULE:
2033 case ISD::SETUGT:
2034 if ((VT == MVT::i32 && C != UINT32_MAX(4294967295U) &&
2035 isLegalArithImmed((uint32_t)(C + 1))) ||
2036 (VT == MVT::i64 && C != UINT64_MAX(18446744073709551615UL) &&
2037 isLegalArithImmed(C + 1ULL))) {
2038 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2039 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2040 RHS = DAG.getConstant(C, dl, VT);
2041 }
2042 break;
2043 }
2044 }
2045 }
2046
2047 // Comparisons are canonicalized so that the RHS operand is simpler than the
2048 // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2049 // can fold some shift+extend operations on the RHS operand, so swap the
2050 // operands if that can be done.
2051 //
2052 // For example:
2053 // lsl w13, w11, #1
2054 // cmp w13, w12
2055 // can be turned into:
2056 // cmp w12, w11, lsl #1
2057 if (!isa<ConstantSDNode>(RHS) ||
2058 !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
2059 SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2060
2061 if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
2062 std::swap(LHS, RHS);
2063 CC = ISD::getSetCCSwappedOperands(CC);
2064 }
2065 }
2066
2067 SDValue Cmp;
2068 AArch64CC::CondCode AArch64CC;
2069 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
2070 const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
2071
2072 // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
2073 // For the i8 operand, the largest immediate is 255, so this can be easily
2074 // encoded in the compare instruction. For the i16 operand, however, the
2075 // largest immediate cannot be encoded in the compare.
2076 // Therefore, use a sign extending load and cmn to avoid materializing the
2077 // -1 constant. For example,
2078 // movz w1, #65535
2079 // ldrh w0, [x0, #0]
2080 // cmp w0, w1
2081 // >
2082 // ldrsh w0, [x0, #0]
2083 // cmn w0, #1
2084 // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
2085 // if and only if (sext LHS) == (sext RHS). The checks are in place to
2086 // ensure both the LHS and RHS are truly zero extended and to make sure the
2087 // transformation is profitable.
2088 if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
2089 cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
2090 cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
2091 LHS.getNode()->hasNUsesOfValue(1, 0)) {
2092 int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
2093 if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
2094 SDValue SExt =
2095 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
2096 DAG.getValueType(MVT::i16));
2097 Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
2098 RHS.getValueType()),
2099 CC, dl, DAG);
2100 AArch64CC = changeIntCCToAArch64CC(CC);
2101 }
2102 }
2103
2104 if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
2105 if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
2106 if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
2107 AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
2108 }
2109 }
2110 }
2111
2112 if (!Cmp) {
2113 Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2114 AArch64CC = changeIntCCToAArch64CC(CC);
2115 }
2116 AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
2117 return Cmp;
2118}
2119
2120static std::pair<SDValue, SDValue>
2121getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2122 assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&(((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::
i64) && "Unsupported value type") ? static_cast<void
> (0) : __assert_fail ("(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && \"Unsupported value type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2123, __PRETTY_FUNCTION__))
2123 "Unsupported value type")(((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::
i64) && "Unsupported value type") ? static_cast<void
> (0) : __assert_fail ("(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && \"Unsupported value type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2123, __PRETTY_FUNCTION__))
;
2124 SDValue Value, Overflow;
2125 SDLoc DL(Op);
2126 SDValue LHS = Op.getOperand(0);
2127 SDValue RHS = Op.getOperand(1);
2128 unsigned Opc = 0;
2129 switch (Op.getOpcode()) {
2130 default:
2131 llvm_unreachable("Unknown overflow instruction!")::llvm::llvm_unreachable_internal("Unknown overflow instruction!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2131)
;
2132 case ISD::SADDO:
2133 Opc = AArch64ISD::ADDS;
2134 CC = AArch64CC::VS;
2135 break;
2136 case ISD::UADDO:
2137 Opc = AArch64ISD::ADDS;
2138 CC = AArch64CC::HS;
2139 break;
2140 case ISD::SSUBO:
2141 Opc = AArch64ISD::SUBS;
2142 CC = AArch64CC::VS;
2143 break;
2144 case ISD::USUBO:
2145 Opc = AArch64ISD::SUBS;
2146 CC = AArch64CC::LO;
2147 break;
2148 // Multiply needs a little bit extra work.
2149 case ISD::SMULO:
2150 case ISD::UMULO: {
2151 CC = AArch64CC::NE;
2152 bool IsSigned = Op.getOpcode() == ISD::SMULO;
2153 if (Op.getValueType() == MVT::i32) {
2154 unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2155 // For a 32 bit multiply with overflow check we want the instruction
2156 // selector to generate a widening multiply (SMADDL/UMADDL). For that we
2157 // need to generate the following pattern:
2158 // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
2159 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2160 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2161 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2162 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2163 DAG.getConstant(0, DL, MVT::i64));
2164 // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2165 // operation. We need to clear out the upper 32 bits, because we used a
2166 // widening multiply that wrote all 64 bits. In the end this should be a
2167 // noop.
2168 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2169 if (IsSigned) {
2170 // The signed overflow check requires more than just a simple check for
2171 // any bit set in the upper 32 bits of the result. These bits could be
2172 // just the sign bits of a negative number. To perform the overflow
2173 // check we have to arithmetic shift right the 32nd bit of the result by
2174 // 31 bits. Then we compare the result to the upper 32 bits.
2175 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2176 DAG.getConstant(32, DL, MVT::i64));
2177 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2178 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2179 DAG.getConstant(31, DL, MVT::i64));
2180 // It is important that LowerBits is last, otherwise the arithmetic
2181 // shift will not be folded into the compare (SUBS).
2182 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2183 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2184 .getValue(1);
2185 } else {
2186 // The overflow check for unsigned multiply is easy. We only need to
2187 // check if any of the upper 32 bits are set. This can be done with a
2188 // CMP (shifted register). For that we need to generate the following
2189 // pattern:
2190 // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2191 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2192 DAG.getConstant(32, DL, MVT::i64));
2193 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2194 Overflow =
2195 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2196 DAG.getConstant(0, DL, MVT::i64),
2197 UpperBits).getValue(1);
2198 }
2199 break;
2200 }
2201 assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type")((Op.getValueType() == MVT::i64 && "Expected an i64 value type"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == MVT::i64 && \"Expected an i64 value type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2201, __PRETTY_FUNCTION__))
;
2202 // For the 64 bit multiply
2203 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2204 if (IsSigned) {
2205 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2206 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2207 DAG.getConstant(63, DL, MVT::i64));
2208 // It is important that LowerBits is last, otherwise the arithmetic
2209 // shift will not be folded into the compare (SUBS).
2210 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2211 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2212 .getValue(1);
2213 } else {
2214 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2215 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2216 Overflow =
2217 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2218 DAG.getConstant(0, DL, MVT::i64),
2219 UpperBits).getValue(1);
2220 }
2221 break;
2222 }
2223 } // switch (...)
2224
2225 if (Opc) {
2226 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2227
2228 // Emit the AArch64 operation with overflow check.
2229 Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2230 Overflow = Value.getValue(1);
2231 }
2232 return std::make_pair(Value, Overflow);
2233}
2234
2235SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2236 RTLIB::Libcall Call) const {
2237 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2238 MakeLibCallOptions CallOptions;
2239 return makeLibCall(DAG, Call, MVT::f128, Ops, CallOptions, SDLoc(Op)).first;
2240}
2241
2242// Returns true if the given Op is the overflow flag result of an overflow
2243// intrinsic operation.
2244static bool isOverflowIntrOpRes(SDValue Op) {
2245 unsigned Opc = Op.getOpcode();
2246 return (Op.getResNo() == 1 &&
2247 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2248 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2249}
2250
2251static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2252 SDValue Sel = Op.getOperand(0);
2253 SDValue Other = Op.getOperand(1);
2254 SDLoc dl(Sel);
2255
2256 // If the operand is an overflow checking operation, invert the condition
2257 // code and kill the Not operation. I.e., transform:
2258 // (xor (overflow_op_bool, 1))
2259 // -->
2260 // (csel 1, 0, invert(cc), overflow_op_bool)
2261 // ... which later gets transformed to just a cset instruction with an
2262 // inverted condition code, rather than a cset + eor sequence.
2263 if (isOneConstant(Other) && isOverflowIntrOpRes(Sel)) {
2264 // Only lower legal XALUO ops.
2265 if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
2266 return SDValue();
2267
2268 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2269 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2270 AArch64CC::CondCode CC;
2271 SDValue Value, Overflow;
2272 std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
2273 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2274 return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
2275 CCVal, Overflow);
2276 }
2277 // If neither operand is a SELECT_CC, give up.
2278 if (Sel.getOpcode() != ISD::SELECT_CC)
2279 std::swap(Sel, Other);
2280 if (Sel.getOpcode() != ISD::SELECT_CC)
2281 return Op;
2282
2283 // The folding we want to perform is:
2284 // (xor x, (select_cc a, b, cc, 0, -1) )
2285 // -->
2286 // (csel x, (xor x, -1), cc ...)
2287 //
2288 // The latter will get matched to a CSINV instruction.
2289
2290 ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
2291 SDValue LHS = Sel.getOperand(0);
2292 SDValue RHS = Sel.getOperand(1);
2293 SDValue TVal = Sel.getOperand(2);
2294 SDValue FVal = Sel.getOperand(3);
2295
2296 // FIXME: This could be generalized to non-integer comparisons.
2297 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
2298 return Op;
2299
2300 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
2301 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
2302
2303 // The values aren't constants, this isn't the pattern we're looking for.
2304 if (!CFVal || !CTVal)
2305 return Op;
2306
2307 // We can commute the SELECT_CC by inverting the condition. This
2308 // might be needed to make this fit into a CSINV pattern.
2309 if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
2310 std::swap(TVal, FVal);
2311 std::swap(CTVal, CFVal);
2312 CC = ISD::getSetCCInverse(CC, true);
2313 }
2314
2315 // If the constants line up, perform the transform!
2316 if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
2317 SDValue CCVal;
2318 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2319
2320 FVal = Other;
2321 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
2322 DAG.getConstant(-1ULL, dl, Other.getValueType()));
2323
2324 return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
2325 CCVal, Cmp);
2326 }
2327
2328 return Op;
2329}
2330
2331static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2332 EVT VT = Op.getValueType();
2333
2334 // Let legalize expand this if it isn't a legal type yet.
2335 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
2336 return SDValue();
2337
2338 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
2339
2340 unsigned Opc;
2341 bool ExtraOp = false;
2342 switch (Op.getOpcode()) {
2343 default:
2344 llvm_unreachable("Invalid code")::llvm::llvm_unreachable_internal("Invalid code", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2344)
;
2345 case ISD::ADDC:
2346 Opc = AArch64ISD::ADDS;
2347 break;
2348 case ISD::SUBC:
2349 Opc = AArch64ISD::SUBS;
2350 break;
2351 case ISD::ADDE:
2352 Opc = AArch64ISD::ADCS;
2353 ExtraOp = true;
2354 break;
2355 case ISD::SUBE:
2356 Opc = AArch64ISD::SBCS;
2357 ExtraOp = true;
2358 break;
2359 }
2360
2361 if (!ExtraOp)
2362 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
2363 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
2364 Op.getOperand(2));
2365}
2366
2367static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2368 // Let legalize expand this if it isn't a legal type yet.
2369 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
2370 return SDValue();
2371
2372 SDLoc dl(Op);
2373 AArch64CC::CondCode CC;
2374 // The actual operation that sets the overflow or carry flag.
2375 SDValue Value, Overflow;
2376 std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
2377
2378 // We use 0 and 1 as false and true values.
2379 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2380 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2381
2382 // We use an inverted condition, because the conditional select is inverted
2383 // too. This will allow it to be selected to a single instruction:
2384 // CSINC Wd, WZR, WZR, invert(cond).
2385 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2386 Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
2387 CCVal, Overflow);
2388
2389 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
2390 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
2391}
2392
2393// Prefetch operands are:
2394// 1: Address to prefetch
2395// 2: bool isWrite
2396// 3: int locality (0 = no locality ... 3 = extreme locality)
2397// 4: bool isDataCache
2398static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2399 SDLoc DL(Op);
2400 unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
2401 unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
2402 unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
2403
2404 bool IsStream = !Locality;
2405 // When the locality number is set
2406 if (Locality) {
2407 // The front-end should have filtered out the out-of-range values
2408 assert(Locality <= 3 && "Prefetch locality out-of-range")((Locality <= 3 && "Prefetch locality out-of-range"
) ? static_cast<void> (0) : __assert_fail ("Locality <= 3 && \"Prefetch locality out-of-range\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2408, __PRETTY_FUNCTION__))
;
2409 // The locality degree is the opposite of the cache speed.
2410 // Put the number the other way around.
2411 // The encoding starts at 0 for level 1
2412 Locality = 3 - Locality;
2413 }
2414
2415 // built the mask value encoding the expected behavior.
2416 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
2417 (!IsData << 3) | // IsDataCache bit
2418 (Locality << 1) | // Cache level bits
2419 (unsigned)IsStream; // Stream bit
2420 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
2421 DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
2422}
2423
2424SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2425 SelectionDAG &DAG) const {
2426 assert(Op.getValueType() == MVT::f128 && "Unexpected lowering")((Op.getValueType() == MVT::f128 && "Unexpected lowering"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == MVT::f128 && \"Unexpected lowering\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2426, __PRETTY_FUNCTION__))
;
2427
2428 RTLIB::Libcall LC;
2429 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
2430
2431 return LowerF128Call(Op, DAG, LC);
2432}
2433
2434SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2435 SelectionDAG &DAG) const {
2436 if (Op.getOperand(0).getValueType() != MVT::f128) {
2437 // It's legal except when f128 is involved
2438 return Op;
2439 }
2440
2441 RTLIB::Libcall LC;
2442 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
2443
2444 // FP_ROUND node has a second operand indicating whether it is known to be
2445 // precise. That doesn't take part in the LibCall so we can't directly use
2446 // LowerF128Call.
2447 SDValue SrcVal = Op.getOperand(0);
2448 MakeLibCallOptions CallOptions;
2449 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, CallOptions,
2450 SDLoc(Op)).first;
2451}
2452
2453SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
2454 SelectionDAG &DAG) const {
2455 // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2456 // Any additional optimization in this function should be recorded
2457 // in the cost tables.
2458 EVT InVT = Op.getOperand(0).getValueType();
2459 EVT VT = Op.getValueType();
2460 unsigned NumElts = InVT.getVectorNumElements();
2461
2462 // f16 conversions are promoted to f32 when full fp16 is not supported.
2463 if (InVT.getVectorElementType() == MVT::f16 &&
2464 !Subtarget->hasFullFP16()) {
2465 MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
2466 SDLoc dl(Op);
2467 return DAG.getNode(
2468 Op.getOpcode(), dl, Op.getValueType(),
2469 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
2470 }
2471
2472 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2473 SDLoc dl(Op);
2474 SDValue Cv =
2475 DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
2476 Op.getOperand(0));
2477 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
2478 }
2479
2480 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2481 SDLoc dl(Op);
2482 MVT ExtVT =
2483 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
2484 VT.getVectorNumElements());
2485 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
2486 return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
2487 }
2488
2489 // Type changing conversions are illegal.
2490 return Op;
2491}
2492
2493SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2494 SelectionDAG &DAG) const {
2495 if (Op.getOperand(0).getValueType().isVector())
2496 return LowerVectorFP_TO_INT(Op, DAG);
2497
2498 // f16 conversions are promoted to f32 when full fp16 is not supported.
2499 if (Op.getOperand(0).getValueType() == MVT::f16 &&
2500 !Subtarget->hasFullFP16()) {
2501 SDLoc dl(Op);
2502 return DAG.getNode(
2503 Op.getOpcode(), dl, Op.getValueType(),
2504 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0)));
2505 }
2506
2507 if (Op.getOperand(0).getValueType() != MVT::f128) {
2508 // It's legal except when f128 is involved
2509 return Op;
2510 }
2511
2512 RTLIB::Libcall LC;
2513 if (Op.getOpcode() == ISD::FP_TO_SINT)
2514 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType());
2515 else
2516 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType());
2517
2518 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2519 MakeLibCallOptions CallOptions;
2520 return makeLibCall(DAG, LC, Op.getValueType(), Ops, CallOptions, SDLoc(Op)).first;
2521}
2522
2523static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2524 // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2525 // Any additional optimization in this function should be recorded
2526 // in the cost tables.
2527 EVT VT = Op.getValueType();
2528 SDLoc dl(Op);
2529 SDValue In = Op.getOperand(0);
2530 EVT InVT = In.getValueType();
2531
2532 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2533 MVT CastVT =
2534 MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
2535 InVT.getVectorNumElements());
2536 In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
2537 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
2538 }
2539
2540 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2541 unsigned CastOpc =
2542 Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2543 EVT CastVT = VT.changeVectorElementTypeToInteger();
2544 In = DAG.getNode(CastOpc, dl, CastVT, In);
2545 return DAG.getNode(Op.getOpcode(), dl, VT, In);
2546 }
2547
2548 return Op;
2549}
2550
2551SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2552 SelectionDAG &DAG) const {
2553 if (Op.getValueType().isVector())
2554 return LowerVectorINT_TO_FP(Op, DAG);
2555
2556 // f16 conversions are promoted to f32 when full fp16 is not supported.
2557 if (Op.getValueType() == MVT::f16 &&
2558 !Subtarget->hasFullFP16()) {
2559 SDLoc dl(Op);
2560 return DAG.getNode(
2561 ISD::FP_ROUND, dl, MVT::f16,
2562 DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)),
2563 DAG.getIntPtrConstant(0, dl));
2564 }
2565
2566 // i128 conversions are libcalls.
2567 if (Op.getOperand(0).getValueType() == MVT::i128)
2568 return SDValue();
2569
2570 // Other conversions are legal, unless it's to the completely software-based
2571 // fp128.
2572 if (Op.getValueType() != MVT::f128)
2573 return Op;
2574
2575 RTLIB::Libcall LC;
2576 if (Op.getOpcode() == ISD::SINT_TO_FP)
2577 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2578 else
2579 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2580
2581 return LowerF128Call(Op, DAG, LC);
2582}
2583
2584SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2585 SelectionDAG &DAG) const {
2586 // For iOS, we want to call an alternative entry point: __sincos_stret,
2587 // which returns the values in two S / D registers.
2588 SDLoc dl(Op);
2589 SDValue Arg = Op.getOperand(0);
2590 EVT ArgVT = Arg.getValueType();
2591 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2592
2593 ArgListTy Args;
2594 ArgListEntry Entry;
2595
2596 Entry.Node = Arg;
2597 Entry.Ty = ArgTy;
2598 Entry.IsSExt = false;
2599 Entry.IsZExt = false;
2600 Args.push_back(Entry);
2601
2602 RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
2603 : RTLIB::SINCOS_STRET_F32;
2604 const char *LibcallName = getLibcallName(LC);
2605 SDValue Callee =
2606 DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
2607
2608 StructType *RetTy = StructType::get(ArgTy, ArgTy);
2609 TargetLowering::CallLoweringInfo CLI(DAG);
2610 CLI.setDebugLoc(dl)
2611 .setChain(DAG.getEntryNode())
2612 .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
2613
2614 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2615 return CallResult.first;
2616}
2617
2618static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2619 if (Op.getValueType() != MVT::f16)
2620 return SDValue();
2621
2622 assert(Op.getOperand(0).getValueType() == MVT::i16)((Op.getOperand(0).getValueType() == MVT::i16) ? static_cast<
void> (0) : __assert_fail ("Op.getOperand(0).getValueType() == MVT::i16"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2622, __PRETTY_FUNCTION__))
;
2623 SDLoc DL(Op);
2624
2625 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
2626 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
2627 return SDValue(
2628 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op,
2629 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
2630 0);
2631}
2632
2633static EVT getExtensionTo64Bits(const EVT &OrigVT) {
2634 if (OrigVT.getSizeInBits() >= 64)
2635 return OrigVT;
2636
2637 assert(OrigVT.isSimple() && "Expecting a simple value type")((OrigVT.isSimple() && "Expecting a simple value type"
) ? static_cast<void> (0) : __assert_fail ("OrigVT.isSimple() && \"Expecting a simple value type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2637, __PRETTY_FUNCTION__))
;
2638
2639 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
2640 switch (OrigSimpleTy) {
2641 default: llvm_unreachable("Unexpected Vector Type")::llvm::llvm_unreachable_internal("Unexpected Vector Type", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2641)
;
2642 case MVT::v2i8:
2643 case MVT::v2i16:
2644 return MVT::v2i32;
2645 case MVT::v4i8:
2646 return MVT::v4i16;
2647 }
2648}
2649
2650static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2651 const EVT &OrigTy,
2652 const EVT &ExtTy,
2653 unsigned ExtOpcode) {
2654 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
2655 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
2656 // 64-bits we need to insert a new extension so that it will be 64-bits.
2657 assert(ExtTy.is128BitVector() && "Unexpected extension size")((ExtTy.is128BitVector() && "Unexpected extension size"
) ? static_cast<void> (0) : __assert_fail ("ExtTy.is128BitVector() && \"Unexpected extension size\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2657, __PRETTY_FUNCTION__))
;
2658 if (OrigTy.getSizeInBits() >= 64)
2659 return N;
2660
2661 // Must extend size to at least 64 bits to be used as an operand for VMULL.
2662 EVT NewVT = getExtensionTo64Bits(OrigTy);
2663
2664 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
2665}
2666
2667static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
2668 bool isSigned) {
2669 EVT VT = N->getValueType(0);
2670
2671 if (N->getOpcode() != ISD::BUILD_VECTOR)
2672 return false;
2673
2674 for (const SDValue &Elt : N->op_values()) {
2675 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
2676 unsigned EltSize = VT.getScalarSizeInBits();
2677 unsigned HalfSize = EltSize / 2;
2678 if (isSigned) {
2679 if (!isIntN(HalfSize, C->getSExtValue()))
2680 return false;
2681 } else {
2682 if (!isUIntN(HalfSize, C->getZExtValue()))
2683 return false;
2684 }
2685 continue;
2686 }
2687 return false;
2688 }
2689
2690 return true;
2691}
2692
2693static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
2694 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
2695 return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG,
2696 N->getOperand(0)->getValueType(0),
2697 N->getValueType(0),
2698 N->getOpcode());
2699
2700 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR")((N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"expected BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2700, __PRETTY_FUNCTION__))
;
2701 EVT VT = N->getValueType(0);
2702 SDLoc dl(N);
2703 unsigned EltSize = VT.getScalarSizeInBits() / 2;
2704 unsigned NumElts = VT.getVectorNumElements();
2705 MVT TruncVT = MVT::getIntegerVT(EltSize);
2706 SmallVector<SDValue, 8> Ops;
2707 for (unsigned i = 0; i != NumElts; ++i) {
2708 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
2709 const APInt &CInt = C->getAPIntValue();
2710 // Element types smaller than 32 bits are not legal, so use i32 elements.
2711 // The values are implicitly truncated so sext vs. zext doesn't matter.
2712 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
2713 }
2714 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
2715}
2716
2717static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
2718 return N->getOpcode() == ISD::SIGN_EXTEND ||
2719 isExtendedBUILD_VECTOR(N, DAG, true);
2720}
2721
2722static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
2723 return N->getOpcode() == ISD::ZERO_EXTEND ||
2724 isExtendedBUILD_VECTOR(N, DAG, false);
2725}
2726
2727static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
2728 unsigned Opcode = N->getOpcode();
2729 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2730 SDNode *N0 = N->getOperand(0).getNode();
2731 SDNode *N1 = N->getOperand(1).getNode();
2732 return N0->hasOneUse() && N1->hasOneUse() &&
2733 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
2734 }
2735 return false;
2736}
2737
2738static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
2739 unsigned Opcode = N->getOpcode();
2740 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2741 SDNode *N0 = N->getOperand(0).getNode();
2742 SDNode *N1 = N->getOperand(1).getNode();
2743 return N0->hasOneUse() && N1->hasOneUse() &&
2744 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
2745 }
2746 return false;
2747}
2748
2749SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
2750 SelectionDAG &DAG) const {
2751 // The rounding mode is in bits 23:22 of the FPSCR.
2752 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
2753 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
2754 // so that the shift + and get folded into a bitfield extract.
2755 SDLoc dl(Op);
2756
2757 SDValue FPCR_64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i64,
2758 DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl,
2759 MVT::i64));
2760 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
2761 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
2762 DAG.getConstant(1U << 22, dl, MVT::i32));
2763 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
2764 DAG.getConstant(22, dl, MVT::i32));
2765 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
2766 DAG.getConstant(3, dl, MVT::i32));
2767}
2768
2769static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2770 // Multiplications are only custom-lowered for 128-bit vectors so that
2771 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
2772 EVT VT = Op.getValueType();
2773 assert(VT.is128BitVector() && VT.isInteger() &&((VT.is128BitVector() && VT.isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2774, __PRETTY_FUNCTION__))
2774 "unexpected type for custom-lowering ISD::MUL")((VT.is128BitVector() && VT.isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2774, __PRETTY_FUNCTION__))
;
2775 SDNode *N0 = Op.getOperand(0).getNode();
2776 SDNode *N1 = Op.getOperand(1).getNode();
2777 unsigned NewOpc = 0;
2778 bool isMLA = false;
2779 bool isN0SExt = isSignExtended(N0, DAG);
2780 bool isN1SExt = isSignExtended(N1, DAG);
2781 if (isN0SExt && isN1SExt)
2782 NewOpc = AArch64ISD::SMULL;
2783 else {
2784 bool isN0ZExt = isZeroExtended(N0, DAG);
2785 bool isN1ZExt = isZeroExtended(N1, DAG);
2786 if (isN0ZExt && isN1ZExt)
2787 NewOpc = AArch64ISD::UMULL;
2788 else if (isN1SExt || isN1ZExt) {
2789 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
2790 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
2791 if (isN1SExt && isAddSubSExt(N0, DAG)) {
2792 NewOpc = AArch64ISD::SMULL;
2793 isMLA = true;
2794 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
2795 NewOpc = AArch64ISD::UMULL;
2796 isMLA = true;
2797 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
2798 std::swap(N0, N1);
2799 NewOpc = AArch64ISD::UMULL;
2800 isMLA = true;
2801 }
2802 }
2803
2804 if (!NewOpc) {
2805 if (VT == MVT::v2i64)
2806 // Fall through to expand this. It is not legal.
2807 return SDValue();
2808 else
2809 // Other vector multiplications are legal.
2810 return Op;
2811 }
2812 }
2813
2814 // Legalize to a S/UMULL instruction
2815 SDLoc DL(Op);
2816 SDValue Op0;
2817 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
2818 if (!isMLA) {
2819 Op0 = skipExtensionForVectorMULL(N0, DAG);
2820 assert(Op0.getValueType().is64BitVector() &&((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2822, __PRETTY_FUNCTION__))
2821 Op1.getValueType().is64BitVector() &&((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2822, __PRETTY_FUNCTION__))
2822 "unexpected types for extended operands to VMULL")((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2822, __PRETTY_FUNCTION__))
;
2823 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
2824 }
2825 // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
2826 // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
2827 // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
2828 SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
2829 SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
2830 EVT Op1VT = Op1.getValueType();
2831 return DAG.getNode(N0->getOpcode(), DL, VT,
2832 DAG.getNode(NewOpc, DL, VT,
2833 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
2834 DAG.getNode(NewOpc, DL, VT,
2835 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
2836}
2837
2838SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2839 SelectionDAG &DAG) const {
2840 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2841 SDLoc dl(Op);
2842 switch (IntNo) {
2843 default: return SDValue(); // Don't custom lower most intrinsics.
2844 case Intrinsic::thread_pointer: {
2845 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2846 return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
2847 }
2848 case Intrinsic::aarch64_neon_abs: {
2849 EVT Ty = Op.getValueType();
2850 if (Ty == MVT::i64) {
2851 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
2852 Op.getOperand(1));
2853 Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
2854 return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
2855 } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
2856 return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
2857 } else {
2858 report_fatal_error("Unexpected type for AArch64 NEON intrinic");
2859 }
2860 }
2861 case Intrinsic::aarch64_neon_smax:
2862 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
2863 Op.getOperand(1), Op.getOperand(2));
2864 case Intrinsic::aarch64_neon_umax:
2865 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
2866 Op.getOperand(1), Op.getOperand(2));
2867 case Intrinsic::aarch64_neon_smin:
2868 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
2869 Op.getOperand(1), Op.getOperand(2));
2870 case Intrinsic::aarch64_neon_umin:
2871 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
2872 Op.getOperand(1), Op.getOperand(2));
2873
2874 case Intrinsic::aarch64_sve_sunpkhi:
2875 return DAG.getNode(AArch64ISD::SUNPKHI, dl, Op.getValueType(),
2876 Op.getOperand(1));
2877 case Intrinsic::aarch64_sve_sunpklo:
2878 return DAG.getNode(AArch64ISD::SUNPKLO, dl, Op.getValueType(),
2879 Op.getOperand(1));
2880 case Intrinsic::aarch64_sve_uunpkhi:
2881 return DAG.getNode(AArch64ISD::UUNPKHI, dl, Op.getValueType(),
2882 Op.getOperand(1));
2883 case Intrinsic::aarch64_sve_uunpklo:
2884 return DAG.getNode(AArch64ISD::UUNPKLO, dl, Op.getValueType(),
2885 Op.getOperand(1));
2886
2887 case Intrinsic::localaddress: {
2888 const auto &MF = DAG.getMachineFunction();
2889 const auto *RegInfo = Subtarget->getRegisterInfo();
2890 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
2891 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
2892 Op.getSimpleValueType());
2893 }
2894
2895 case Intrinsic::eh_recoverfp: {
2896 // FIXME: This needs to be implemented to correctly handle highly aligned
2897 // stack objects. For now we simply return the incoming FP. Refer D53541
2898 // for more details.
2899 SDValue FnOp = Op.getOperand(1);
2900 SDValue IncomingFPOp = Op.getOperand(2);
2901 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
2902 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
2903 if (!Fn)
2904 report_fatal_error(
2905 "llvm.eh.recoverfp must take a function as the first argument");
2906 return IncomingFPOp;
2907 }
2908 }
2909}
2910
2911bool AArch64TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
2912 return ExtVal.getValueType().isScalableVector();
2913}
2914
2915// Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
2916static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
2917 EVT VT, EVT MemVT,
2918 SelectionDAG &DAG) {
2919 assert(VT.isVector() && "VT should be a vector type")((VT.isVector() && "VT should be a vector type") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"VT should be a vector type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2919, __PRETTY_FUNCTION__))
;
2920 assert(MemVT == MVT::v4i8 && VT == MVT::v4i16)((MemVT == MVT::v4i8 && VT == MVT::v4i16) ? static_cast
<void> (0) : __assert_fail ("MemVT == MVT::v4i8 && VT == MVT::v4i16"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2920, __PRETTY_FUNCTION__))
;
2921
2922 SDValue Value = ST->getValue();
2923
2924 // It first extend the promoted v4i16 to v8i16, truncate to v8i8, and extract
2925 // the word lane which represent the v4i8 subvector. It optimizes the store
2926 // to:
2927 //
2928 // xtn v0.8b, v0.8h
2929 // str s0, [x0]
2930
2931 SDValue Undef = DAG.getUNDEF(MVT::i16);
2932 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
2933 {Undef, Undef, Undef, Undef});
2934
2935 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
2936 Value, UndefVec);
2937 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
2938
2939 Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
2940 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
2941 Trunc, DAG.getConstant(0, DL, MVT::i64));
2942
2943 return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
2944 ST->getBasePtr(), ST->getMemOperand());
2945}
2946
2947// Custom lowering for any store, vector or scalar and/or default or with
2948// a truncate operations. Currently only custom lower truncate operation
2949// from vector v4i16 to v4i8.
2950SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
2951 SelectionDAG &DAG) const {
2952 SDLoc Dl(Op);
2953 StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
2954 assert (StoreNode && "Can only custom lower store nodes")((StoreNode && "Can only custom lower store nodes") ?
static_cast<void> (0) : __assert_fail ("StoreNode && \"Can only custom lower store nodes\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2954, __PRETTY_FUNCTION__))
;
2955
2956 SDValue Value = StoreNode->getValue();
2957
2958 EVT VT = Value.getValueType();
2959 EVT MemVT = StoreNode->getMemoryVT();
2960
2961 assert (VT.isVector() && "Can only custom lower vector store types")((VT.isVector() && "Can only custom lower vector store types"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can only custom lower vector store types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2961, __PRETTY_FUNCTION__))
;
2962
2963 unsigned AS = StoreNode->getAddressSpace();
2964 unsigned Align = StoreNode->getAlignment();
2965 if (Align < MemVT.getStoreSize() &&
2966 !allowsMisalignedMemoryAccesses(
2967 MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) {
2968 return scalarizeVectorStore(StoreNode, DAG);
2969 }
2970
2971 if (StoreNode->isTruncatingStore()) {
2972 return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
2973 }
2974
2975 return SDValue();
2976}
2977
2978SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
2979 SelectionDAG &DAG) const {
2980 LLVM_DEBUG(dbgs() << "Custom lowering: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Custom lowering: "; } }
while (false)
;
2981 LLVM_DEBUG(Op.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { Op.dump(); } } while (false)
;
2982
2983 switch (Op.getOpcode()) {
2984 default:
2985 llvm_unreachable("unimplemented operand")::llvm::llvm_unreachable_internal("unimplemented operand", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2985)
;
2986 return SDValue();
2987 case ISD::BITCAST:
2988 return LowerBITCAST(Op, DAG);
2989 case ISD::GlobalAddress:
2990 return LowerGlobalAddress(Op, DAG);
2991 case ISD::GlobalTLSAddress:
2992 return LowerGlobalTLSAddress(Op, DAG);
2993 case ISD::SETCC:
2994 return LowerSETCC(Op, DAG);
2995 case ISD::BR_CC:
2996 return LowerBR_CC(Op, DAG);
2997 case ISD::SELECT:
2998 return LowerSELECT(Op, DAG);
2999 case ISD::SELECT_CC:
3000 return LowerSELECT_CC(Op, DAG);
3001 case ISD::JumpTable:
3002 return LowerJumpTable(Op, DAG);
3003 case ISD::BR_JT:
3004 return LowerBR_JT(Op, DAG);
3005 case ISD::ConstantPool:
3006 return LowerConstantPool(Op, DAG);
3007 case ISD::BlockAddress:
3008 return LowerBlockAddress(Op, DAG);
3009 case ISD::VASTART:
3010 return LowerVASTART(Op, DAG);
3011 case ISD::VACOPY:
3012 return LowerVACOPY(Op, DAG);
3013 case ISD::VAARG:
3014 return LowerVAARG(Op, DAG);
3015 case ISD::ADDC:
3016 case ISD::ADDE:
3017 case ISD::SUBC:
3018 case ISD::SUBE:
3019 return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
3020 case ISD::SADDO:
3021 case ISD::UADDO:
3022 case ISD::SSUBO:
3023 case ISD::USUBO:
3024 case ISD::SMULO:
3025 case ISD::UMULO:
3026 return LowerXALUO(Op, DAG);
3027 case ISD::FADD:
3028 return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
3029 case ISD::FSUB:
3030 return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
3031 case ISD::FMUL:
3032 return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
3033 case ISD::FDIV:
3034 return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
3035 case ISD::FP_ROUND:
3036 return LowerFP_ROUND(Op, DAG);
3037 case ISD::FP_EXTEND:
3038 return LowerFP_EXTEND(Op, DAG);
3039 case ISD::FRAMEADDR:
3040 return LowerFRAMEADDR(Op, DAG);
3041 case ISD::SPONENTRY:
3042 return LowerSPONENTRY(Op, DAG);
3043 case ISD::RETURNADDR:
3044 return LowerRETURNADDR(Op, DAG);
3045 case ISD::ADDROFRETURNADDR:
3046 return LowerADDROFRETURNADDR(Op, DAG);
3047 case ISD::INSERT_VECTOR_ELT:
3048 return LowerINSERT_VECTOR_ELT(Op, DAG);
3049 case ISD::EXTRACT_VECTOR_ELT:
3050 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3051 case ISD::BUILD_VECTOR:
3052 return LowerBUILD_VECTOR(Op, DAG);
3053 case ISD::VECTOR_SHUFFLE:
3054 return LowerVECTOR_SHUFFLE(Op, DAG);
3055 case ISD::SPLAT_VECTOR:
3056 return LowerSPLAT_VECTOR(Op, DAG);
3057 case ISD::EXTRACT_SUBVECTOR:
3058 return LowerEXTRACT_SUBVECTOR(Op, DAG);
3059 case ISD::SRA:
3060 case ISD::SRL:
3061 case ISD::SHL:
3062 return LowerVectorSRA_SRL_SHL(Op, DAG);
3063 case ISD::SHL_PARTS:
3064 return LowerShiftLeftParts(Op, DAG);
3065 case ISD::SRL_PARTS:
3066 case ISD::SRA_PARTS:
3067 return LowerShiftRightParts(Op, DAG);
3068 case ISD::CTPOP:
3069 return LowerCTPOP(Op, DAG);
3070 case ISD::FCOPYSIGN:
3071 return LowerFCOPYSIGN(Op, DAG);
3072 case ISD::OR:
3073 return LowerVectorOR(Op, DAG);
3074 case ISD::XOR:
3075 return LowerXOR(Op, DAG);
3076 case ISD::PREFETCH:
3077 return LowerPREFETCH(Op, DAG);
3078 case ISD::SINT_TO_FP:
3079 case ISD::UINT_TO_FP:
3080 return LowerINT_TO_FP(Op, DAG);
3081 case ISD::FP_TO_SINT:
3082 case ISD::FP_TO_UINT:
3083 return LowerFP_TO_INT(Op, DAG);
3084 case ISD::FSINCOS:
3085 return LowerFSINCOS(Op, DAG);
3086 case ISD::FLT_ROUNDS_:
3087 return LowerFLT_ROUNDS_(Op, DAG);
3088 case ISD::MUL:
3089 return LowerMUL(Op, DAG);
3090 case ISD::INTRINSIC_WO_CHAIN:
3091 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3092 case ISD::STORE:
3093 return LowerSTORE(Op, DAG);
3094 case ISD::VECREDUCE_ADD:
3095 case ISD::VECREDUCE_SMAX:
3096 case ISD::VECREDUCE_SMIN:
3097 case ISD::VECREDUCE_UMAX:
3098 case ISD::VECREDUCE_UMIN:
3099 case ISD::VECREDUCE_FMAX:
3100 case ISD::VECREDUCE_FMIN:
3101 return LowerVECREDUCE(Op, DAG);
3102 case ISD::ATOMIC_LOAD_SUB:
3103 return LowerATOMIC_LOAD_SUB(Op, DAG);
3104 case ISD::ATOMIC_LOAD_AND:
3105 return LowerATOMIC_LOAD_AND(Op, DAG);
3106 case ISD::DYNAMIC_STACKALLOC:
3107 return LowerDYNAMIC_STACKALLOC(Op, DAG);
3108 }
3109}
3110
3111//===----------------------------------------------------------------------===//
3112// Calling Convention Implementation
3113//===----------------------------------------------------------------------===//
3114
3115/// Selects the correct CCAssignFn for a given CallingConvention value.
3116CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
3117 bool IsVarArg) const {
3118 switch (CC) {
3119 default:
3120 report_fatal_error("Unsupported calling convention.");
3121 case CallingConv::AArch64_SVE_VectorCall:
3122 // Calling SVE functions is currently not yet supported.
3123 report_fatal_error("Unsupported calling convention.");
3124 case CallingConv::WebKit_JS:
3125 return CC_AArch64_WebKit_JS;
3126 case CallingConv::GHC:
3127 return CC_AArch64_GHC;
3128 case CallingConv::C:
3129 case CallingConv::Fast:
3130 case CallingConv::PreserveMost:
3131 case CallingConv::CXX_FAST_TLS:
3132 case CallingConv::Swift:
3133 if (Subtarget->isTargetWindows() && IsVarArg)
3134 return CC_AArch64_Win64_VarArg;
3135 if (!Subtarget->isTargetDarwin())
3136 return CC_AArch64_AAPCS;
3137 if (!IsVarArg)
3138 return CC_AArch64_DarwinPCS;
3139 return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg
3140 : CC_AArch64_DarwinPCS_VarArg;
3141 case CallingConv::Win64:
3142 return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
3143 case CallingConv::CFGuard_Check:
3144 return CC_AArch64_Win64_CFGuard_Check;
3145 case CallingConv::AArch64_VectorCall:
3146 return CC_AArch64_AAPCS;
3147 }
3148}
3149
3150CCAssignFn *
3151AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
3152 return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
3153 : RetCC_AArch64_AAPCS;
3154}
3155
3156SDValue AArch64TargetLowering::LowerFormalArguments(
3157 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3158 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3159 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3160 MachineFunction &MF = DAG.getMachineFunction();
3161 MachineFrameInfo &MFI = MF.getFrameInfo();
3162 bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
3163
3164 // Assign locations to all of the incoming arguments.
3165 SmallVector<CCValAssign, 16> ArgLocs;
3166 DenseMap<unsigned, SDValue> CopiedRegs;
3167 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3168 *DAG.getContext());
3169
3170 // At this point, Ins[].VT may already be promoted to i32. To correctly
3171 // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
3172 // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
3173 // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
3174 // we use a special version of AnalyzeFormalArguments to pass in ValVT and
3175 // LocVT.
3176 unsigned NumArgs = Ins.size();
3177 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
3178 unsigned CurArgIdx = 0;
3179 for (unsigned i = 0; i != NumArgs; ++i) {
3180 MVT ValVT = Ins[i].VT;
3181 if (Ins[i].isOrigArg()) {
3182 std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
3183 CurArgIdx = Ins[i].getOrigArgIndex();
3184
3185 // Get type of the original argument.
3186 EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
3187 /*AllowUnknown*/ true);
3188 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
3189 // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3190 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3191 ValVT = MVT::i8;
3192 else if (ActualMVT == MVT::i16)
3193 ValVT = MVT::i16;
3194 }
3195 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3196 bool Res =
3197 AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
3198 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3198, __PRETTY_FUNCTION__))
;
3199 (void)Res;
3200 }
3201 assert(ArgLocs.size() == Ins.size())((ArgLocs.size() == Ins.size()) ? static_cast<void> (0)
: __assert_fail ("ArgLocs.size() == Ins.size()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3201, __PRETTY_FUNCTION__))
;
3202 SmallVector<SDValue, 16> ArgValues;
3203 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3204 CCValAssign &VA = ArgLocs[i];
3205
3206 if (Ins[i].Flags.isByVal()) {
3207 // Byval is used for HFAs in the PCS, but the system should work in a
3208 // non-compliant manner for larger structs.
3209 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3210 int Size = Ins[i].Flags.getByValSize();
3211 unsigned NumRegs = (Size + 7) / 8;
3212
3213 // FIXME: This works on big-endian for composite byvals, which are the common
3214 // case. It should also work for fundamental types too.
3215 unsigned FrameIdx =
3216 MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
3217 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
3218 InVals.push_back(FrameIdxN);
3219
3220 continue;
3221 }
3222
3223 SDValue ArgValue;
3224 if (VA.isRegLoc()) {
3225 // Arguments stored in registers.
3226 EVT RegVT = VA.getLocVT();
3227 const TargetRegisterClass *RC;
3228
3229 if (RegVT == MVT::i32)
3230 RC = &AArch64::GPR32RegClass;
3231 else if (RegVT == MVT::i64)
3232 RC = &AArch64::GPR64RegClass;
3233 else if (RegVT == MVT::f16)
3234 RC = &AArch64::FPR16RegClass;
3235 else if (RegVT == MVT::f32)
3236 RC = &AArch64::FPR32RegClass;
3237 else if (RegVT == MVT::f64 || RegVT.is64BitVector())
3238 RC = &AArch64::FPR64RegClass;
3239 else if (RegVT == MVT::f128 || RegVT.is128BitVector())
3240 RC = &AArch64::FPR128RegClass;
3241 else if (RegVT.isScalableVector() &&
3242 RegVT.getVectorElementType() == MVT::i1)
3243 RC = &AArch64::PPRRegClass;
3244 else if (RegVT.isScalableVector())
3245 RC = &AArch64::ZPRRegClass;
3246 else
3247 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering")::llvm::llvm_unreachable_internal("RegVT not supported by FORMAL_ARGUMENTS Lowering"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3247)
;
3248
3249 // Transform the arguments in physical registers into virtual ones.
3250 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3251 ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3252
3253 // If this is an 8, 16 or 32-bit value, it is really passed promoted
3254 // to 64 bits. Insert an assert[sz]ext to capture this, then
3255 // truncate to the right size.
3256 switch (VA.getLocInfo()) {
3257 default:
3258 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3258)
;
3259 case CCValAssign::Full:
3260 break;
3261 case CCValAssign::Indirect:
3262 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3263, __PRETTY_FUNCTION__))
3263 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3263, __PRETTY_FUNCTION__))
;
3264 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3264)
;
3265 case CCValAssign::BCvt:
3266 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
3267 break;
3268 case CCValAssign::AExt:
3269 case CCValAssign::SExt:
3270 case CCValAssign::ZExt:
3271 break;
3272 case CCValAssign::AExtUpper:
3273 ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
3274 DAG.getConstant(32, DL, RegVT));
3275 ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
3276 break;
3277 }
3278 } else { // VA.isRegLoc()
3279 assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem")((VA.isMemLoc() && "CCValAssign is neither reg nor mem"
) ? static_cast<void> (0) : __assert_fail ("VA.isMemLoc() && \"CCValAssign is neither reg nor mem\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3279, __PRETTY_FUNCTION__))
;
3280 unsigned ArgOffset = VA.getLocMemOffset();
3281 unsigned ArgSize = VA.getValVT().getSizeInBits() / 8;
3282
3283 uint32_t BEAlign = 0;
3284 if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
3285 !Ins[i].Flags.isInConsecutiveRegs())
3286 BEAlign = 8 - ArgSize;
3287
3288 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
3289
3290 // Create load nodes to retrieve arguments from the stack.
3291 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3292
3293 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
3294 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
3295 MVT MemVT = VA.getValVT();
3296
3297 switch (VA.getLocInfo()) {
3298 default:
3299 break;
3300 case CCValAssign::Trunc:
3301 case CCValAssign::BCvt:
3302 MemVT = VA.getLocVT();
3303 break;
3304 case CCValAssign::Indirect:
3305 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3306, __PRETTY_FUNCTION__))
3306 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3306, __PRETTY_FUNCTION__))
;
3307 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3307)
;
3308 case CCValAssign::SExt:
3309 ExtType = ISD::SEXTLOAD;
3310 break;
3311 case CCValAssign::ZExt:
3312 ExtType = ISD::ZEXTLOAD;
3313 break;
3314 case CCValAssign::AExt:
3315 ExtType = ISD::EXTLOAD;
3316 break;
3317 }
3318
3319 ArgValue = DAG.getExtLoad(
3320 ExtType, DL, VA.getLocVT(), Chain, FIN,
3321 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
3322 MemVT);
3323
3324 }
3325 if (Subtarget->isTargetILP32() && Ins[i].Flags.isPointer())
3326 ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
3327 ArgValue, DAG.getValueType(MVT::i32));
3328 InVals.push_back(ArgValue);
3329 }
3330
3331 // varargs
3332 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3333 if (isVarArg) {
3334 if (!Subtarget->isTargetDarwin() || IsWin64) {
3335 // The AAPCS variadic function ABI is identical to the non-variadic
3336 // one. As a result there may be more arguments in registers and we should
3337 // save them for future reference.
3338 // Win64 variadic functions also pass arguments in registers, but all float
3339 // arguments are passed in integer registers.
3340 saveVarArgRegisters(CCInfo, DAG, DL, Chain);
3341 }
3342
3343 // This will point to the next argument passed via stack.
3344 unsigned StackOffset = CCInfo.getNextStackOffset();
3345 // We currently pass all varargs at 8-byte alignment, or 4 for ILP32
3346 StackOffset = alignTo(StackOffset, Subtarget->isTargetILP32() ? 4 : 8);
3347 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
3348
3349 if (MFI.hasMustTailInVarArgFunc()) {
3350 SmallVector<MVT, 2> RegParmTypes;
3351 RegParmTypes.push_back(MVT::i64);
3352 RegParmTypes.push_back(MVT::f128);
3353 // Compute the set of forwarded registers. The rest are scratch.
3354 SmallVectorImpl<ForwardedRegister> &Forwards =
3355 FuncInfo->getForwardedMustTailRegParms();
3356 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes,
3357 CC_AArch64_AAPCS);
3358
3359 // Conservatively forward X8, since it might be used for aggregate return.
3360 if (!CCInfo.isAllocated(AArch64::X8)) {
3361 unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
3362 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
3363 }
3364 }
3365 }
3366
3367 // On Windows, InReg pointers must be returned, so record the pointer in a
3368 // virtual register at the start of the function so it can be returned in the
3369 // epilogue.
3370 if (IsWin64) {
3371 for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
3372 if (Ins[I].Flags.isInReg()) {
3373 assert(!FuncInfo->getSRetReturnReg())((!FuncInfo->getSRetReturnReg()) ? static_cast<void>
(0) : __assert_fail ("!FuncInfo->getSRetReturnReg()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3373, __PRETTY_FUNCTION__))
;
3374
3375 MVT PtrTy = getPointerTy(DAG.getDataLayout());
3376 Register Reg =
3377 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
3378 FuncInfo->setSRetReturnReg(Reg);
3379
3380 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
3381 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
3382 break;
3383 }
3384 }
3385 }
3386
3387 unsigned StackArgSize = CCInfo.getNextStackOffset();
3388 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3389 if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
3390 // This is a non-standard ABI so by fiat I say we're allowed to make full
3391 // use of the stack area to be popped, which must be aligned to 16 bytes in
3392 // any case:
3393 StackArgSize = alignTo(StackArgSize, 16);
3394
3395 // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
3396 // a multiple of 16.
3397 FuncInfo->setArgumentStackToRestore(StackArgSize);
3398
3399 // This realignment carries over to the available bytes below. Our own
3400 // callers will guarantee the space is free by giving an aligned value to
3401 // CALLSEQ_START.
3402 }
3403 // Even if we're not expected to free up the space, it's useful to know how
3404 // much is there while considering tail calls (because we can reuse it).
3405 FuncInfo->setBytesInStackArgArea(StackArgSize);
3406
3407 if (Subtarget->hasCustomCallingConv())
3408 Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
3409
3410 return Chain;
3411}
3412
3413void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
3414 SelectionDAG &DAG,
3415 const SDLoc &DL,
3416 SDValue &Chain) const {
3417 MachineFunction &MF = DAG.getMachineFunction();
3418 MachineFrameInfo &MFI = MF.getFrameInfo();
3419 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3420 auto PtrVT = getPointerTy(DAG.getDataLayout());
3421 bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
3422
3423 SmallVector<SDValue, 8> MemOps;
3424
3425 static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
3426 AArch64::X3, AArch64::X4, AArch64::X5,
3427 AArch64::X6, AArch64::X7 };
3428 static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
3429 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
3430
3431 unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
3432 int GPRIdx = 0;
3433 if (GPRSaveSize != 0) {
3434 if (IsWin64) {
3435 GPRIdx = MFI.CreateFixedObject(GPRSaveSize, -(int)GPRSaveSize, false);
3436 if (GPRSaveSize & 15)
3437 // The extra size here, if triggered, will always be 8.
3438 MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
3439 } else
3440 GPRIdx = MFI.CreateStackObject(GPRSaveSize, 8, false);
3441
3442 SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
3443
3444 for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
3445 unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
3446 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3447 SDValue Store = DAG.getStore(
3448 Val.getValue(1), DL, Val, FIN,
3449 IsWin64
3450 ? MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
3451 GPRIdx,
3452 (i - FirstVariadicGPR) * 8)
3453 : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8));
3454 MemOps.push_back(Store);
3455 FIN =
3456 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
3457 }
3458 }
3459 FuncInfo->setVarArgsGPRIndex(GPRIdx);
3460 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
3461
3462 if (Subtarget->hasFPARMv8() && !IsWin64) {
3463 static const MCPhysReg FPRArgRegs[] = {
3464 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
3465 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
3466 static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
3467 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
3468
3469 unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
3470 int FPRIdx = 0;
3471 if (FPRSaveSize != 0) {
3472 FPRIdx = MFI.CreateStackObject(FPRSaveSize, 16, false);
3473
3474 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
3475
3476 for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
3477 unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
3478 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
3479
3480 SDValue Store = DAG.getStore(
3481 Val.getValue(1), DL, Val, FIN,
3482 MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16));
3483 MemOps.push_back(Store);
3484 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
3485 DAG.getConstant(16, DL, PtrVT));
3486 }
3487 }
3488 FuncInfo->setVarArgsFPRIndex(FPRIdx);
3489 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
3490 }
3491
3492 if (!MemOps.empty()) {
3493 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3494 }
3495}
3496
3497/// LowerCallResult - Lower the result values of a call into the
3498/// appropriate copies out of appropriate physical registers.
3499SDValue AArch64TargetLowering::LowerCallResult(
3500 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
3501 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3502 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
3503 SDValue ThisVal) const {
3504 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
3505 ? RetCC_AArch64_WebKit_JS
3506 : RetCC_AArch64_AAPCS;
3507 // Assign locations to each value returned by this call.
3508 SmallVector<CCValAssign, 16> RVLocs;
3509 DenseMap<unsigned, SDValue> CopiedRegs;
3510 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3511 *DAG.getContext());
3512 CCInfo.AnalyzeCallResult(Ins, RetCC);
3513
3514 // Copy all of the result registers out of their specified physreg.
3515 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3516 CCValAssign VA = RVLocs[i];
3517
3518 // Pass 'this' value directly from the argument to return value, to avoid
3519 // reg unit interference
3520 if (i == 0 && isThisReturn) {
3521 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&((!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
"unexpected return calling convention register assignment") ?
static_cast<void> (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i64 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3522, __PRETTY_FUNCTION__))
3522 "unexpected return calling convention register assignment")((!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
"unexpected return calling convention register assignment") ?
static_cast<void> (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i64 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3522, __PRETTY_FUNCTION__))
;
3523 InVals.push_back(ThisVal);
3524 continue;
3525 }
3526
3527 // Avoid copying a physreg twice since RegAllocFast is incompetent and only
3528 // allows one use of a physreg per block.
3529 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
3530 if (!Val) {
3531 Val =
3532 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
3533 Chain = Val.getValue(1);
3534 InFlag = Val.getValue(2);
3535 CopiedRegs[VA.getLocReg()] = Val;
3536 }
3537
3538 switch (VA.getLocInfo()) {
3539 default:
3540 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3540)
;
3541 case CCValAssign::Full:
3542 break;
3543 case CCValAssign::BCvt:
3544 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
3545 break;
3546 case CCValAssign::AExtUpper:
3547 Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val,
3548 DAG.getConstant(32, DL, VA.getLocVT()));
3549 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3550 case CCValAssign::AExt:
3551 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3552 case CCValAssign::ZExt:
3553 Val = DAG.getZExtOrTrunc(Val, DL, VA.getValVT());
3554 break;
3555 }
3556
3557 InVals.push_back(Val);
3558 }
3559
3560 return Chain;
3561}
3562
3563/// Return true if the calling convention is one that we can guarantee TCO for.
3564static bool canGuaranteeTCO(CallingConv::ID CC) {
3565 return CC == CallingConv::Fast;
3566}
3567
3568/// Return true if we might ever do TCO for calls with this calling convention.
3569static bool mayTailCallThisCC(CallingConv::ID CC) {
3570 switch (CC) {
3571 case CallingConv::C:
3572 case CallingConv::PreserveMost:
3573 case CallingConv::Swift:
3574 return true;
3575 default:
3576 return canGuaranteeTCO(CC);
3577 }
3578}
3579
3580bool AArch64TargetLowering::isEligibleForTailCallOptimization(
3581 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
3582 const SmallVectorImpl<ISD::OutputArg> &Outs,
3583 const SmallVectorImpl<SDValue> &OutVals,
3584 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
3585 if (!mayTailCallThisCC(CalleeCC))
3586 return false;
3587
3588 MachineFunction &MF = DAG.getMachineFunction();
3589 const Function &CallerF = MF.getFunction();
3590 CallingConv::ID CallerCC = CallerF.getCallingConv();
3591 bool CCMatch = CallerCC == CalleeCC;
3592
3593 // Byval parameters hand the function a pointer directly into the stack area
3594 // we want to reuse during a tail call. Working around this *is* possible (see
3595 // X86) but less efficient and uglier in LowerCall.
3596 for (Function::const_arg_iterator i = CallerF.arg_begin(),
3597 e = CallerF.arg_end();
3598 i != e; ++i) {
3599 if (i->hasByValAttr())
3600 return false;
3601
3602 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
3603 // In this case, it is necessary to save/restore X0 in the callee. Tail
3604 // call opt interferes with this. So we disable tail call opt when the
3605 // caller has an argument with "inreg" attribute.
3606
3607 // FIXME: Check whether the callee also has an "inreg" argument.
3608 if (i->hasInRegAttr())
3609 return false;
3610 }
3611
3612 if (getTargetMachine().Options.GuaranteedTailCallOpt)
3613 return canGuaranteeTCO(CalleeCC) && CCMatch;
3614
3615 // Externally-defined functions with weak linkage should not be
3616 // tail-called on AArch64 when the OS does not support dynamic
3617 // pre-emption of symbols, as the AAELF spec requires normal calls
3618 // to undefined weak functions to be replaced with a NOP or jump to the
3619 // next instruction. The behaviour of branch instructions in this
3620 // situation (as used for tail calls) is implementation-defined, so we
3621 // cannot rely on the linker replacing the tail call with a return.
3622 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3623 const GlobalValue *GV = G->getGlobal();
3624 const Triple &TT = getTargetMachine().getTargetTriple();
3625 if (GV->hasExternalWeakLinkage() &&
3626 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
3627 return false;
3628 }
3629
3630 // Now we search for cases where we can use a tail call without changing the
3631 // ABI. Sibcall is used in some places (particularly gcc) to refer to this
3632 // concept.
3633
3634 // I want anyone implementing a new calling convention to think long and hard
3635 // about this assert.
3636 assert((!isVarArg || CalleeCC == CallingConv::C) &&(((!isVarArg || CalleeCC == CallingConv::C) && "Unexpected variadic calling convention"
) ? static_cast<void> (0) : __assert_fail ("(!isVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3637, __PRETTY_FUNCTION__))
3637 "Unexpected variadic calling convention")(((!isVarArg || CalleeCC == CallingConv::C) && "Unexpected variadic calling convention"
) ? static_cast<void> (0) : __assert_fail ("(!isVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3637, __PRETTY_FUNCTION__))
;
3638
3639 LLVMContext &C = *DAG.getContext();
3640 if (isVarArg && !Outs.empty()) {
3641 // At least two cases here: if caller is fastcc then we can't have any
3642 // memory arguments (we'd be expected to clean up the stack afterwards). If
3643 // caller is C then we could potentially use its argument area.
3644
3645 // FIXME: for now we take the most conservative of these in both cases:
3646 // disallow all variadic memory operands.
3647 SmallVector<CCValAssign, 16> ArgLocs;
3648 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3649
3650 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
3651 for (const CCValAssign &ArgLoc : ArgLocs)
3652 if (!ArgLoc.isRegLoc())
3653 return false;
3654 }
3655
3656 // Check that the call results are passed in the same way.
3657 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
3658 CCAssignFnForCall(CalleeCC, isVarArg),
3659 CCAssignFnForCall(CallerCC, isVarArg)))
3660 return false;
3661 // The callee has to preserve all registers the caller needs to preserve.
3662 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
3663 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3664 if (!CCMatch) {
3665 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3666 if (Subtarget->hasCustomCallingConv()) {
3667 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
3668 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
3669 }
3670 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3671 return false;
3672 }
3673
3674 // Nothing more to check if the callee is taking no arguments
3675 if (Outs.empty())
3676 return true;
3677
3678 SmallVector<CCValAssign, 16> ArgLocs;
3679 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3680
3681 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
3682
3683 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3684
3685 // If the stack arguments for this call do not fit into our own save area then
3686 // the call cannot be made tail.
3687 if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
3688 return false;
3689
3690 const MachineRegisterInfo &MRI = MF.getRegInfo();
3691 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
3692 return false;
3693
3694 return true;
3695}
3696
3697SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
3698 SelectionDAG &DAG,
3699 MachineFrameInfo &MFI,
3700 int ClobberedFI) const {
3701 SmallVector<SDValue, 8> ArgChains;
3702 int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
3703 int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
3704
3705 // Include the original chain at the beginning of the list. When this is
3706 // used by target LowerCall hooks, this helps legalize find the
3707 // CALLSEQ_BEGIN node.
3708 ArgChains.push_back(Chain);
3709
3710 // Add a chain value for each stack argument corresponding
3711 for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
3712 UE = DAG.getEntryNode().getNode()->use_end();
3713 U != UE; ++U)
3714 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3715 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3716 if (FI->getIndex() < 0) {
3717 int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
3718 int64_t InLastByte = InFirstByte;
3719 InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
3720
3721 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
3722 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
3723 ArgChains.push_back(SDValue(L, 1));
3724 }
3725
3726 // Build a tokenfactor for all the chains.
3727 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
3728}
3729
3730bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
3731 bool TailCallOpt) const {
3732 return CallCC == CallingConv::Fast && TailCallOpt;
3733}
3734
3735/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
3736/// and add input and output parameter nodes.
3737SDValue
3738AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
3739 SmallVectorImpl<SDValue> &InVals) const {
3740 SelectionDAG &DAG = CLI.DAG;
3741 SDLoc &DL = CLI.DL;
3742 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
3743 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
3744 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
3745 SDValue Chain = CLI.Chain;
3746 SDValue Callee = CLI.Callee;
3747 bool &IsTailCall = CLI.IsTailCall;
3748 CallingConv::ID CallConv = CLI.CallConv;
3749 bool IsVarArg = CLI.IsVarArg;
3750
3751 MachineFunction &MF = DAG.getMachineFunction();
3752 MachineFunction::CallSiteInfo CSInfo;
3753 bool IsThisReturn = false;
3754
3755 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3756 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3757 bool IsSibCall = false;
3758
3759 if (IsTailCall) {
3760 // Check if it's really possible to do a tail call.
3761 IsTailCall = isEligibleForTailCallOptimization(
3762 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
3763 if (!IsTailCall && CLI.CS && CLI.CS.isMustTailCall())
3764 report_fatal_error("failed to perform tail call elimination on a call "
3765 "site marked musttail");
3766
3767 // A sibling call is one where we're under the usual C ABI and not planning
3768 // to change that but can still do a tail call:
3769 if (!TailCallOpt && IsTailCall)
3770 IsSibCall = true;
3771
3772 if (IsTailCall)
3773 ++NumTailCalls;
3774 }
3775
3776 // Analyze operands of the call, assigning locations to each operand.
3777 SmallVector<CCValAssign, 16> ArgLocs;
3778 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
3779 *DAG.getContext());
3780
3781 if (IsVarArg) {
3782 // Handle fixed and variable vector arguments differently.
3783 // Variable vector arguments always go into memory.
3784 unsigned NumArgs = Outs.size();
3785
3786 for (unsigned i = 0; i != NumArgs; ++i) {
3787 MVT ArgVT = Outs[i].VT;
3788 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3789 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv,
3790 /*IsVarArg=*/ !Outs[i].IsFixed);
3791 bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
3792 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3792, __PRETTY_FUNCTION__))
;
3793 (void)Res;
3794 }
3795 } else {
3796 // At this point, Outs[].VT may already be promoted to i32. To correctly
3797 // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
3798 // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
3799 // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
3800 // we use a special version of AnalyzeCallOperands to pass in ValVT and
3801 // LocVT.
3802 unsigned NumArgs = Outs.size();
3803 for (unsigned i = 0; i != NumArgs; ++i) {
3804 MVT ValVT = Outs[i].VT;
3805 // Get type of the original argument.
3806 EVT ActualVT = getValueType(DAG.getDataLayout(),
3807 CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
3808 /*AllowUnknown*/ true);
3809 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
3810 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3811 // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3812 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3813 ValVT = MVT::i8;
3814 else if (ActualMVT == MVT::i16)
3815 ValVT = MVT::i16;
3816
3817 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3818 bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
3819 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3819, __PRETTY_FUNCTION__))
;
3820 (void)Res;
3821 }
3822 }
3823
3824 // Get a count of how many bytes are to be pushed on the stack.
3825 unsigned NumBytes = CCInfo.getNextStackOffset();
3826
3827 if (IsSibCall) {
3828 // Since we're not changing the ABI to make this a tail call, the memory
3829 // operands are already available in the caller's incoming argument space.
3830 NumBytes = 0;
3831 }
3832
3833 // FPDiff is the byte offset of the call's argument area from the callee's.
3834 // Stores to callee stack arguments will be placed in FixedStackSlots offset
3835 // by this amount for a tail call. In a sibling call it must be 0 because the
3836 // caller will deallocate the entire stack and the callee still expects its
3837 // arguments to begin at SP+0. Completely unused for non-tail calls.
3838 int FPDiff = 0;
3839
3840 if (IsTailCall && !IsSibCall) {
3841 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
3842
3843 // Since callee will pop argument stack as a tail call, we must keep the
3844 // popped size 16-byte aligned.
3845 NumBytes = alignTo(NumBytes, 16);
3846
3847 // FPDiff will be negative if this tail call requires more space than we
3848 // would automatically have in our incoming argument space. Positive if we
3849 // can actually shrink the stack.
3850 FPDiff = NumReusableBytes - NumBytes;
3851
3852 // The stack pointer must be 16-byte aligned at all times it's used for a
3853 // memory operation, which in practice means at *all* times and in
3854 // particular across call boundaries. Therefore our own arguments started at
3855 // a 16-byte aligned SP and the delta applied for the tail call should
3856 // satisfy the same constraint.
3857 assert(FPDiff % 16 == 0 && "unaligned stack on tail call")((FPDiff % 16 == 0 && "unaligned stack on tail call")
? static_cast<void> (0) : __assert_fail ("FPDiff % 16 == 0 && \"unaligned stack on tail call\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3857, __PRETTY_FUNCTION__))
;
3858 }
3859
3860 // Adjust the stack pointer for the new arguments...
3861 // These operations are automatically eliminated by the prolog/epilog pass
3862 if (!IsSibCall)
3863 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
3864
3865 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
3866 getPointerTy(DAG.getDataLayout()));
3867
3868 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3869 SmallSet<unsigned, 8> RegsUsed;
3870 SmallVector<SDValue, 8> MemOpChains;
3871 auto PtrVT = getPointerTy(DAG.getDataLayout());
3872
3873 if (IsVarArg && CLI.CS && CLI.CS.isMustTailCall()) {
3874 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
3875 for (const auto &F : Forwards) {
3876 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
3877 RegsToPass.emplace_back(F.PReg, Val);
3878 }
3879 }
3880
3881 // Walk the register/memloc assignments, inserting copies/loads.
3882 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
3883 ++i, ++realArgIdx) {
3884 CCValAssign &VA = ArgLocs[i];
3885 SDValue Arg = OutVals[realArgIdx];
3886 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
3887
3888 // Promote the value if needed.
3889 switch (VA.getLocInfo()) {
3890 default:
3891 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3891)
;
3892 case CCValAssign::Full:
3893 break;
3894 case CCValAssign::SExt:
3895 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
3896 break;
3897 case CCValAssign::ZExt:
3898 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3899 break;
3900 case CCValAssign::AExt:
3901 if (Outs[realArgIdx].ArgVT == MVT::i1) {
3902 // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
3903 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
3904 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
3905 }
3906 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3907 break;
3908 case CCValAssign::AExtUpper:
3909 assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits")((VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::i32 && \"only expect 32 -> 64 upper bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3909, __PRETTY_FUNCTION__))
;
3910 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3911 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
3912 DAG.getConstant(32, DL, VA.getLocVT()));
3913 break;
3914 case CCValAssign::BCvt:
3915 Arg = DAG.getBitcast(VA.getLocVT(), Arg);
3916 break;
3917 case CCValAssign::Trunc:
3918 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
3919 break;
3920 case CCValAssign::FPExt:
3921 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
3922 break;
3923 case CCValAssign::Indirect:
3924 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3925, __PRETTY_FUNCTION__))
3925 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3925, __PRETTY_FUNCTION__))
;
3926 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3926)
;
3927 }
3928
3929 if (VA.isRegLoc()) {
3930 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
3931 Outs[0].VT == MVT::i64) {
3932 assert(VA.getLocVT() == MVT::i64 &&((VA.getLocVT() == MVT::i64 && "unexpected calling convention register assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocVT() == MVT::i64 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3933, __PRETTY_FUNCTION__))
3933 "unexpected calling convention register assignment")((VA.getLocVT() == MVT::i64 && "unexpected calling convention register assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocVT() == MVT::i64 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3933, __PRETTY_FUNCTION__))
;
3934 assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&((!Ins.empty() && Ins[0].VT == MVT::i64 && "unexpected use of 'returned'"
) ? static_cast<void> (0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i64 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3935, __PRETTY_FUNCTION__))
3935 "unexpected use of 'returned'")((!Ins.empty() && Ins[0].VT == MVT::i64 && "unexpected use of 'returned'"
) ? static_cast<void> (0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i64 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3935, __PRETTY_FUNCTION__))
;
3936 IsThisReturn = true;
3937 }
3938 if (RegsUsed.count(VA.getLocReg())) {
3939 // If this register has already been used then we're trying to pack
3940 // parts of an [N x i32] into an X-register. The extension type will
3941 // take care of putting the two halves in the right place but we have to
3942 // combine them.
3943 SDValue &Bits =
3944 std::find_if(RegsToPass.begin(), RegsToPass.end(),
3945 [=](const std::pair<unsigned, SDValue> &Elt) {
3946 return Elt.first == VA.getLocReg();
3947 })
3948 ->second;
3949 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
3950 // Call site info is used for function's parameter entry value
3951 // tracking. For now we track only simple cases when parameter
3952 // is transferred through whole register.
3953 CSInfo.erase(std::remove_if(CSInfo.begin(), CSInfo.end(),
3954 [&VA](MachineFunction::ArgRegPair ArgReg) {
3955 return ArgReg.Reg == VA.getLocReg();
3956 }),
3957 CSInfo.end());
3958 } else {
3959 RegsToPass.emplace_back(VA.getLocReg(), Arg);
3960 RegsUsed.insert(VA.getLocReg());
3961 const TargetOptions &Options = DAG.getTarget().Options;
3962 if (Options.EnableDebugEntryValues)
3963 CSInfo.emplace_back(VA.getLocReg(), i);
3964 }
3965 } else {
3966 assert(VA.isMemLoc())((VA.isMemLoc()) ? static_cast<void> (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3966, __PRETTY_FUNCTION__))
;
3967
3968 SDValue DstAddr;
3969 MachinePointerInfo DstInfo;
3970
3971 // FIXME: This works on big-endian for composite byvals, which are the
3972 // common case. It should also work for fundamental types too.
3973 uint32_t BEAlign = 0;
3974 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
3975 : VA.getValVT().getSizeInBits();
3976 OpSize = (OpSize + 7) / 8;
3977 if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
3978 !Flags.isInConsecutiveRegs()) {
3979 if (OpSize < 8)
3980 BEAlign = 8 - OpSize;
3981 }
3982 unsigned LocMemOffset = VA.getLocMemOffset();
3983 int32_t Offset = LocMemOffset + BEAlign;
3984 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3985 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
3986
3987 if (IsTailCall) {
3988 Offset = Offset + FPDiff;
3989 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
3990
3991 DstAddr = DAG.getFrameIndex(FI, PtrVT);
3992 DstInfo =
3993 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
3994
3995 // Make sure any stack arguments overlapping with where we're storing
3996 // are loaded before this eventual operation. Otherwise they'll be
3997 // clobbered.
3998 Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
3999 } else {
4000 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
4001
4002 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
4003 DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(),
4004 LocMemOffset);
4005 }
4006
4007 if (Outs[i].Flags.isByVal()) {
4008 SDValue SizeNode =
4009 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
4010 SDValue Cpy = DAG.getMemcpy(
4011 Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
4012 /*isVol = */ false, /*AlwaysInline = */ false,
4013 /*isTailCall = */ false,
4014 DstInfo, MachinePointerInfo());
4015
4016 MemOpChains.push_back(Cpy);
4017 } else {
4018 // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
4019 // promoted to a legal register type i32, we should truncate Arg back to
4020 // i1/i8/i16.
4021 if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
4022 VA.getValVT() == MVT::i16)
4023 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
4024
4025 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
4026 MemOpChains.push_back(Store);
4027 }
4028 }
4029 }
4030
4031 if (!MemOpChains.empty())
4032 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
4033
4034 // Build a sequence of copy-to-reg nodes chained together with token chain
4035 // and flag operands which copy the outgoing args into the appropriate regs.
4036 SDValue InFlag;
4037 for (auto &RegToPass : RegsToPass) {
4038 Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
4039 RegToPass.second, InFlag);
4040 InFlag = Chain.getValue(1);
4041 }
4042
4043 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
4044 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
4045 // node so that legalize doesn't hack it.
4046 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
4047 auto GV = G->getGlobal();
4048 if (Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine()) ==
4049 AArch64II::MO_GOT) {
4050 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT);
4051 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
4052 } else if (Subtarget->isTargetCOFF() && GV->hasDLLImportStorageClass()) {
4053 assert(Subtarget->isTargetWindows() &&((Subtarget->isTargetWindows() && "Windows is the only supported COFF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4054, __PRETTY_FUNCTION__))
4054 "Windows is the only supported COFF target")((Subtarget->isTargetWindows() && "Windows is the only supported COFF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4054, __PRETTY_FUNCTION__))
;
4055 Callee = getGOT(G, DAG, AArch64II::MO_DLLIMPORT);
4056 } else {
4057 const GlobalValue *GV = G->getGlobal();
4058 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
4059 }
4060 } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
4061 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
4062 Subtarget->isTargetMachO()) {
4063 const char *Sym = S->getSymbol();
4064 Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
4065 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
4066 } else {
4067 const char *Sym = S->getSymbol();
4068 Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
4069 }
4070 }
4071
4072 // We don't usually want to end the call-sequence here because we would tidy
4073 // the frame up *after* the call, however in the ABI-changing tail-call case
4074 // we've carefully laid out the parameters so that when sp is reset they'll be
4075 // in the correct location.
4076 if (IsTailCall && !IsSibCall) {
4077 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4078 DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
4079 InFlag = Chain.getValue(1);
4080 }
4081
4082 std::vector<SDValue> Ops;
4083 Ops.push_back(Chain);
4084 Ops.push_back(Callee);
4085
4086 if (IsTailCall) {
4087 // Each tail call may have to adjust the stack by a different amount, so
4088 // this information must travel along with the operation for eventual
4089 // consumption by emitEpilogue.
4090 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
4091 }
4092
4093 // Add argument registers to the end of the list so that they are known live
4094 // into the call.
4095 for (auto &RegToPass : RegsToPass)
4096 Ops.push_back(DAG.getRegister(RegToPass.first,
4097 RegToPass.second.getValueType()));
4098
4099 // Check callee args/returns for SVE registers and set calling convention
4100 // accordingly.
4101 if (CallConv == CallingConv::C) {
4102 bool CalleeOutSVE = any_of(Outs, [](ISD::OutputArg &Out){
4103 return Out.VT.isScalableVector();
4104 });
4105 bool CalleeInSVE = any_of(Ins, [](ISD::InputArg &In){
4106 return In.VT.isScalableVector();
4107 });
4108
4109 if (CalleeInSVE || CalleeOutSVE)
4110 CallConv = CallingConv::AArch64_SVE_VectorCall;
4111 }
4112
4113 // Add a register mask operand representing the call-preserved registers.
4114 const uint32_t *Mask;
4115 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4116 if (IsThisReturn) {
4117 // For 'this' returns, use the X0-preserving mask if applicable
4118 Mask = TRI->getThisReturnPreservedMask(MF, CallConv);
4119 if (!Mask) {
4120 IsThisReturn = false;
4121 Mask = TRI->getCallPreservedMask(MF, CallConv);
4122 }
4123 } else
4124 Mask = TRI->getCallPreservedMask(MF, CallConv);
4125
4126 if (Subtarget->hasCustomCallingConv())
4127 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
4128
4129 if (TRI->isAnyArgRegReserved(MF))
4130 TRI->emitReservedArgRegCallError(MF);
4131
4132 assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention"
) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4132, __PRETTY_FUNCTION__))
;
4133 Ops.push_back(DAG.getRegisterMask(Mask));
4134
4135 if (InFlag.getNode())
4136 Ops.push_back(InFlag);
4137
4138 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4139
4140 // If we're doing a tall call, use a TC_RETURN here rather than an
4141 // actual call instruction.
4142 if (IsTailCall) {
4143 MF.getFrameInfo().setHasTailCall();
4144 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
4145 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
4146 return Ret;
4147 }
4148
4149 // Returns a chain and a flag for retval copy to use.
4150 Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
4151 InFlag = Chain.getValue(1);
4152 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
4153
4154 uint64_t CalleePopBytes =
4155 DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
4156
4157 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4158 DAG.getIntPtrConstant(CalleePopBytes, DL, true),
4159 InFlag, DL);
4160 if (!Ins.empty())
4161 InFlag = Chain.getValue(1);
4162
4163 // Handle result values, copying them out of physregs into vregs that we
4164 // return.
4165 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
4166 InVals, IsThisReturn,
4167 IsThisReturn ? OutVals[0] : SDValue());
4168}
4169
4170bool AArch64TargetLowering::CanLowerReturn(
4171 CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
4172 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
4173 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
4174 ? RetCC_AArch64_WebKit_JS
4175 : RetCC_AArch64_AAPCS;
4176 SmallVector<CCValAssign, 16> RVLocs;
4177 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
4178 return CCInfo.CheckReturn(Outs, RetCC);
4179}
4180
4181SDValue
4182AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4183 bool isVarArg,
4184 const SmallVectorImpl<ISD::OutputArg> &Outs,
4185 const SmallVectorImpl<SDValue> &OutVals,
4186 const SDLoc &DL, SelectionDAG &DAG) const {
4187 auto &MF = DAG.getMachineFunction();
4188 auto *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4189
4190 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
4191 ? RetCC_AArch64_WebKit_JS
4192 : RetCC_AArch64_AAPCS;
4193 SmallVector<CCValAssign, 16> RVLocs;
4194 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
4195 *DAG.getContext());
4196 CCInfo.AnalyzeReturn(Outs, RetCC);
4197
4198 // Copy the result values into the output registers.
4199 SDValue Flag;
4200 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
4201 SmallSet<unsigned, 4> RegsUsed;
4202 for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
4203 ++i, ++realRVLocIdx) {
4204 CCValAssign &VA = RVLocs[i];
4205 assert(VA.isRegLoc() && "Can only return in registers!")((VA.isRegLoc() && "Can only return in registers!") ?
static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4205, __PRETTY_FUNCTION__))
;
4206 SDValue Arg = OutVals[realRVLocIdx];
4207
4208 switch (VA.getLocInfo()) {
4209 default:
4210 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4210)
;
4211 case CCValAssign::Full:
4212 if (Outs[i].ArgVT == MVT::i1) {
4213 // AAPCS requires i1 to be zero-extended to i8 by the producer of the
4214 // value. This is strictly redundant on Darwin (which uses "zeroext
4215 // i1"), but will be optimised out before ISel.
4216 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
4217 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
4218 }
4219 break;
4220 case CCValAssign::BCvt:
4221 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
4222 break;
4223 case CCValAssign::AExt:
4224 case CCValAssign::ZExt:
4225 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4226 break;
4227 case CCValAssign::AExtUpper:
4228 assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits")((VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::i32 && \"only expect 32 -> 64 upper bits\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4228, __PRETTY_FUNCTION__))
;
4229 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4230 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
4231 DAG.getConstant(32, DL, VA.getLocVT()));
4232 break;
4233 }
4234
4235 if (RegsUsed.count(VA.getLocReg())) {
4236 SDValue &Bits =
4237 std::find_if(RetVals.begin(), RetVals.end(),
4238 [=](const std::pair<unsigned, SDValue> &Elt) {
4239 return Elt.first == VA.getLocReg();
4240 })
4241 ->second;
4242 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
4243 } else {
4244 RetVals.emplace_back(VA.getLocReg(), Arg);
4245 RegsUsed.insert(VA.getLocReg());
4246 }
4247 }
4248
4249 SmallVector<SDValue, 4> RetOps(1, Chain);
4250 for (auto &RetVal : RetVals) {
4251 Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Flag);
4252 Flag = Chain.getValue(1);
4253 RetOps.push_back(
4254 DAG.getRegister(RetVal.first, RetVal.second.getValueType()));
4255 }
4256
4257 // Windows AArch64 ABIs require that for returning structs by value we copy
4258 // the sret argument into X0 for the return.
4259 // We saved the argument into a virtual register in the entry block,
4260 // so now we copy the value out and into X0.
4261 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
4262 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
4263 getPointerTy(MF.getDataLayout()));
4264
4265 unsigned RetValReg = AArch64::X0;
4266 Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag);
4267 Flag = Chain.getValue(1);
4268
4269 RetOps.push_back(
4270 DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
4271 }
4272
4273 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4274 const MCPhysReg *I =
4275 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
4276 if (I) {
4277 for (; *I; ++I) {
4278 if (AArch64::GPR64RegClass.contains(*I))
4279 RetOps.push_back(DAG.getRegister(*I, MVT::i64));
4280 else if (AArch64::FPR64RegClass.contains(*I))
4281 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
4282 else
4283 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4283)
;
4284 }
4285 }
4286
4287 RetOps[0] = Chain; // Update chain.
4288
4289 // Add the flag if we have it.
4290 if (Flag.getNode())
4291 RetOps.push_back(Flag);
4292
4293 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
4294}
4295
4296//===----------------------------------------------------------------------===//
4297// Other Lowering Code
4298//===----------------------------------------------------------------------===//
4299
4300SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
4301 SelectionDAG &DAG,
4302 unsigned Flag) const {
4303 return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty,
4304 N->getOffset(), Flag);
4305}
4306
4307SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
4308 SelectionDAG &DAG,
4309 unsigned Flag) const {
4310 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
4311}
4312
4313SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
4314 SelectionDAG &DAG,
4315 unsigned Flag) const {
4316 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
4317 N->getOffset(), Flag);
4318}
4319
4320SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
4321 SelectionDAG &DAG,
4322 unsigned Flag) const {
4323 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
4324}
4325
4326// (loadGOT sym)
4327template <class NodeTy>
4328SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
4329 unsigned Flags) const {
4330 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getGOT\n"
; } } while (false)
;
4331 SDLoc DL(N);
4332 EVT Ty = getPointerTy(DAG.getDataLayout());
4333 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
4334 // FIXME: Once remat is capable of dealing with instructions with register
4335 // operands, expand this into two nodes instead of using a wrapper node.
4336 return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
4337}
4338
4339// (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
4340template <class NodeTy>
4341SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
4342 unsigned Flags) const {
4343 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddrLarge\n"
; } } while (false)
;
4344 SDLoc DL(N);
4345 EVT Ty = getPointerTy(DAG.getDataLayout());
4346 const unsigned char MO_NC = AArch64II::MO_NC;
4347 return DAG.getNode(
4348 AArch64ISD::WrapperLarge, DL, Ty,
4349 getTargetNode(N, Ty, DAG, AArch64II::MO_G3 | Flags),
4350 getTargetNode(N, Ty, DAG, AArch64II::MO_G2 | MO_NC | Flags),
4351 getTargetNode(N, Ty, DAG, AArch64II::MO_G1 | MO_NC | Flags),
4352 getTargetNode(N, Ty, DAG, AArch64II::MO_G0 | MO_NC | Flags));
4353}
4354
4355// (addlow (adrp %hi(sym)) %lo(sym))
4356template <class NodeTy>
4357SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4358 unsigned Flags) const {
4359 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddr\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddr\n"
; } } while (false)
;
4360 SDLoc DL(N);
4361 EVT Ty = getPointerTy(DAG.getDataLayout());
4362 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
4363 SDValue Lo = getTargetNode(N, Ty, DAG,
4364 AArch64II::MO_PAGEOFF | AArch64II::MO_NC | Flags);
4365 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
4366 return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
4367}
4368
4369// (adr sym)
4370template <class NodeTy>
4371SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
4372 unsigned Flags) const {
4373 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrTiny\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddrTiny\n"
; } } while (false)
;
4374 SDLoc DL(N);
4375 EVT Ty = getPointerTy(DAG.getDataLayout());
4376 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
4377 return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
4378}
4379
4380SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4381 SelectionDAG &DAG) const {
4382 GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
4383 const GlobalValue *GV = GN->getGlobal();
4384 unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
4385
4386 if (OpFlags != AArch64II::MO_NO_FLAG)
4387 assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&((cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
"unexpected offset in global node") ? static_cast<void>
(0) : __assert_fail ("cast<GlobalAddressSDNode>(Op)->getOffset() == 0 && \"unexpected offset in global node\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4388, __PRETTY_FUNCTION__))
4388 "unexpected offset in global node")((cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
"unexpected offset in global node") ? static_cast<void>
(0) : __assert_fail ("cast<GlobalAddressSDNode>(Op)->getOffset() == 0 && \"unexpected offset in global node\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4388, __PRETTY_FUNCTION__))
;
4389
4390 // This also catches the large code model case for Darwin, and tiny code
4391 // model with got relocations.
4392 if ((OpFlags & AArch64II::MO_GOT) != 0) {
4393 return getGOT(GN, DAG, OpFlags);
4394 }
4395
4396 SDValue Result;
4397 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
4398 Result = getAddrLarge(GN, DAG, OpFlags);
4399 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4400 Result = getAddrTiny(GN, DAG, OpFlags);
4401 } else {
4402 Result = getAddr(GN, DAG, OpFlags);
4403 }
4404 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4405 SDLoc DL(GN);
4406 if (OpFlags & (AArch64II::MO_DLLIMPORT | AArch64II::MO_COFFSTUB))
4407 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4408 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4409 return Result;
4410}
4411
4412/// Convert a TLS address reference into the correct sequence of loads
4413/// and calls to compute the variable's address (for Darwin, currently) and
4414/// return an SDValue containing the final node.
4415
4416/// Darwin only has one TLS scheme which must be capable of dealing with the
4417/// fully general situation, in the worst case. This means:
4418/// + "extern __thread" declaration.
4419/// + Defined in a possibly unknown dynamic library.
4420///
4421/// The general system is that each __thread variable has a [3 x i64] descriptor
4422/// which contains information used by the runtime to calculate the address. The
4423/// only part of this the compiler needs to know about is the first xword, which
4424/// contains a function pointer that must be called with the address of the
4425/// entire descriptor in "x0".
4426///
4427/// Since this descriptor may be in a different unit, in general even the
4428/// descriptor must be accessed via an indirect load. The "ideal" code sequence
4429/// is:
4430/// adrp x0, _var@TLVPPAGE
4431/// ldr x0, [x0, _var@TLVPPAGEOFF] ; x0 now contains address of descriptor
4432/// ldr x1, [x0] ; x1 contains 1st entry of descriptor,
4433/// ; the function pointer
4434/// blr x1 ; Uses descriptor address in x0
4435/// ; Address of _var is now in x0.
4436///
4437/// If the address of _var's descriptor *is* known to the linker, then it can
4438/// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
4439/// a slight efficiency gain.
4440SDValue
4441AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
4442 SelectionDAG &DAG) const {
4443 assert(Subtarget->isTargetDarwin() &&((Subtarget->isTargetDarwin() && "This function expects a Darwin target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4444, __PRETTY_FUNCTION__))
4444 "This function expects a Darwin target")((Subtarget->isTargetDarwin() && "This function expects a Darwin target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4444, __PRETTY_FUNCTION__))
;
4445
4446 SDLoc DL(Op);
4447 MVT PtrVT = getPointerTy(DAG.getDataLayout());
4448 MVT PtrMemVT = getPointerMemTy(DAG.getDataLayout());
4449 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
4450
4451 SDValue TLVPAddr =
4452 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4453 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
4454
4455 // The first entry in the descriptor is a function pointer that we must call
4456 // to obtain the address of the variable.
4457 SDValue Chain = DAG.getEntryNode();
4458 SDValue FuncTLVGet = DAG.getLoad(
4459 PtrMemVT, DL, Chain, DescAddr,
4460 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
4461 /* Alignment = */ PtrMemVT.getSizeInBits() / 8,
4462 MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
4463 Chain = FuncTLVGet.getValue(1);
4464
4465 // Extend loaded pointer if necessary (i.e. if ILP32) to DAG pointer.
4466 FuncTLVGet = DAG.getZExtOrTrunc(FuncTLVGet, DL, PtrVT);
4467
4468 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
4469 MFI.setAdjustsStack(true);
4470
4471 // TLS calls preserve all registers except those that absolutely must be
4472 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
4473 // silly).
4474 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4475 const uint32_t *Mask = TRI->getTLSCallPreservedMask();
4476 if (Subtarget->hasCustomCallingConv())
4477 TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
4478
4479 // Finally, we can make the call. This is just a degenerate version of a
4480 // normal AArch64 call node: x0 takes the address of the descriptor, and
4481 // returns the address of the variable in this thread.
4482 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
4483 Chain =
4484 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
4485 Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
4486 DAG.getRegisterMask(Mask), Chain.getValue(1));
4487 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
4488}
4489
4490/// When accessing thread-local variables under either the general-dynamic or
4491/// local-dynamic system, we make a "TLS-descriptor" call. The variable will
4492/// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
4493/// is a function pointer to carry out the resolution.
4494///
4495/// The sequence is:
4496/// adrp x0, :tlsdesc:var
4497/// ldr x1, [x0, #:tlsdesc_lo12:var]
4498/// add x0, x0, #:tlsdesc_lo12:var
4499/// .tlsdesccall var
4500/// blr x1
4501/// (TPIDR_EL0 offset now in x0)
4502///
4503/// The above sequence must be produced unscheduled, to enable the linker to
4504/// optimize/relax this sequence.
4505/// Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the
4506/// above sequence, and expanded really late in the compilation flow, to ensure
4507/// the sequence is produced as per above.
4508SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
4509 const SDLoc &DL,
4510 SelectionDAG &DAG) const {
4511 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4512
4513 SDValue Chain = DAG.getEntryNode();
4514 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4515
4516 Chain =
4517 DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
4518 SDValue Glue = Chain.getValue(1);
4519
4520 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
4521}
4522
4523SDValue
4524AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
4525 SelectionDAG &DAG) const {
4526 assert(Subtarget->isTargetELF() && "This function expects an ELF target")((Subtarget->isTargetELF() && "This function expects an ELF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetELF() && \"This function expects an ELF target\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4526, __PRETTY_FUNCTION__))
;
4527 if (getTargetMachine().getCodeModel() == CodeModel::Large)
4528 report_fatal_error("ELF TLS only supported in small memory model");
4529 // Different choices can be made for the maximum size of the TLS area for a
4530 // module. For the small address model, the default TLS size is 16MiB and the
4531 // maximum TLS size is 4GiB.
4532 // FIXME: add -mtls-size command line option and make it control the 16MiB
4533 // vs. 4GiB code sequence generation.
4534 // FIXME: add tiny codemodel support. We currently generate the same code as
4535 // small, which may be larger than needed.
4536 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4537
4538 TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
4539
4540 if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
4541 if (Model == TLSModel::LocalDynamic)
4542 Model = TLSModel::GeneralDynamic;
4543 }
4544
4545 SDValue TPOff;
4546 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4547 SDLoc DL(Op);
4548 const GlobalValue *GV = GA->getGlobal();
4549
4550 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
4551
4552 if (Model == TLSModel::LocalExec) {
4553 SDValue HiVar = DAG.getTargetGlobalAddress(
4554 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4555 SDValue LoVar = DAG.getTargetGlobalAddress(
4556 GV, DL, PtrVT, 0,
4557 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4558
4559 SDValue TPWithOff_lo =
4560 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
4561 HiVar,
4562 DAG.getTargetConstant(0, DL, MVT::i32)),
4563 0);
4564 SDValue TPWithOff =
4565 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPWithOff_lo,
4566 LoVar,
4567 DAG.getTargetConstant(0, DL, MVT::i32)),
4568 0);
4569 return TPWithOff;
4570 } else if (Model == TLSModel::InitialExec) {
4571 TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4572 TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
4573 } else if (Model == TLSModel::LocalDynamic) {
4574 // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
4575 // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
4576 // the beginning of the module's TLS region, followed by a DTPREL offset
4577 // calculation.
4578
4579 // These accesses will need deduplicating if there's more than one.
4580 AArch64FunctionInfo *MFI =
4581 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
4582 MFI->incNumLocalDynamicTLSAccesses();
4583
4584 // The call needs a relocation too for linker relaxation. It doesn't make
4585 // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4586 // the address.
4587 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
4588 AArch64II::MO_TLS);
4589
4590 // Now we can calculate the offset from TPIDR_EL0 to this module's
4591 // thread-local area.
4592 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4593
4594 // Now use :dtprel_whatever: operations to calculate this variable's offset
4595 // in its thread-storage area.
4596 SDValue HiVar = DAG.getTargetGlobalAddress(
4597 GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4598 SDValue LoVar = DAG.getTargetGlobalAddress(
4599 GV, DL, MVT::i64, 0,
4600 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4601
4602 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
4603 DAG.getTargetConstant(0, DL, MVT::i32)),
4604 0);
4605 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
4606 DAG.getTargetConstant(0, DL, MVT::i32)),
4607 0);
4608 } else if (Model == TLSModel::GeneralDynamic) {
4609 // The call needs a relocation too for linker relaxation. It doesn't make
4610 // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4611 // the address.
4612 SDValue SymAddr =
4613 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4614
4615 // Finally we can make a call to calculate the offset from tpidr_el0.
4616 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4617 } else
4618 llvm_unreachable("Unsupported ELF TLS access model")::llvm::llvm_unreachable_internal("Unsupported ELF TLS access model"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4618)
;
4619
4620 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
4621}
4622
4623SDValue
4624AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
4625 SelectionDAG &DAG) const {
4626 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering")((Subtarget->isTargetWindows() && "Windows specific TLS lowering"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows specific TLS lowering\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4626, __PRETTY_FUNCTION__))
;
4627
4628 SDValue Chain = DAG.getEntryNode();
4629 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4630 SDLoc DL(Op);
4631
4632 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
4633
4634 // Load the ThreadLocalStoragePointer from the TEB
4635 // A pointer to the TLS array is located at offset 0x58 from the TEB.
4636 SDValue TLSArray =
4637 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
4638 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
4639 Chain = TLSArray.getValue(1);
4640
4641 // Load the TLS index from the C runtime;
4642 // This does the same as getAddr(), but without having a GlobalAddressSDNode.
4643 // This also does the same as LOADgot, but using a generic i32 load,
4644 // while LOADgot only loads i64.
4645 SDValue TLSIndexHi =
4646 DAG.getTargetExternalSymbol("_tls_index", PtrVT, AArch64II::MO_PAGE);
4647 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
4648 "_tls_index", PtrVT, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4649 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
4650 SDValue TLSIndex =
4651 DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
4652 TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
4653 Chain = TLSIndex.getValue(1);
4654
4655 // The pointer to the thread's TLS data area is at the TLS Index scaled by 8
4656 // offset into the TLSArray.
4657 TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
4658 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
4659 DAG.getConstant(3, DL, PtrVT));
4660 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
4661 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
4662 MachinePointerInfo());
4663 Chain = TLS.getValue(1);
4664
4665 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4666 const GlobalValue *GV = GA->getGlobal();
4667 SDValue TGAHi = DAG.getTargetGlobalAddress(
4668 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4669 SDValue TGALo = DAG.getTargetGlobalAddress(
4670 GV, DL, PtrVT, 0,
4671 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4672
4673 // Add the offset from the start of the .tls section (section base).
4674 SDValue Addr =
4675 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
4676 DAG.getTargetConstant(0, DL, MVT::i32)),
4677 0);
4678 Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
4679 return Addr;
4680}
4681
4682SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
4683 SelectionDAG &DAG) const {
4684 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4685 if (DAG.getTarget().useEmulatedTLS())
4686 return LowerToTLSEmulatedModel(GA, DAG);
4687
4688 if (Subtarget->isTargetDarwin())
4689 return LowerDarwinGlobalTLSAddress(Op, DAG);
4690 if (Subtarget->isTargetELF())
4691 return LowerELFGlobalTLSAddress(Op, DAG);
4692 if (Subtarget->isTargetWindows())
4693 return LowerWindowsGlobalTLSAddress(Op, DAG);
4694
4695 llvm_unreachable("Unexpected platform trying to use TLS")::llvm::llvm_unreachable_internal("Unexpected platform trying to use TLS"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4695)
;
4696}
4697
4698SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4699 SDValue Chain = Op.getOperand(0);
4700 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
4701 SDValue LHS = Op.getOperand(2);
4702 SDValue RHS = Op.getOperand(3);
4703 SDValue Dest = Op.getOperand(4);
4704 SDLoc dl(Op);
4705
4706 MachineFunction &MF = DAG.getMachineFunction();
4707 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
4708 // will not be produced, as they are conditional branch instructions that do
4709 // not set flags.
4710 bool ProduceNonFlagSettingCondBr =
4711 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
4712
4713 // Handle f128 first, since lowering it will result in comparing the return
4714 // value of a libcall against zero, which is just what the rest of LowerBR_CC
4715 // is expecting to deal with.
4716 if (LHS.getValueType() == MVT::f128) {
4717 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
4718
4719 // If softenSetCCOperands returned a scalar, we need to compare the result
4720 // against zero to select between true and false values.
4721 if (!RHS.getNode()) {
4722 RHS = DAG.getConstant(0, dl, LHS.getValueType());
4723 CC = ISD::SETNE;
4724 }
4725 }
4726
4727 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
4728 // instruction.
4729 if (isOverflowIntrOpRes(LHS) && isOneConstant(RHS) &&
4730 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
4731 // Only lower legal XALUO ops.
4732 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
4733 return SDValue();
4734
4735 // The actual operation with overflow check.
4736 AArch64CC::CondCode OFCC;
4737 SDValue Value, Overflow;
4738 std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
4739
4740 if (CC == ISD::SETNE)
4741 OFCC = getInvertedCondCode(OFCC);
4742 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
4743
4744 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4745 Overflow);
4746 }
4747
4748 if (LHS.getValueType().isInteger()) {
4749 assert((LHS.getValueType() == RHS.getValueType()) &&(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4750, __PRETTY_FUNCTION__))
4750 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64))(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4750, __PRETTY_FUNCTION__))
;
4751
4752 // If the RHS of the comparison is zero, we can potentially fold this
4753 // to a specialized branch.
4754 const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
4755 if (RHSC && RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
4756 if (CC == ISD::SETEQ) {
4757 // See if we can use a TBZ to fold in an AND as well.
4758 // TBZ has a smaller branch displacement than CBZ. If the offset is
4759 // out of bounds, a late MI-layer pass rewrites branches.
4760 // 403.gcc is an example that hits this case.
4761 if (LHS.getOpcode() == ISD::AND &&
4762 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4763 isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4764 SDValue Test = LHS.getOperand(0);
4765 uint64_t Mask = LHS.getConstantOperandVal(1);
4766 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
4767 DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4768 Dest);
4769 }
4770
4771 return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
4772 } else if (CC == ISD::SETNE) {
4773 // See if we can use a TBZ to fold in an AND as well.
4774 // TBZ has a smaller branch displacement than CBZ. If the offset is
4775 // out of bounds, a late MI-layer pass rewrites branches.
4776 // 403.gcc is an example that hits this case.
4777 if (LHS.getOpcode() == ISD::AND &&
4778 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4779 isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4780 SDValue Test = LHS.getOperand(0);
4781 uint64_t Mask = LHS.getConstantOperandVal(1);
4782 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
4783 DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4784 Dest);
4785 }
4786
4787 return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
4788 } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
4789 // Don't combine AND since emitComparison converts the AND to an ANDS
4790 // (a.k.a. TST) and the test in the test bit and branch instruction
4791 // becomes redundant. This would also increase register pressure.
4792 uint64_t Mask = LHS.getValueSizeInBits() - 1;
4793 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
4794 DAG.getConstant(Mask, dl, MVT::i64), Dest);
4795 }
4796 }
4797 if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
4798 LHS.getOpcode() != ISD::AND && ProduceNonFlagSettingCondBr) {
4799 // Don't combine AND since emitComparison converts the AND to an ANDS
4800 // (a.k.a. TST) and the test in the test bit and branch instruction
4801 // becomes redundant. This would also increase register pressure.
4802 uint64_t Mask = LHS.getValueSizeInBits() - 1;
4803 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
4804 DAG.getConstant(Mask, dl, MVT::i64), Dest);
4805 }
4806
4807 SDValue CCVal;
4808 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4809 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4810 Cmp);
4811 }
4812
4813 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4814, __PRETTY_FUNCTION__))
4814 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4814, __PRETTY_FUNCTION__))
;
4815
4816 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
4817 // clean. Some of them require two branches to implement.
4818 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4819 AArch64CC::CondCode CC1, CC2;
4820 changeFPCCToAArch64CC(CC, CC1, CC2);
4821 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4822 SDValue BR1 =
4823 DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
4824 if (CC2 != AArch64CC::AL) {
4825 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4826 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
4827 Cmp);
4828 }
4829
4830 return BR1;
4831}
4832
4833SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
4834 SelectionDAG &DAG) const {
4835 EVT VT = Op.getValueType();
4836 SDLoc DL(Op);
4837
4838 SDValue In1 = Op.getOperand(0);
4839 SDValue In2 = Op.getOperand(1);
4840 EVT SrcVT = In2.getValueType();
4841
4842 if (SrcVT.bitsLT(VT))
4843 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
4844 else if (SrcVT.bitsGT(VT))
4845 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
4846
4847 EVT VecVT;
4848 uint64_t EltMask;
4849 SDValue VecVal1, VecVal2;
4850
4851 auto setVecVal = [&] (int Idx) {
4852 if (!VT.isVector()) {
4853 VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4854 DAG.getUNDEF(VecVT), In1);
4855 VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4856 DAG.getUNDEF(VecVT), In2);
4857 } else {
4858 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
4859 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
4860 }
4861 };
4862
4863 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
4864 VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32);
4865 EltMask = 0x80000000ULL;
4866 setVecVal(AArch64::ssub);
4867 } else if (VT == MVT::f64 || VT == MVT::v2f64) {
4868 VecVT = MVT::v2i64;
4869
4870 // We want to materialize a mask with the high bit set, but the AdvSIMD
4871 // immediate moves cannot materialize that in a single instruction for
4872 // 64-bit elements. Instead, materialize zero and then negate it.
4873 EltMask = 0;
4874
4875 setVecVal(AArch64::dsub);
4876 } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) {
4877 VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16);
4878 EltMask = 0x8000ULL;
4879 setVecVal(AArch64::hsub);
4880 } else {
4881 llvm_unreachable("Invalid type for copysign!")::llvm::llvm_unreachable_internal("Invalid type for copysign!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4881)
;
4882 }
4883
4884 SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
4885
4886 // If we couldn't materialize the mask above, then the mask vector will be
4887 // the zero vector, and we need to negate it here.
4888 if (VT == MVT::f64 || VT == MVT::v2f64) {
4889 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
4890 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
4891 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
4892 }
4893
4894 SDValue Sel =
4895 DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
4896
4897 if (VT == MVT::f16)
4898 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
4899 if (VT == MVT::f32)
4900 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
4901 else if (VT == MVT::f64)
4902 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
4903 else
4904 return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
4905}
4906
4907SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
4908 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
4909 Attribute::NoImplicitFloat))
4910 return SDValue();
4911
4912 if (!Subtarget->hasNEON())
4913 return SDValue();
4914
4915 // While there is no integer popcount instruction, it can
4916 // be more efficiently lowered to the following sequence that uses
4917 // AdvSIMD registers/instructions as long as the copies to/from
4918 // the AdvSIMD registers are cheap.
4919 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
4920 // CNT V0.8B, V0.8B // 8xbyte pop-counts
4921 // ADDV B0, V0.8B // sum 8xbyte pop-counts
4922 // UMOV X0, V0.B[0] // copy byte result back to integer reg
4923 SDValue Val = Op.getOperand(0);
4924 SDLoc DL(Op);
4925 EVT VT = Op.getValueType();
4926
4927 if (VT == MVT::i32 || VT == MVT::i64) {
4928 if (VT == MVT::i32)
4929 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
4930 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
4931
4932 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
4933 SDValue UaddLV = DAG.getNode(
4934 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
4935 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
4936
4937 if (VT == MVT::i64)
4938 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
4939 return UaddLV;
4940 }
4941
4942 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4944, __PRETTY_FUNCTION__))
4943 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4944, __PRETTY_FUNCTION__))
4944 "Unexpected type for custom ctpop lowering")(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4944, __PRETTY_FUNCTION__))
;
4945
4946 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4947 Val = DAG.getBitcast(VT8Bit, Val);
4948 Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
4949
4950 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
4951 unsigned EltSize = 8;
4952 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
4953 while (EltSize != VT.getScalarSizeInBits()) {
4954 EltSize *= 2;
4955 NumElts /= 2;
4956 MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
4957 Val = DAG.getNode(
4958 ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
4959 DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
4960 }
4961
4962 return Val;
4963}
4964
4965SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
4966
4967 if (Op.getValueType().isVector())
4968 return LowerVSETCC(Op, DAG);
4969
4970 SDValue LHS = Op.getOperand(0);
4971 SDValue RHS = Op.getOperand(1);
4972 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
4973 SDLoc dl(Op);
4974
4975 // We chose ZeroOrOneBooleanContents, so use zero and one.
4976 EVT VT = Op.getValueType();
4977 SDValue TVal = DAG.getConstant(1, dl, VT);
4978 SDValue FVal = DAG.getConstant(0, dl, VT);
4979
4980 // Handle f128 first, since one possible outcome is a normal integer
4981 // comparison which gets picked up by the next if statement.
4982 if (LHS.getValueType() == MVT::f128) {
4983 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
4984
4985 // If softenSetCCOperands returned a scalar, use it.
4986 if (!RHS.getNode()) {
4987 assert(LHS.getValueType() == Op.getValueType() &&((LHS.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4988, __PRETTY_FUNCTION__))
4988 "Unexpected setcc expansion!")((LHS.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4988, __PRETTY_FUNCTION__))
;
4989 return LHS;
4990 }
4991 }
4992
4993 if (LHS.getValueType().isInteger()) {
4994 SDValue CCVal;
4995 SDValue Cmp =
4996 getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl);
4997
4998 // Note that we inverted the condition above, so we reverse the order of
4999 // the true and false operands here. This will allow the setcc to be
5000 // matched to a single CSINC instruction.
5001 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
5002 }
5003
5004 // Now we know we're dealing with FP values.
5005 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5006, __PRETTY_FUNCTION__))
5006 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5006, __PRETTY_FUNCTION__))
;
5007
5008 // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead
5009 // and do the comparison.
5010 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5011
5012 AArch64CC::CondCode CC1, CC2;
5013 changeFPCCToAArch64CC(CC, CC1, CC2);
5014 if (CC2 == AArch64CC::AL) {
5015 changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2);
5016 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5017
5018 // Note that we inverted the condition above, so we reverse the order of
5019 // the true and false operands here. This will allow the setcc to be
5020 // matched to a single CSINC instruction.
5021 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
5022 } else {
5023 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
5024 // totally clean. Some of them require two CSELs to implement. As is in
5025 // this case, we emit the first CSEL and then emit a second using the output
5026 // of the first as the RHS. We're effectively OR'ing the two CC's together.
5027
5028 // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
5029 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5030 SDValue CS1 =
5031 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
5032
5033 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5034 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
5035 }
5036}
5037
5038SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
5039 SDValue RHS, SDValue TVal,
5040 SDValue FVal, const SDLoc &dl,
5041 SelectionDAG &DAG) const {
5042 // Handle f128 first, because it will result in a comparison of some RTLIB
5043 // call result against zero.
5044 if (LHS.getValueType() == MVT::f128) {
1
Taking true branch
5045 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
5046
5047 // If softenSetCCOperands returned a scalar, we need to compare the result
5048 // against zero to select between true and false values.
5049 if (!RHS.getNode()) {
2
Assuming the condition is false
3
Taking false branch
5050 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5051 CC = ISD::SETNE;
5052 }
5053 }
5054
5055 // Also handle f16, for which we need to do a f32 comparison.
5056 if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
4
Taking false branch
5057 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
5058 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
5059 }
5060
5061 // Next, handle integers.
5062 if (LHS.getValueType().isInteger()) {
5
Taking true branch
5063 assert((LHS.getValueType() == RHS.getValueType()) &&(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5064, __PRETTY_FUNCTION__))
5064 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64))(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5064, __PRETTY_FUNCTION__))
;
5065
5066 unsigned Opcode = AArch64ISD::CSEL;
5067
5068 // If both the TVal and the FVal are constants, see if we can swap them in
5069 // order to for a CSINV or CSINC out of them.
5070 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
5071 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
6
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
21
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
5072
5073 if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
22
Assuming 'CTVal' is null
5074 std::swap(TVal, FVal);
5075 std::swap(CTVal, CFVal);
5076 CC = ISD::getSetCCInverse(CC, true);
5077 } else if (CTVal
22.1
'CTVal' is null
22.1
'CTVal' is null
22.1
'CTVal' is null
&& CFVal && CTVal->isOne() && CFVal->isNullValue()) {
5078 std::swap(TVal, FVal);
5079 std::swap(CTVal, CFVal);
5080 CC = ISD::getSetCCInverse(CC, true);
5081 } else if (TVal.getOpcode() == ISD::XOR) {
23
Calling 'SDValue::getOpcode'
5082 // If TVal is a NOT we want to swap TVal and FVal so that we can match
5083 // with a CSINV rather than a CSEL.
5084 if (isAllOnesConstant(TVal.getOperand(1))) {
5085 std::swap(TVal, FVal);
5086 std::swap(CTVal, CFVal);
5087 CC = ISD::getSetCCInverse(CC, true);
5088 }
5089 } else if (TVal.getOpcode() == ISD::SUB) {
5090 // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
5091 // that we can match with a CSNEG rather than a CSEL.
5092 if (isNullConstant(TVal.getOperand(0))) {
5093 std::swap(TVal, FVal);
5094 std::swap(CTVal, CFVal);
5095 CC = ISD::getSetCCInverse(CC, true);
5096 }
5097 } else if (CTVal && CFVal) {
5098 const int64_t TrueVal = CTVal->getSExtValue();
5099 const int64_t FalseVal = CFVal->getSExtValue();
5100 bool Swap = false;
5101
5102 // If both TVal and FVal are constants, see if FVal is the
5103 // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
5104 // instead of a CSEL in that case.
5105 if (TrueVal == ~FalseVal) {
5106 Opcode = AArch64ISD::CSINV;
5107 } else if (TrueVal == -FalseVal) {
5108 Opcode = AArch64ISD::CSNEG;
5109 } else if (TVal.getValueType() == MVT::i32) {
5110 // If our operands are only 32-bit wide, make sure we use 32-bit
5111 // arithmetic for the check whether we can use CSINC. This ensures that
5112 // the addition in the check will wrap around properly in case there is
5113 // an overflow (which would not be the case if we do the check with
5114 // 64-bit arithmetic).
5115 const uint32_t TrueVal32 = CTVal->getZExtValue();
5116 const uint32_t FalseVal32 = CFVal->getZExtValue();
5117
5118 if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
5119 Opcode = AArch64ISD::CSINC;
5120
5121 if (TrueVal32 > FalseVal32) {
5122 Swap = true;
5123 }
5124 }
5125 // 64-bit check whether we can use CSINC.
5126 } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
5127 Opcode = AArch64ISD::CSINC;
5128
5129 if (TrueVal > FalseVal) {
5130 Swap = true;
5131 }
5132 }
5133
5134 // Swap TVal and FVal if necessary.
5135 if (Swap) {
5136 std::swap(TVal, FVal);
5137 std::swap(CTVal, CFVal);
5138 CC = ISD::getSetCCInverse(CC, true);
5139 }
5140
5141 if (Opcode != AArch64ISD::CSEL) {
5142 // Drop FVal since we can get its value by simply inverting/negating
5143 // TVal.
5144 FVal = TVal;
5145 }
5146 }
5147
5148 // Avoid materializing a constant when possible by reusing a known value in
5149 // a register. However, don't perform this optimization if the known value
5150 // is one, zero or negative one in the case of a CSEL. We can always
5151 // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the
5152 // FVal, respectively.
5153 ConstantSDNode *RHSVal = dyn_cast<ConstantSDNode>(RHS);
5154 if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() &&
5155 !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) {
5156 AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
5157 // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to
5158 // "a != C ? x : a" to avoid materializing C.
5159 if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ)
5160 TVal = LHS;
5161 else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE)
5162 FVal = LHS;
5163 } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) {
5164 assert (CTVal && CFVal && "Expected constant operands for CSNEG.")((CTVal && CFVal && "Expected constant operands for CSNEG."
) ? static_cast<void> (0) : __assert_fail ("CTVal && CFVal && \"Expected constant operands for CSNEG.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5164, __PRETTY_FUNCTION__))
;
5165 // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to
5166 // avoid materializing C.
5167 AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
5168 if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
5169 Opcode = AArch64ISD::CSINV;
5170 TVal = LHS;
5171 FVal = DAG.getConstant(0, dl, FVal.getValueType());
5172 }
5173 }
5174
5175 SDValue CCVal;
5176 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
5177 EVT VT = TVal.getValueType();
5178 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
5179 }
5180
5181 // Now we know we're dealing with FP values.
5182 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5183, __PRETTY_FUNCTION__))
5183 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5183, __PRETTY_FUNCTION__))
;
5184 assert(LHS.getValueType() == RHS.getValueType())((LHS.getValueType() == RHS.getValueType()) ? static_cast<
void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5184, __PRETTY_FUNCTION__))
;
5185 EVT VT = TVal.getValueType();
5186 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5187
5188 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
5189 // clean. Some of them require two CSELs to implement.
5190 AArch64CC::CondCode CC1, CC2;
5191 changeFPCCToAArch64CC(CC, CC1, CC2);
5192
5193 if (DAG.getTarget().Options.UnsafeFPMath) {
5194 // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and
5195 // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0.
5196 ConstantFPSDNode *RHSVal = dyn_cast<ConstantFPSDNode>(RHS);
5197 if (RHSVal && RHSVal->isZero()) {
5198 ConstantFPSDNode *CFVal = dyn_cast<ConstantFPSDNode>(FVal);
5199 ConstantFPSDNode *CTVal = dyn_cast<ConstantFPSDNode>(TVal);
5200
5201 if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
5202 CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType())
5203 TVal = LHS;
5204 else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) &&
5205 CFVal && CFVal->isZero() &&
5206 FVal.getValueType() == LHS.getValueType())
5207 FVal = LHS;
5208 }
5209 }
5210
5211 // Emit first, and possibly only, CSEL.
5212 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5213 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
5214
5215 // If we need a second CSEL, emit it, using the output of the first as the
5216 // RHS. We're effectively OR'ing the two CC's together.
5217 if (CC2 != AArch64CC::AL) {
5218 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5219 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
5220 }
5221
5222 // Otherwise, return the output of the first CSEL.
5223 return CS1;
5224}
5225
5226SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
5227 SelectionDAG &DAG) const {
5228 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5229 SDValue LHS = Op.getOperand(0);
5230 SDValue RHS = Op.getOperand(1);
5231 SDValue TVal = Op.getOperand(2);
5232 SDValue FVal = Op.getOperand(3);
5233 SDLoc DL(Op);
5234 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5235}
5236
5237SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
5238 SelectionDAG &DAG) const {
5239 SDValue CCVal = Op->getOperand(0);
5240 SDValue TVal = Op->getOperand(1);
5241 SDValue FVal = Op->getOperand(2);
5242 SDLoc DL(Op);
5243
5244 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
5245 // instruction.
5246 if (isOverflowIntrOpRes(CCVal)) {
5247 // Only lower legal XALUO ops.
5248 if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
5249 return SDValue();
5250
5251 AArch64CC::CondCode OFCC;
5252 SDValue Value, Overflow;
5253 std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
5254 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
5255
5256 return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
5257 CCVal, Overflow);
5258 }
5259
5260 // Lower it the same way as we would lower a SELECT_CC node.
5261 ISD::CondCode CC;
5262 SDValue LHS, RHS;
5263 if (CCVal.getOpcode() == ISD::SETCC) {
5264 LHS = CCVal.getOperand(0);
5265 RHS = CCVal.getOperand(1);
5266 CC = cast<CondCodeSDNode>(CCVal->getOperand(2))->get();
5267 } else {
5268 LHS = CCVal;
5269 RHS = DAG.getConstant(0, DL, CCVal.getValueType());
5270 CC = ISD::SETNE;
5271 }
5272 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5273}
5274
5275SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
5276 SelectionDAG &DAG) const {
5277 // Jump table entries as PC relative offsets. No additional tweaking
5278 // is necessary here. Just get the address of the jump table.
5279 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
5280
5281 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5282 !Subtarget->isTargetMachO()) {
5283 return getAddrLarge(JT, DAG);
5284 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5285 return getAddrTiny(JT, DAG);
5286 }
5287 return getAddr(JT, DAG);
5288}
5289
5290SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
5291 SelectionDAG &DAG) const {
5292 // Jump table entries as PC relative offsets. No additional tweaking
5293 // is necessary here. Just get the address of the jump table.
5294 SDLoc DL(Op);
5295 SDValue JT = Op.getOperand(1);
5296 SDValue Entry = Op.getOperand(2);
5297 int JTI = cast<JumpTableSDNode>(JT.getNode())->getIndex();
5298
5299 SDNode *Dest =
5300 DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
5301 Entry, DAG.getTargetJumpTable(JTI, MVT::i32));
5302 return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0),
5303 SDValue(Dest, 0));
5304}
5305
5306SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
5307 SelectionDAG &DAG) const {
5308 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
5309
5310 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
5311 // Use the GOT for the large code model on iOS.
5312 if (Subtarget->isTargetMachO()) {
5313 return getGOT(CP, DAG);
5314 }
5315 return getAddrLarge(CP, DAG);
5316 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5317 return getAddrTiny(CP, DAG);
5318 } else {
5319 return getAddr(CP, DAG);
5320 }
5321}
5322
5323SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
5324 SelectionDAG &DAG) const {
5325 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Op);
5326 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5327 !Subtarget->isTargetMachO()) {
5328 return getAddrLarge(BA, DAG);
5329 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5330 return getAddrTiny(BA, DAG);
5331 }
5332 return getAddr(BA, DAG);
5333}
5334
5335SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
5336 SelectionDAG &DAG) const {
5337 AArch64FunctionInfo *FuncInfo =
5338 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
5339
5340 SDLoc DL(Op);
5341 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
5342 getPointerTy(DAG.getDataLayout()));
5343 FR = DAG.getZExtOrTrunc(FR, DL, getPointerMemTy(DAG.getDataLayout()));
5344 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5345 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
5346 MachinePointerInfo(SV));
5347}
5348
5349SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
5350 SelectionDAG &DAG) const {
5351 AArch64FunctionInfo *FuncInfo =
5352 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
5353
5354 SDLoc DL(Op);
5355 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
5356 ? FuncInfo->getVarArgsGPRIndex()
5357 : FuncInfo->getVarArgsStackIndex(),
5358 getPointerTy(DAG.getDataLayout()));
5359 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5360 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
5361 MachinePointerInfo(SV));
5362}
5363
5364SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
5365 SelectionDAG &DAG) const {
5366 // The layout of the va_list struct is specified in the AArch64 Procedure Call
5367 // Standard, section B.3.
5368 MachineFunction &MF = DAG.getMachineFunction();
5369 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5370 auto PtrVT = getPointerTy(DAG.getDataLayout());
5371 SDLoc DL(Op);
5372
5373 SDValue Chain = Op.getOperand(0);
5374 SDValue VAList = Op.getOperand(1);
5375 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5376 SmallVector<SDValue, 4> MemOps;
5377
5378 // void *__stack at offset 0
5379 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
5380 MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
5381 MachinePointerInfo(SV), /* Alignment = */ 8));
5382
5383 // void *__gr_top at offset 8
5384 int GPRSize = FuncInfo->getVarArgsGPRSize();
5385 if (GPRSize > 0) {
5386 SDValue GRTop, GRTopAddr;
5387
5388 GRTopAddr =
5389 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT));
5390
5391 GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
5392 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
5393 DAG.getConstant(GPRSize, DL, PtrVT));
5394
5395 MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
5396 MachinePointerInfo(SV, 8),
5397 /* Alignment = */ 8));
5398 }
5399
5400 // void *__vr_top at offset 16
5401 int FPRSize = FuncInfo->getVarArgsFPRSize();
5402 if (FPRSize > 0) {
5403 SDValue VRTop, VRTopAddr;
5404 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5405 DAG.getConstant(16, DL, PtrVT));
5406
5407 VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
5408 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
5409 DAG.getConstant(FPRSize, DL, PtrVT));
5410
5411 MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
5412 MachinePointerInfo(SV, 16),
5413 /* Alignment = */ 8));
5414 }
5415
5416 // int __gr_offs at offset 24
5417 SDValue GROffsAddr =
5418 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT));
5419 MemOps.push_back(DAG.getStore(
5420 Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32), GROffsAddr,
5421 MachinePointerInfo(SV, 24), /* Alignment = */ 4));
5422
5423 // int __vr_offs at offset 28
5424 SDValue VROffsAddr =
5425 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT));
5426 MemOps.push_back(DAG.getStore(
5427 Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32), VROffsAddr,
5428 MachinePointerInfo(SV, 28), /* Alignment = */ 4));
5429
5430 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
5431}
5432
5433SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
5434 SelectionDAG &DAG) const {
5435 MachineFunction &MF = DAG.getMachineFunction();
5436
5437 if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
5438 return LowerWin64_VASTART(Op, DAG);
5439 else if (Subtarget->isTargetDarwin())
5440 return LowerDarwin_VASTART(Op, DAG);
5441 else
5442 return LowerAAPCS_VASTART(Op, DAG);
5443}
5444
5445SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
5446 SelectionDAG &DAG) const {
5447 // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
5448 // pointer.
5449 SDLoc DL(Op);
5450 unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
5451 unsigned VaListSize = (Subtarget->isTargetDarwin() ||
5452 Subtarget->isTargetWindows()) ? PtrSize : 32;
5453 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
5454 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
5455
5456 return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
5457 DAG.getConstant(VaListSize, DL, MVT::i32), PtrSize,
5458 false, false, false, MachinePointerInfo(DestSV),
5459 MachinePointerInfo(SrcSV));
5460}
5461
5462SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
5463 assert(Subtarget->isTargetDarwin() &&((Subtarget->isTargetDarwin() && "automatic va_arg instruction only works on Darwin"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"automatic va_arg instruction only works on Darwin\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5464, __PRETTY_FUNCTION__))
5464 "automatic va_arg instruction only works on Darwin")((Subtarget->isTargetDarwin() && "automatic va_arg instruction only works on Darwin"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"automatic va_arg instruction only works on Darwin\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5464, __PRETTY_FUNCTION__))
;
5465
5466 const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5467 EVT VT = Op.getValueType();
5468 SDLoc DL(Op);
5469 SDValue Chain = Op.getOperand(0);
5470 SDValue Addr = Op.getOperand(1);
5471 unsigned Align = Op.getConstantOperandVal(3);
5472 unsigned MinSlotSize = Subtarget->isTargetILP32() ? 4 : 8;
5473 auto PtrVT = getPointerTy(DAG.getDataLayout());
5474 auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
5475 SDValue VAList =
5476 DAG.getLoad(PtrMemVT, DL, Chain, Addr, MachinePointerInfo(V));
5477 Chain = VAList.getValue(1);
5478 VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
5479
5480 if (Align > MinSlotSize) {
5481 assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2")((((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("((Align & (Align - 1)) == 0) && \"Expected Align to be a power of 2\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5481, __PRETTY_FUNCTION__))
;
5482 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5483 DAG.getConstant(Align - 1, DL, PtrVT));
5484 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
5485 DAG.getConstant(-(int64_t)Align, DL, PtrVT));
5486 }
5487
5488 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
5489 unsigned ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
5490
5491 // Scalar integer and FP values smaller than 64 bits are implicitly extended
5492 // up to 64 bits. At the very least, we have to increase the striding of the
5493 // vaargs list to match this, and for FP values we need to introduce
5494 // FP_ROUND nodes as well.
5495 if (VT.isInteger() && !VT.isVector())
5496 ArgSize = std::max(ArgSize, MinSlotSize);
5497 bool NeedFPTrunc = false;
5498 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
5499 ArgSize = 8;
5500 NeedFPTrunc = true;
5501 }
5502
5503 // Increment the pointer, VAList, to the next vaarg
5504 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5505 DAG.getConstant(ArgSize, DL, PtrVT));
5506 VANext = DAG.getZExtOrTrunc(VANext, DL, PtrMemVT);
5507
5508 // Store the incremented VAList to the legalized pointer
5509 SDValue APStore =
5510 DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
5511
5512 // Load the actual argument out of the pointer VAList
5513 if (NeedFPTrunc) {
5514 // Load the value as an f64.
5515 SDValue WideFP =
5516 DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
5517 // Round the value down to an f32.
5518 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
5519 DAG.getIntPtrConstant(1, DL));
5520 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
5521 // Merge the rounded value with the chain output of the load.
5522 return DAG.getMergeValues(Ops, DL);
5523 }
5524
5525 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
5526}
5527
5528SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
5529 SelectionDAG &DAG) const {
5530 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5531 MFI.setFrameAddressIsTaken(true);
5532
5533 EVT VT = Op.getValueType();
5534 SDLoc DL(Op);
5535 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5536 SDValue FrameAddr =
5537 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
5538 while (Depth--)
5539 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
5540 MachinePointerInfo());
5541
5542 if (Subtarget->isTargetILP32())
5543 FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr,
5544 DAG.getValueType(VT));
5545
5546 return FrameAddr;
5547}
5548
5549SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
5550 SelectionDAG &DAG) const {
5551 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5552
5553 EVT VT = getPointerTy(DAG.getDataLayout());
5554 SDLoc DL(Op);
5555 int FI = MFI.CreateFixedObject(4, 0, false);
5556 return DAG.getFrameIndex(FI, VT);
5557}
5558
5559#define GET_REGISTER_MATCHER
5560#include "AArch64GenAsmMatcher.inc"
5561
5562// FIXME? Maybe this could be a TableGen attribute on some registers and
5563// this table could be generated automatically from RegInfo.
5564Register AArch64TargetLowering::
5565getRegisterByName(const char* RegName, EVT VT, const MachineFunction &MF) const {
5566 Register Reg = MatchRegisterName(RegName);
5567 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
5568 const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
5569 unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
5570 if (!Subtarget->isXRegisterReserved(DwarfRegNum))
5571 Reg = 0;
5572 }
5573 if (Reg)
5574 return Reg;
5575 report_fatal_error(Twine("Invalid register name \""
5576 + StringRef(RegName) + "\"."));
5577}
5578
5579SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
5580 SelectionDAG &DAG) const {
5581 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
5582
5583 EVT VT = Op.getValueType();
5584 SDLoc DL(Op);
5585
5586 SDValue FrameAddr =
5587 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
5588 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5589
5590 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
5591}
5592
5593SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
5594 SelectionDAG &DAG) const {
5595 MachineFunction &MF = DAG.getMachineFunction();
5596 MachineFrameInfo &MFI = MF.getFrameInfo();
5597 MFI.setReturnAddressIsTaken(true);
5598
5599 EVT VT = Op.getValueType();
5600 SDLoc DL(Op);
5601 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5602 if (Depth) {
5603 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5604 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5605 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
5606 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
5607 MachinePointerInfo());
5608 }
5609
5610 // Return LR, which contains the return address. Mark it an implicit live-in.
5611 unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
5612 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
5613}
5614
5615/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
5616/// i64 values and take a 2 x i64 value to shift plus a shift amount.
5617SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
5618 SelectionDAG &DAG) const {
5619 assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5619, __PRETTY_FUNCTION__))
;
5620 EVT VT = Op.getValueType();
5621 unsigned VTBits = VT.getSizeInBits();
5622 SDLoc dl(Op);
5623 SDValue ShOpLo = Op.getOperand(0);
5624 SDValue ShOpHi = Op.getOperand(1);
5625 SDValue ShAmt = Op.getOperand(2);
5626 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
5627
5628 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS)((Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::
SRL_PARTS) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5628, __PRETTY_FUNCTION__))
;
5629
5630 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5631 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5632 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
5633
5634 // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which
5635 // is "undef". We wanted 0, so CSEL it directly.
5636 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5637 ISD::SETEQ, dl, DAG);
5638 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5639 HiBitsForLo =
5640 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5641 HiBitsForLo, CCVal, Cmp);
5642
5643 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5644 DAG.getConstant(VTBits, dl, MVT::i64));
5645
5646 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
5647 SDValue LoForNormalShift =
5648 DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo);
5649
5650 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5651 dl, DAG);
5652 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5653 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
5654 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5655 LoForNormalShift, CCVal, Cmp);
5656
5657 // AArch64 shifts larger than the register width are wrapped rather than
5658 // clamped, so we can't just emit "hi >> x".
5659 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
5660 SDValue HiForBigShift =
5661 Opc == ISD::SRA
5662 ? DAG.getNode(Opc, dl, VT, ShOpHi,
5663 DAG.getConstant(VTBits - 1, dl, MVT::i64))
5664 : DAG.getConstant(0, dl, VT);
5665 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5666 HiForNormalShift, CCVal, Cmp);
5667
5668 SDValue Ops[2] = { Lo, Hi };
5669 return DAG.getMergeValues(Ops, dl);
5670}
5671
5672/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
5673/// i64 values and take a 2 x i64 value to shift plus a shift amount.
5674SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
5675 SelectionDAG &DAG) const {
5676 assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5676, __PRETTY_FUNCTION__))
;
5677 EVT VT = Op.getValueType();
5678 unsigned VTBits = VT.getSizeInBits();
5679 SDLoc dl(Op);
5680 SDValue ShOpLo = Op.getOperand(0);
5681 SDValue ShOpHi = Op.getOperand(1);
5682 SDValue ShAmt = Op.getOperand(2);
5683
5684 assert(Op.getOpcode() == ISD::SHL_PARTS)((Op.getOpcode() == ISD::SHL_PARTS) ? static_cast<void>
(0) : __assert_fail ("Op.getOpcode() == ISD::SHL_PARTS", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5684, __PRETTY_FUNCTION__))
;
5685 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5686 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5687 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
5688
5689 // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which
5690 // is "undef". We wanted 0, so CSEL it directly.
5691 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5692 ISD::SETEQ, dl, DAG);
5693 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5694 LoBitsForHi =
5695 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5696 LoBitsForHi, CCVal, Cmp);
5697
5698 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5699 DAG.getConstant(VTBits, dl, MVT::i64));
5700 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
5701 SDValue HiForNormalShift =
5702 DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi);
5703
5704 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
5705
5706 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5707 dl, DAG);
5708 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5709 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5710 HiForNormalShift, CCVal, Cmp);
5711
5712 // AArch64 shifts of larger than register sizes are wrapped rather than
5713 // clamped, so we can't just emit "lo << a" if a is too big.
5714 SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
5715 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5716 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5717 LoForNormalShift, CCVal, Cmp);
5718
5719 SDValue Ops[2] = { Lo, Hi };
5720 return DAG.getMergeValues(Ops, dl);
5721}
5722
5723bool AArch64TargetLowering::isOffsetFoldingLegal(
5724 const GlobalAddressSDNode *GA) const {
5725 // Offsets are folded in the DAG combine rather than here so that we can
5726 // intelligently choose an offset based on the uses.
5727 return false;
5728}
5729
5730bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
5731 bool OptForSize) const {
5732 bool IsLegal = false;
5733 // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and
5734 // 16-bit case when target has full fp16 support.
5735 // FIXME: We should be able to handle f128 as well with a clever lowering.
5736 const APInt ImmInt = Imm.bitcastToAPInt();
5737 if (VT == MVT::f64)
5738 IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero();
5739 else if (VT == MVT::f32)
5740 IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero();
5741 else if (VT == MVT::f16 && Subtarget->hasFullFP16())
5742 IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero();
5743 // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to
5744 // generate that fmov.
5745
5746 // If we can not materialize in immediate field for fmov, check if the
5747 // value can be encoded as the immediate operand of a logical instruction.
5748 // The immediate value will be created with either MOVZ, MOVN, or ORR.
5749 if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) {
5750 // The cost is actually exactly the same for mov+fmov vs. adrp+ldr;
5751 // however the mov+fmov sequence is always better because of the reduced
5752 // cache pressure. The timings are still the same if you consider
5753 // movw+movk+fmov vs. adrp+ldr (it's one instruction longer, but the
5754 // movw+movk is fused). So we limit up to 2 instrdduction at most.
5755 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
5756 AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(),
5757 Insn);
5758 unsigned Limit = (OptForSize ? 1 : (Subtarget->hasFuseLiterals() ? 5 : 2));
5759 IsLegal = Insn.size() <= Limit;
5760 }
5761
5762 LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << (IsLegal ? "Legal " : "Illegal "
) << VT.getEVTString() << " imm value: "; Imm.dump
();; } } while (false)
5763 << " imm value: "; Imm.dump();)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << (IsLegal ? "Legal " : "Illegal "
) << VT.getEVTString() << " imm value: "; Imm.dump
();; } } while (false)
;
5764 return IsLegal;
5765}
5766
5767//===----------------------------------------------------------------------===//
5768// AArch64 Optimization Hooks
5769//===----------------------------------------------------------------------===//
5770
5771static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
5772 SDValue Operand, SelectionDAG &DAG,
5773 int &ExtraSteps) {
5774 EVT VT = Operand.getValueType();
5775 if (ST->hasNEON() &&
5776 (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
5777 VT == MVT::f32 || VT == MVT::v1f32 ||
5778 VT == MVT::v2f32 || VT == MVT::v4f32)) {
5779 if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
5780 // For the reciprocal estimates, convergence is quadratic, so the number
5781 // of digits is doubled after each iteration. In ARMv8, the accuracy of
5782 // the initial estimate is 2^-8. Thus the number of extra steps to refine
5783 // the result for float (23 mantissa bits) is 2 and for double (52
5784 // mantissa bits) is 3.
5785 ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
5786
5787 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
5788 }
5789
5790 return SDValue();
5791}
5792
5793SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
5794 SelectionDAG &DAG, int Enabled,
5795 int &ExtraSteps,
5796 bool &UseOneConst,
5797 bool Reciprocal) const {
5798 if (Enabled == ReciprocalEstimate::Enabled ||
5799 (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt()))
5800 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
5801 DAG, ExtraSteps)) {
5802 SDLoc DL(Operand);
5803 EVT VT = Operand.getValueType();
5804
5805 SDNodeFlags Flags;
5806 Flags.setAllowReassociation(true);
5807
5808 // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
5809 // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
5810 for (int i = ExtraSteps; i > 0; --i) {
5811 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
5812 Flags);
5813 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
5814 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5815 }
5816 if (!Reciprocal) {
5817 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
5818 VT);
5819 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5820 SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
5821
5822 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
5823 // Correct the result if the operand is 0.0.
5824 Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL,
5825 VT, Eq, Operand, Estimate);
5826 }
5827
5828 ExtraSteps = 0;
5829 return Estimate;
5830 }
5831
5832 return SDValue();
5833}
5834
5835SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
5836 SelectionDAG &DAG, int Enabled,
5837 int &ExtraSteps) const {
5838 if (Enabled == ReciprocalEstimate::Enabled)
5839 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
5840 DAG, ExtraSteps)) {
5841 SDLoc DL(Operand);
5842 EVT VT = Operand.getValueType();
5843
5844 SDNodeFlags Flags;
5845 Flags.setAllowReassociation(true);
5846
5847 // Newton reciprocal iteration: E * (2 - X * E)
5848 // AArch64 reciprocal iteration instruction: (2 - M * N)
5849 for (int i = ExtraSteps; i > 0; --i) {
5850 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
5851 Estimate, Flags);
5852 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5853 }
5854
5855 ExtraSteps = 0;
5856 return Estimate;
5857 }
5858
5859 return SDValue();
5860}
5861
5862//===----------------------------------------------------------------------===//
5863// AArch64 Inline Assembly Support
5864//===----------------------------------------------------------------------===//
5865
5866// Table of Constraints
5867// TODO: This is the current set of constraints supported by ARM for the
5868// compiler, not all of them may make sense.
5869//
5870// r - A general register
5871// w - An FP/SIMD register of some size in the range v0-v31
5872// x - An FP/SIMD register of some size in the range v0-v15
5873// I - Constant that can be used with an ADD instruction
5874// J - Constant that can be used with a SUB instruction
5875// K - Constant that can be used with a 32-bit logical instruction
5876// L - Constant that can be used with a 64-bit logical instruction
5877// M - Constant that can be used as a 32-bit MOV immediate
5878// N - Constant that can be used as a 64-bit MOV immediate
5879// Q - A memory reference with base register and no offset
5880// S - A symbolic address
5881// Y - Floating point constant zero
5882// Z - Integer constant zero
5883//
5884// Note that general register operands will be output using their 64-bit x
5885// register name, whatever the size of the variable, unless the asm operand
5886// is prefixed by the %w modifier. Floating-point and SIMD register operands
5887// will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
5888// %q modifier.
5889const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5890 // At this point, we have to lower this constraint to something else, so we
5891 // lower it to an "r" or "w". However, by doing this we will force the result
5892 // to be in register, while the X constraint is much more permissive.
5893 //
5894 // Although we are correct (we are free to emit anything, without
5895 // constraints), we might break use cases that would expect us to be more
5896 // efficient and emit something else.
5897 if (!Subtarget->hasFPARMv8())
5898 return "r";