Bug Summary

File:include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1159, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64ISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn374877/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn374877=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-15-233810-7101-1 -x c++ /build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp

/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp

1//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64ExpandImm.h"
14#include "AArch64ISelLowering.h"
15#include "AArch64CallingConvention.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64PerfectShuffle.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ArrayRef.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SmallSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/StringSwitch.h"
31#include "llvm/ADT/Triple.h"
32#include "llvm/ADT/Twine.h"
33#include "llvm/Analysis/VectorUtils.h"
34#include "llvm/CodeGen/CallingConvLower.h"
35#include "llvm/CodeGen/MachineBasicBlock.h"
36#include "llvm/CodeGen/MachineFrameInfo.h"
37#include "llvm/CodeGen/MachineFunction.h"
38#include "llvm/CodeGen/MachineInstr.h"
39#include "llvm/CodeGen/MachineInstrBuilder.h"
40#include "llvm/CodeGen/MachineMemOperand.h"
41#include "llvm/CodeGen/MachineRegisterInfo.h"
42#include "llvm/CodeGen/RuntimeLibcalls.h"
43#include "llvm/CodeGen/SelectionDAG.h"
44#include "llvm/CodeGen/SelectionDAGNodes.h"
45#include "llvm/CodeGen/TargetCallingConv.h"
46#include "llvm/CodeGen/TargetInstrInfo.h"
47#include "llvm/CodeGen/ValueTypes.h"
48#include "llvm/IR/Attributes.h"
49#include "llvm/IR/Constants.h"
50#include "llvm/IR/DataLayout.h"
51#include "llvm/IR/DebugLoc.h"
52#include "llvm/IR/DerivedTypes.h"
53#include "llvm/IR/Function.h"
54#include "llvm/IR/GetElementPtrTypeIterator.h"
55#include "llvm/IR/GlobalValue.h"
56#include "llvm/IR/IRBuilder.h"
57#include "llvm/IR/Instruction.h"
58#include "llvm/IR/Instructions.h"
59#include "llvm/IR/IntrinsicInst.h"
60#include "llvm/IR/Intrinsics.h"
61#include "llvm/IR/Module.h"
62#include "llvm/IR/OperandTraits.h"
63#include "llvm/IR/PatternMatch.h"
64#include "llvm/IR/Type.h"
65#include "llvm/IR/Use.h"
66#include "llvm/IR/Value.h"
67#include "llvm/MC/MCRegisterInfo.h"
68#include "llvm/Support/Casting.h"
69#include "llvm/Support/CodeGen.h"
70#include "llvm/Support/CommandLine.h"
71#include "llvm/Support/Compiler.h"
72#include "llvm/Support/Debug.h"
73#include "llvm/Support/ErrorHandling.h"
74#include "llvm/Support/KnownBits.h"
75#include "llvm/Support/MachineValueType.h"
76#include "llvm/Support/MathExtras.h"
77#include "llvm/Support/raw_ostream.h"
78#include "llvm/Target/TargetMachine.h"
79#include "llvm/Target/TargetOptions.h"
80#include <algorithm>
81#include <bitset>
82#include <cassert>
83#include <cctype>
84#include <cstdint>
85#include <cstdlib>
86#include <iterator>
87#include <limits>
88#include <tuple>
89#include <utility>
90#include <vector>
91
92using namespace llvm;
93using namespace llvm::PatternMatch;
94
95#define DEBUG_TYPE"aarch64-lower" "aarch64-lower"
96
97STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"aarch64-lower", "NumTailCalls"
, "Number of tail calls"}
;
98STATISTIC(NumShiftInserts, "Number of vector shift inserts")static llvm::Statistic NumShiftInserts = {"aarch64-lower", "NumShiftInserts"
, "Number of vector shift inserts"}
;
99STATISTIC(NumOptimizedImms, "Number of times immediates were optimized")static llvm::Statistic NumOptimizedImms = {"aarch64-lower", "NumOptimizedImms"
, "Number of times immediates were optimized"}
;
100
101static cl::opt<bool>
102EnableAArch64SlrGeneration("aarch64-shift-insert-generation", cl::Hidden,
103 cl::desc("Allow AArch64 SLI/SRI formation"),
104 cl::init(false));
105
106// FIXME: The necessary dtprel relocations don't seem to be supported
107// well in the GNU bfd and gold linkers at the moment. Therefore, by
108// default, for now, fall back to GeneralDynamic code generation.
109cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
110 "aarch64-elf-ldtls-generation", cl::Hidden,
111 cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
112 cl::init(false));
113
114static cl::opt<bool>
115EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
116 cl::desc("Enable AArch64 logical imm instruction "
117 "optimization"),
118 cl::init(true));
119
120/// Value type used for condition codes.
121static const MVT MVT_CC = MVT::i32;
122
123AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
124 const AArch64Subtarget &STI)
125 : TargetLowering(TM), Subtarget(&STI) {
126 // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
127 // we have to make something up. Arbitrarily, choose ZeroOrOne.
128 setBooleanContents(ZeroOrOneBooleanContent);
129 // When comparing vectors the result sets the different elements in the
130 // vector to all-one or all-zero.
131 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
132
133 // Set up the register classes.
134 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
135 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
136
137 if (Subtarget->hasFPARMv8()) {
138 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
139 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
140 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
141 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
142 }
143
144 if (Subtarget->hasNEON()) {
145 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
146 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
147 // Someone set us up the NEON.
148 addDRTypeForNEON(MVT::v2f32);
149 addDRTypeForNEON(MVT::v8i8);
150 addDRTypeForNEON(MVT::v4i16);
151 addDRTypeForNEON(MVT::v2i32);
152 addDRTypeForNEON(MVT::v1i64);
153 addDRTypeForNEON(MVT::v1f64);
154 addDRTypeForNEON(MVT::v4f16);
155
156 addQRTypeForNEON(MVT::v4f32);
157 addQRTypeForNEON(MVT::v2f64);
158 addQRTypeForNEON(MVT::v16i8);
159 addQRTypeForNEON(MVT::v8i16);
160 addQRTypeForNEON(MVT::v4i32);
161 addQRTypeForNEON(MVT::v2i64);
162 addQRTypeForNEON(MVT::v8f16);
163 }
164
165 if (Subtarget->hasSVE()) {
166 // Add legal sve predicate types
167 addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
168 addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
169 addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
170 addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
171
172 // Add legal sve data types
173 addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
174 addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
175 addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
176 addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
177
178 addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
179 addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
180 addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
181 addRegisterClass(MVT::nxv1f32, &AArch64::ZPRRegClass);
182 addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
183 addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
184 addRegisterClass(MVT::nxv1f64, &AArch64::ZPRRegClass);
185 addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
186 }
187
188 // Compute derived properties from the register classes
189 computeRegisterProperties(Subtarget->getRegisterInfo());
190
191 // Provide all sorts of operation actions
192 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
193 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
194 setOperationAction(ISD::SETCC, MVT::i32, Custom);
195 setOperationAction(ISD::SETCC, MVT::i64, Custom);
196 setOperationAction(ISD::SETCC, MVT::f16, Custom);
197 setOperationAction(ISD::SETCC, MVT::f32, Custom);
198 setOperationAction(ISD::SETCC, MVT::f64, Custom);
199 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
200 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
201 setOperationAction(ISD::BRCOND, MVT::Other, Expand);
202 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
203 setOperationAction(ISD::BR_CC, MVT::i64, Custom);
204 setOperationAction(ISD::BR_CC, MVT::f16, Custom);
205 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
206 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
207 setOperationAction(ISD::SELECT, MVT::i32, Custom);
208 setOperationAction(ISD::SELECT, MVT::i64, Custom);
209 setOperationAction(ISD::SELECT, MVT::f16, Custom);
210 setOperationAction(ISD::SELECT, MVT::f32, Custom);
211 setOperationAction(ISD::SELECT, MVT::f64, Custom);
212 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
213 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
214 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
215 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
216 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
217 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
218 setOperationAction(ISD::JumpTable, MVT::i64, Custom);
219
220 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
221 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
222 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
223
224 setOperationAction(ISD::FREM, MVT::f32, Expand);
225 setOperationAction(ISD::FREM, MVT::f64, Expand);
226 setOperationAction(ISD::FREM, MVT::f80, Expand);
227
228 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
229
230 // Custom lowering hooks are needed for XOR
231 // to fold it into CSINC/CSINV.
232 setOperationAction(ISD::XOR, MVT::i32, Custom);
233 setOperationAction(ISD::XOR, MVT::i64, Custom);
234
235 // Virtually no operation on f128 is legal, but LLVM can't expand them when
236 // there's a valid register class, so we need custom operations in most cases.
237 setOperationAction(ISD::FABS, MVT::f128, Expand);
238 setOperationAction(ISD::FADD, MVT::f128, Custom);
239 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
240 setOperationAction(ISD::FCOS, MVT::f128, Expand);
241 setOperationAction(ISD::FDIV, MVT::f128, Custom);
242 setOperationAction(ISD::FMA, MVT::f128, Expand);
243 setOperationAction(ISD::FMUL, MVT::f128, Custom);
244 setOperationAction(ISD::FNEG, MVT::f128, Expand);
245 setOperationAction(ISD::FPOW, MVT::f128, Expand);
246 setOperationAction(ISD::FREM, MVT::f128, Expand);
247 setOperationAction(ISD::FRINT, MVT::f128, Expand);
248 setOperationAction(ISD::FSIN, MVT::f128, Expand);
249 setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
250 setOperationAction(ISD::FSQRT, MVT::f128, Expand);
251 setOperationAction(ISD::FSUB, MVT::f128, Custom);
252 setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
253 setOperationAction(ISD::SETCC, MVT::f128, Custom);
254 setOperationAction(ISD::BR_CC, MVT::f128, Custom);
255 setOperationAction(ISD::SELECT, MVT::f128, Custom);
256 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
257 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
258
259 // Lowering for many of the conversions is actually specified by the non-f128
260 // type. The LowerXXX function will be trivial when f128 isn't involved.
261 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
262 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
263 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
264 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
265 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
266 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
267 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
268 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
269 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
270 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
271 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
272 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
273 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
274 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
275
276 // Variable arguments.
277 setOperationAction(ISD::VASTART, MVT::Other, Custom);
278 setOperationAction(ISD::VAARG, MVT::Other, Custom);
279 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
280 setOperationAction(ISD::VAEND, MVT::Other, Expand);
281
282 // Variable-sized objects.
283 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
284 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
285
286 if (Subtarget->isTargetWindows())
287 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
288 else
289 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
290
291 // Constant pool entries
292 setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
293
294 // BlockAddress
295 setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
296
297 // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
298 setOperationAction(ISD::ADDC, MVT::i32, Custom);
299 setOperationAction(ISD::ADDE, MVT::i32, Custom);
300 setOperationAction(ISD::SUBC, MVT::i32, Custom);
301 setOperationAction(ISD::SUBE, MVT::i32, Custom);
302 setOperationAction(ISD::ADDC, MVT::i64, Custom);
303 setOperationAction(ISD::ADDE, MVT::i64, Custom);
304 setOperationAction(ISD::SUBC, MVT::i64, Custom);
305 setOperationAction(ISD::SUBE, MVT::i64, Custom);
306
307 // AArch64 lacks both left-rotate and popcount instructions.
308 setOperationAction(ISD::ROTL, MVT::i32, Expand);
309 setOperationAction(ISD::ROTL, MVT::i64, Expand);
310 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
311 setOperationAction(ISD::ROTL, VT, Expand);
312 setOperationAction(ISD::ROTR, VT, Expand);
313 }
314
315 // AArch64 doesn't have {U|S}MUL_LOHI.
316 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
317 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
318
319 setOperationAction(ISD::CTPOP, MVT::i32, Custom);
320 setOperationAction(ISD::CTPOP, MVT::i64, Custom);
321
322 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
323 setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
324 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
325 setOperationAction(ISD::SDIVREM, VT, Expand);
326 setOperationAction(ISD::UDIVREM, VT, Expand);
327 }
328 setOperationAction(ISD::SREM, MVT::i32, Expand);
329 setOperationAction(ISD::SREM, MVT::i64, Expand);
330 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
331 setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
332 setOperationAction(ISD::UREM, MVT::i32, Expand);
333 setOperationAction(ISD::UREM, MVT::i64, Expand);
334
335 // Custom lower Add/Sub/Mul with overflow.
336 setOperationAction(ISD::SADDO, MVT::i32, Custom);
337 setOperationAction(ISD::SADDO, MVT::i64, Custom);
338 setOperationAction(ISD::UADDO, MVT::i32, Custom);
339 setOperationAction(ISD::UADDO, MVT::i64, Custom);
340 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
341 setOperationAction(ISD::SSUBO, MVT::i64, Custom);
342 setOperationAction(ISD::USUBO, MVT::i32, Custom);
343 setOperationAction(ISD::USUBO, MVT::i64, Custom);
344 setOperationAction(ISD::SMULO, MVT::i32, Custom);
345 setOperationAction(ISD::SMULO, MVT::i64, Custom);
346 setOperationAction(ISD::UMULO, MVT::i32, Custom);
347 setOperationAction(ISD::UMULO, MVT::i64, Custom);
348
349 setOperationAction(ISD::FSIN, MVT::f32, Expand);
350 setOperationAction(ISD::FSIN, MVT::f64, Expand);
351 setOperationAction(ISD::FCOS, MVT::f32, Expand);
352 setOperationAction(ISD::FCOS, MVT::f64, Expand);
353 setOperationAction(ISD::FPOW, MVT::f32, Expand);
354 setOperationAction(ISD::FPOW, MVT::f64, Expand);
355 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
356 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
357 if (Subtarget->hasFullFP16())
358 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
359 else
360 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
361
362 setOperationAction(ISD::FREM, MVT::f16, Promote);
363 setOperationAction(ISD::FREM, MVT::v4f16, Expand);
364 setOperationAction(ISD::FREM, MVT::v8f16, Expand);
365 setOperationAction(ISD::FPOW, MVT::f16, Promote);
366 setOperationAction(ISD::FPOW, MVT::v4f16, Expand);
367 setOperationAction(ISD::FPOW, MVT::v8f16, Expand);
368 setOperationAction(ISD::FPOWI, MVT::f16, Promote);
369 setOperationAction(ISD::FPOWI, MVT::v4f16, Expand);
370 setOperationAction(ISD::FPOWI, MVT::v8f16, Expand);
371 setOperationAction(ISD::FCOS, MVT::f16, Promote);
372 setOperationAction(ISD::FCOS, MVT::v4f16, Expand);
373 setOperationAction(ISD::FCOS, MVT::v8f16, Expand);
374 setOperationAction(ISD::FSIN, MVT::f16, Promote);
375 setOperationAction(ISD::FSIN, MVT::v4f16, Expand);
376 setOperationAction(ISD::FSIN, MVT::v8f16, Expand);
377 setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
378 setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
379 setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
380 setOperationAction(ISD::FEXP, MVT::f16, Promote);
381 setOperationAction(ISD::FEXP, MVT::v4f16, Expand);
382 setOperationAction(ISD::FEXP, MVT::v8f16, Expand);
383 setOperationAction(ISD::FEXP2, MVT::f16, Promote);
384 setOperationAction(ISD::FEXP2, MVT::v4f16, Expand);
385 setOperationAction(ISD::FEXP2, MVT::v8f16, Expand);
386 setOperationAction(ISD::FLOG, MVT::f16, Promote);
387 setOperationAction(ISD::FLOG, MVT::v4f16, Expand);
388 setOperationAction(ISD::FLOG, MVT::v8f16, Expand);
389 setOperationAction(ISD::FLOG2, MVT::f16, Promote);
390 setOperationAction(ISD::FLOG2, MVT::v4f16, Expand);
391 setOperationAction(ISD::FLOG2, MVT::v8f16, Expand);
392 setOperationAction(ISD::FLOG10, MVT::f16, Promote);
393 setOperationAction(ISD::FLOG10, MVT::v4f16, Expand);
394 setOperationAction(ISD::FLOG10, MVT::v8f16, Expand);
395
396 if (!Subtarget->hasFullFP16()) {
397 setOperationAction(ISD::SELECT, MVT::f16, Promote);
398 setOperationAction(ISD::SELECT_CC, MVT::f16, Promote);
399 setOperationAction(ISD::SETCC, MVT::f16, Promote);
400 setOperationAction(ISD::BR_CC, MVT::f16, Promote);
401 setOperationAction(ISD::FADD, MVT::f16, Promote);
402 setOperationAction(ISD::FSUB, MVT::f16, Promote);
403 setOperationAction(ISD::FMUL, MVT::f16, Promote);
404 setOperationAction(ISD::FDIV, MVT::f16, Promote);
405 setOperationAction(ISD::FMA, MVT::f16, Promote);
406 setOperationAction(ISD::FNEG, MVT::f16, Promote);
407 setOperationAction(ISD::FABS, MVT::f16, Promote);
408 setOperationAction(ISD::FCEIL, MVT::f16, Promote);
409 setOperationAction(ISD::FSQRT, MVT::f16, Promote);
410 setOperationAction(ISD::FFLOOR, MVT::f16, Promote);
411 setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
412 setOperationAction(ISD::FRINT, MVT::f16, Promote);
413 setOperationAction(ISD::FROUND, MVT::f16, Promote);
414 setOperationAction(ISD::FTRUNC, MVT::f16, Promote);
415 setOperationAction(ISD::FMINNUM, MVT::f16, Promote);
416 setOperationAction(ISD::FMAXNUM, MVT::f16, Promote);
417 setOperationAction(ISD::FMINIMUM, MVT::f16, Promote);
418 setOperationAction(ISD::FMAXIMUM, MVT::f16, Promote);
419
420 // promote v4f16 to v4f32 when that is known to be safe.
421 setOperationAction(ISD::FADD, MVT::v4f16, Promote);
422 setOperationAction(ISD::FSUB, MVT::v4f16, Promote);
423 setOperationAction(ISD::FMUL, MVT::v4f16, Promote);
424 setOperationAction(ISD::FDIV, MVT::v4f16, Promote);
425 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote);
426 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Promote);
427 AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32);
428 AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32);
429 AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32);
430 AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32);
431 AddPromotedToType(ISD::FP_EXTEND, MVT::v4f16, MVT::v4f32);
432 AddPromotedToType(ISD::FP_ROUND, MVT::v4f16, MVT::v4f32);
433
434 setOperationAction(ISD::FABS, MVT::v4f16, Expand);
435 setOperationAction(ISD::FNEG, MVT::v4f16, Expand);
436 setOperationAction(ISD::FROUND, MVT::v4f16, Expand);
437 setOperationAction(ISD::FMA, MVT::v4f16, Expand);
438 setOperationAction(ISD::SETCC, MVT::v4f16, Expand);
439 setOperationAction(ISD::BR_CC, MVT::v4f16, Expand);
440 setOperationAction(ISD::SELECT, MVT::v4f16, Expand);
441 setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand);
442 setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand);
443 setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand);
444 setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand);
445 setOperationAction(ISD::FCEIL, MVT::v4f16, Expand);
446 setOperationAction(ISD::FRINT, MVT::v4f16, Expand);
447 setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand);
448 setOperationAction(ISD::FSQRT, MVT::v4f16, Expand);
449
450 setOperationAction(ISD::FABS, MVT::v8f16, Expand);
451 setOperationAction(ISD::FADD, MVT::v8f16, Expand);
452 setOperationAction(ISD::FCEIL, MVT::v8f16, Expand);
453 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand);
454 setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
455 setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand);
456 setOperationAction(ISD::FMA, MVT::v8f16, Expand);
457 setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
458 setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand);
459 setOperationAction(ISD::FNEG, MVT::v8f16, Expand);
460 setOperationAction(ISD::FROUND, MVT::v8f16, Expand);
461 setOperationAction(ISD::FRINT, MVT::v8f16, Expand);
462 setOperationAction(ISD::FSQRT, MVT::v8f16, Expand);
463 setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
464 setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand);
465 setOperationAction(ISD::SETCC, MVT::v8f16, Expand);
466 setOperationAction(ISD::BR_CC, MVT::v8f16, Expand);
467 setOperationAction(ISD::SELECT, MVT::v8f16, Expand);
468 setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand);
469 setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand);
470 }
471
472 // AArch64 has implementations of a lot of rounding-like FP operations.
473 for (MVT Ty : {MVT::f32, MVT::f64}) {
474 setOperationAction(ISD::FFLOOR, Ty, Legal);
475 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
476 setOperationAction(ISD::FCEIL, Ty, Legal);
477 setOperationAction(ISD::FRINT, Ty, Legal);
478 setOperationAction(ISD::FTRUNC, Ty, Legal);
479 setOperationAction(ISD::FROUND, Ty, Legal);
480 setOperationAction(ISD::FMINNUM, Ty, Legal);
481 setOperationAction(ISD::FMAXNUM, Ty, Legal);
482 setOperationAction(ISD::FMINIMUM, Ty, Legal);
483 setOperationAction(ISD::FMAXIMUM, Ty, Legal);
484 setOperationAction(ISD::LROUND, Ty, Legal);
485 setOperationAction(ISD::LLROUND, Ty, Legal);
486 setOperationAction(ISD::LRINT, Ty, Legal);
487 setOperationAction(ISD::LLRINT, Ty, Legal);
488 }
489
490 if (Subtarget->hasFullFP16()) {
491 setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
492 setOperationAction(ISD::FFLOOR, MVT::f16, Legal);
493 setOperationAction(ISD::FCEIL, MVT::f16, Legal);
494 setOperationAction(ISD::FRINT, MVT::f16, Legal);
495 setOperationAction(ISD::FTRUNC, MVT::f16, Legal);
496 setOperationAction(ISD::FROUND, MVT::f16, Legal);
497 setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
498 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
499 setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
500 setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
501 }
502
503 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
504
505 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
506
507 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
508 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
509 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
510 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
511 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
512
513 // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
514 // This requires the Performance Monitors extension.
515 if (Subtarget->hasPerfMon())
516 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
517
518 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
519 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
520 // Issue __sincos_stret if available.
521 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
522 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
523 } else {
524 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
525 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
526 }
527
528 // Make floating-point constants legal for the large code model, so they don't
529 // become loads from the constant pool.
530 if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
531 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
532 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
533 }
534
535 // AArch64 does not have floating-point extending loads, i1 sign-extending
536 // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
537 for (MVT VT : MVT::fp_valuetypes()) {
538 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
539 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
540 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
541 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
542 }
543 for (MVT VT : MVT::integer_valuetypes())
544 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
545
546 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
547 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
548 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
549 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
550 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
551 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
552 setTruncStoreAction(MVT::f128, MVT::f16, Expand);
553
554 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
555 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
556
557 // Indexed loads and stores are supported.
558 for (unsigned im = (unsigned)ISD::PRE_INC;
559 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
560 setIndexedLoadAction(im, MVT::i8, Legal);
561 setIndexedLoadAction(im, MVT::i16, Legal);
562 setIndexedLoadAction(im, MVT::i32, Legal);
563 setIndexedLoadAction(im, MVT::i64, Legal);
564 setIndexedLoadAction(im, MVT::f64, Legal);
565 setIndexedLoadAction(im, MVT::f32, Legal);
566 setIndexedLoadAction(im, MVT::f16, Legal);
567 setIndexedStoreAction(im, MVT::i8, Legal);
568 setIndexedStoreAction(im, MVT::i16, Legal);
569 setIndexedStoreAction(im, MVT::i32, Legal);
570 setIndexedStoreAction(im, MVT::i64, Legal);
571 setIndexedStoreAction(im, MVT::f64, Legal);
572 setIndexedStoreAction(im, MVT::f32, Legal);
573 setIndexedStoreAction(im, MVT::f16, Legal);
574 }
575
576 // Trap.
577 setOperationAction(ISD::TRAP, MVT::Other, Legal);
578 if (Subtarget->isTargetWindows())
579 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
580
581 // We combine OR nodes for bitfield operations.
582 setTargetDAGCombine(ISD::OR);
583 // Try to create BICs for vector ANDs.
584 setTargetDAGCombine(ISD::AND);
585
586 // Vector add and sub nodes may conceal a high-half opportunity.
587 // Also, try to fold ADD into CSINC/CSINV..
588 setTargetDAGCombine(ISD::ADD);
589 setTargetDAGCombine(ISD::SUB);
590 setTargetDAGCombine(ISD::SRL);
591 setTargetDAGCombine(ISD::XOR);
592 setTargetDAGCombine(ISD::SINT_TO_FP);
593 setTargetDAGCombine(ISD::UINT_TO_FP);
594
595 setTargetDAGCombine(ISD::FP_TO_SINT);
596 setTargetDAGCombine(ISD::FP_TO_UINT);
597 setTargetDAGCombine(ISD::FDIV);
598
599 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
600
601 setTargetDAGCombine(ISD::ANY_EXTEND);
602 setTargetDAGCombine(ISD::ZERO_EXTEND);
603 setTargetDAGCombine(ISD::SIGN_EXTEND);
604 setTargetDAGCombine(ISD::BITCAST);
605 setTargetDAGCombine(ISD::CONCAT_VECTORS);
606 setTargetDAGCombine(ISD::STORE);
607 if (Subtarget->supportsAddressTopByteIgnored())
608 setTargetDAGCombine(ISD::LOAD);
609
610 setTargetDAGCombine(ISD::MUL);
611
612 setTargetDAGCombine(ISD::SELECT);
613 setTargetDAGCombine(ISD::VSELECT);
614
615 setTargetDAGCombine(ISD::INTRINSIC_VOID);
616 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
617 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
618
619 setTargetDAGCombine(ISD::GlobalAddress);
620
621 // In case of strict alignment, avoid an excessive number of byte wide stores.
622 MaxStoresPerMemsetOptSize = 8;
623 MaxStoresPerMemset = Subtarget->requiresStrictAlign()
624 ? MaxStoresPerMemsetOptSize : 32;
625
626 MaxGluedStoresPerMemcpy = 4;
627 MaxStoresPerMemcpyOptSize = 4;
628 MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
629 ? MaxStoresPerMemcpyOptSize : 16;
630
631 MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
632
633 MaxLoadsPerMemcmpOptSize = 4;
634 MaxLoadsPerMemcmp = Subtarget->requiresStrictAlign()
635 ? MaxLoadsPerMemcmpOptSize : 8;
636
637 setStackPointerRegisterToSaveRestore(AArch64::SP);
638
639 setSchedulingPreference(Sched::Hybrid);
640
641 EnableExtLdPromotion = true;
642
643 // Set required alignment.
644 setMinFunctionAlignment(Align(4));
645 // Set preferred alignments.
646 setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
647 setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
648
649 // Only change the limit for entries in a jump table if specified by
650 // the sub target, but not at the command line.
651 unsigned MaxJT = STI.getMaximumJumpTableSize();
652 if (MaxJT && getMaximumJumpTableSize() == UINT_MAX(2147483647 *2U +1U))
653 setMaximumJumpTableSize(MaxJT);
654
655 setHasExtractBitsInsn(true);
656
657 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
658
659 if (Subtarget->hasNEON()) {
660 // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
661 // silliness like this:
662 setOperationAction(ISD::FABS, MVT::v1f64, Expand);
663 setOperationAction(ISD::FADD, MVT::v1f64, Expand);
664 setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
665 setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
666 setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
667 setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
668 setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
669 setOperationAction(ISD::FMA, MVT::v1f64, Expand);
670 setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
671 setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
672 setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
673 setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
674 setOperationAction(ISD::FREM, MVT::v1f64, Expand);
675 setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
676 setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
677 setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
678 setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
679 setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
680 setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
681 setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
682 setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
683 setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
684 setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
685 setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
686 setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
687
688 setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
689 setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
690 setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
691 setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
692 setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
693
694 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
695
696 // AArch64 doesn't have a direct vector ->f32 conversion instructions for
697 // elements smaller than i32, so promote the input to i32 first.
698 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
699 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
700 // i8 vector elements also need promotion to i32 for v8i8
701 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
702 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
703 // Similarly, there is no direct i32 -> f64 vector conversion instruction.
704 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
705 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
706 setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
707 setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
708 // Or, direct i32 -> f16 vector conversion. Set it so custom, so the
709 // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
710 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
711 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
712
713 if (Subtarget->hasFullFP16()) {
714 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
715 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
716 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
717 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
718 } else {
719 // when AArch64 doesn't have fullfp16 support, promote the input
720 // to i32 first.
721 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
722 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
723 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
724 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
725 }
726
727 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
728 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
729
730 // AArch64 doesn't have MUL.2d:
731 setOperationAction(ISD::MUL, MVT::v2i64, Expand);
732 // Custom handling for some quad-vector types to detect MULL.
733 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
734 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
735 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
736
737 // Vector reductions
738 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
739 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
740 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
741 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
742 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
743 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
744 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
745 }
746 for (MVT VT : { MVT::v4f16, MVT::v2f32,
747 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
748 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
749 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
750 }
751
752 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
753 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
754 // Likewise, narrowing and extending vector loads/stores aren't handled
755 // directly.
756 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
757 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
758
759 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
760 setOperationAction(ISD::MULHS, VT, Legal);
761 setOperationAction(ISD::MULHU, VT, Legal);
762 } else {
763 setOperationAction(ISD::MULHS, VT, Expand);
764 setOperationAction(ISD::MULHU, VT, Expand);
765 }
766 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
767 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
768
769 setOperationAction(ISD::BSWAP, VT, Expand);
770 setOperationAction(ISD::CTTZ, VT, Expand);
771
772 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
773 setTruncStoreAction(VT, InnerVT, Expand);
774 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
775 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
776 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
777 }
778 }
779
780 // AArch64 has implementations of a lot of rounding-like FP operations.
781 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
782 setOperationAction(ISD::FFLOOR, Ty, Legal);
783 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
784 setOperationAction(ISD::FCEIL, Ty, Legal);
785 setOperationAction(ISD::FRINT, Ty, Legal);
786 setOperationAction(ISD::FTRUNC, Ty, Legal);
787 setOperationAction(ISD::FROUND, Ty, Legal);
788 }
789
790 if (Subtarget->hasFullFP16()) {
791 for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
792 setOperationAction(ISD::FFLOOR, Ty, Legal);
793 setOperationAction(ISD::FNEARBYINT, Ty, Legal);
794 setOperationAction(ISD::FCEIL, Ty, Legal);
795 setOperationAction(ISD::FRINT, Ty, Legal);
796 setOperationAction(ISD::FTRUNC, Ty, Legal);
797 setOperationAction(ISD::FROUND, Ty, Legal);
798 }
799 }
800
801 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
802 }
803
804 PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
805}
806
807void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
808 assert(VT.isVector() && "VT should be a vector type")((VT.isVector() && "VT should be a vector type") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"VT should be a vector type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 808, __PRETTY_FUNCTION__))
;
809
810 if (VT.isFloatingPoint()) {
811 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
812 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
813 setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
814 }
815
816 // Mark vector float intrinsics as expand.
817 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
818 setOperationAction(ISD::FSIN, VT, Expand);
819 setOperationAction(ISD::FCOS, VT, Expand);
820 setOperationAction(ISD::FPOW, VT, Expand);
821 setOperationAction(ISD::FLOG, VT, Expand);
822 setOperationAction(ISD::FLOG2, VT, Expand);
823 setOperationAction(ISD::FLOG10, VT, Expand);
824 setOperationAction(ISD::FEXP, VT, Expand);
825 setOperationAction(ISD::FEXP2, VT, Expand);
826
827 // But we do support custom-lowering for FCOPYSIGN.
828 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
829 }
830
831 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
832 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
833 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
834 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
835 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
836 setOperationAction(ISD::SRA, VT, Custom);
837 setOperationAction(ISD::SRL, VT, Custom);
838 setOperationAction(ISD::SHL, VT, Custom);
839 setOperationAction(ISD::OR, VT, Custom);
840 setOperationAction(ISD::SETCC, VT, Custom);
841 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
842
843 setOperationAction(ISD::SELECT, VT, Expand);
844 setOperationAction(ISD::SELECT_CC, VT, Expand);
845 setOperationAction(ISD::VSELECT, VT, Expand);
846 for (MVT InnerVT : MVT::all_valuetypes())
847 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
848
849 // CNT supports only B element sizes, then use UADDLP to widen.
850 if (VT != MVT::v8i8 && VT != MVT::v16i8)
851 setOperationAction(ISD::CTPOP, VT, Custom);
852
853 setOperationAction(ISD::UDIV, VT, Expand);
854 setOperationAction(ISD::SDIV, VT, Expand);
855 setOperationAction(ISD::UREM, VT, Expand);
856 setOperationAction(ISD::SREM, VT, Expand);
857 setOperationAction(ISD::FREM, VT, Expand);
858
859 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
860 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
861
862 if (!VT.isFloatingPoint())
863 setOperationAction(ISD::ABS, VT, Legal);
864
865 // [SU][MIN|MAX] are available for all NEON types apart from i64.
866 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
867 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
868 setOperationAction(Opcode, VT, Legal);
869
870 // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
871 if (VT.isFloatingPoint() &&
872 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
873 for (unsigned Opcode :
874 {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM})
875 setOperationAction(Opcode, VT, Legal);
876
877 if (Subtarget->isLittleEndian()) {
878 for (unsigned im = (unsigned)ISD::PRE_INC;
879 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
880 setIndexedLoadAction(im, VT, Legal);
881 setIndexedStoreAction(im, VT, Legal);
882 }
883 }
884}
885
886void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
887 addRegisterClass(VT, &AArch64::FPR64RegClass);
888 addTypeForNEON(VT, MVT::v2i32);
889}
890
891void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
892 addRegisterClass(VT, &AArch64::FPR128RegClass);
893 addTypeForNEON(VT, MVT::v4i32);
894}
895
896EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &, LLVMContext &,
897 EVT VT) const {
898 if (!VT.isVector())
899 return MVT::i32;
900 return VT.changeVectorElementTypeToInteger();
901}
902
903static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
904 const APInt &Demanded,
905 TargetLowering::TargetLoweringOpt &TLO,
906 unsigned NewOpc) {
907 uint64_t OldImm = Imm, NewImm, Enc;
908 uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
909
910 // Return if the immediate is already all zeros, all ones, a bimm32 or a
911 // bimm64.
912 if (Imm == 0 || Imm == Mask ||
913 AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
914 return false;
915
916 unsigned EltSize = Size;
917 uint64_t DemandedBits = Demanded.getZExtValue();
918
919 // Clear bits that are not demanded.
920 Imm &= DemandedBits;
921
922 while (true) {
923 // The goal here is to set the non-demanded bits in a way that minimizes
924 // the number of switching between 0 and 1. In order to achieve this goal,
925 // we set the non-demanded bits to the value of the preceding demanded bits.
926 // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
927 // non-demanded bit), we copy bit0 (1) to the least significant 'x',
928 // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
929 // The final result is 0b11000011.
930 uint64_t NonDemandedBits = ~DemandedBits;
931 uint64_t InvertedImm = ~Imm & DemandedBits;
932 uint64_t RotatedImm =
933 ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
934 NonDemandedBits;
935 uint64_t Sum = RotatedImm + NonDemandedBits;
936 bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
937 uint64_t Ones = (Sum + Carry) & NonDemandedBits;
938 NewImm = (Imm | Ones) & Mask;
939
940 // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
941 // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
942 // we halve the element size and continue the search.
943 if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
944 break;
945
946 // We cannot shrink the element size any further if it is 2-bits.
947 if (EltSize == 2)
948 return false;
949
950 EltSize /= 2;
951 Mask >>= EltSize;
952 uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
953
954 // Return if there is mismatch in any of the demanded bits of Imm and Hi.
955 if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
956 return false;
957
958 // Merge the upper and lower halves of Imm and DemandedBits.
959 Imm |= Hi;
960 DemandedBits |= DemandedBitsHi;
961 }
962
963 ++NumOptimizedImms;
964
965 // Replicate the element across the register width.
966 while (EltSize < Size) {
967 NewImm |= NewImm << EltSize;
968 EltSize *= 2;
969 }
970
971 (void)OldImm;
972 assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&((((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
"demanded bits should never be altered") ? static_cast<void
> (0) : __assert_fail ("((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 && \"demanded bits should never be altered\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 973, __PRETTY_FUNCTION__))
973 "demanded bits should never be altered")((((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
"demanded bits should never be altered") ? static_cast<void
> (0) : __assert_fail ("((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 && \"demanded bits should never be altered\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 973, __PRETTY_FUNCTION__))
;
974 assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm")((OldImm != NewImm && "the new imm shouldn't be equal to the old imm"
) ? static_cast<void> (0) : __assert_fail ("OldImm != NewImm && \"the new imm shouldn't be equal to the old imm\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 974, __PRETTY_FUNCTION__))
;
975
976 // Create the new constant immediate node.
977 EVT VT = Op.getValueType();
978 SDLoc DL(Op);
979 SDValue New;
980
981 // If the new constant immediate is all-zeros or all-ones, let the target
982 // independent DAG combine optimize this node.
983 if (NewImm == 0 || NewImm == OrigMask) {
984 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
985 TLO.DAG.getConstant(NewImm, DL, VT));
986 // Otherwise, create a machine node so that target independent DAG combine
987 // doesn't undo this optimization.
988 } else {
989 Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
990 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
991 New = SDValue(
992 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
993 }
994
995 return TLO.CombineTo(Op, New);
996}
997
998bool AArch64TargetLowering::targetShrinkDemandedConstant(
999 SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const {
1000 // Delay this optimization to as late as possible.
1001 if (!TLO.LegalOps)
1002 return false;
1003
1004 if (!EnableOptimizeLogicalImm)
1005 return false;
1006
1007 EVT VT = Op.getValueType();
1008 if (VT.isVector())
1009 return false;
1010
1011 unsigned Size = VT.getSizeInBits();
1012 assert((Size == 32 || Size == 64) &&(((Size == 32 || Size == 64) && "i32 or i64 is expected after legalization."
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"i32 or i64 is expected after legalization.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1013, __PRETTY_FUNCTION__))
1013 "i32 or i64 is expected after legalization.")(((Size == 32 || Size == 64) && "i32 or i64 is expected after legalization."
) ? static_cast<void> (0) : __assert_fail ("(Size == 32 || Size == 64) && \"i32 or i64 is expected after legalization.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1013, __PRETTY_FUNCTION__))
;
1014
1015 // Exit early if we demand all bits.
1016 if (Demanded.countPopulation() == Size)
1017 return false;
1018
1019 unsigned NewOpc;
1020 switch (Op.getOpcode()) {
1021 default:
1022 return false;
1023 case ISD::AND:
1024 NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1025 break;
1026 case ISD::OR:
1027 NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1028 break;
1029 case ISD::XOR:
1030 NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1031 break;
1032 }
1033 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1034 if (!C)
1035 return false;
1036 uint64_t Imm = C->getZExtValue();
1037 return optimizeLogicalImm(Op, Size, Imm, Demanded, TLO, NewOpc);
1038}
1039
1040/// computeKnownBitsForTargetNode - Determine which of the bits specified in
1041/// Mask are known to be either zero or one and return them Known.
1042void AArch64TargetLowering::computeKnownBitsForTargetNode(
1043 const SDValue Op, KnownBits &Known,
1044 const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
1045 switch (Op.getOpcode()) {
1046 default:
1047 break;
1048 case AArch64ISD::CSEL: {
1049 KnownBits Known2;
1050 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
1051 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
1052 Known.Zero &= Known2.Zero;
1053 Known.One &= Known2.One;
1054 break;
1055 }
1056 case AArch64ISD::LOADgot:
1057 case AArch64ISD::ADDlow: {
1058 if (!Subtarget->isTargetILP32())
1059 break;
1060 // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
1061 Known.Zero = APInt::getHighBitsSet(64, 32);
1062 break;
1063 }
1064 case ISD::INTRINSIC_W_CHAIN: {
1065 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1066 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1067 switch (IntID) {
1068 default: return;
1069 case Intrinsic::aarch64_ldaxr:
1070 case Intrinsic::aarch64_ldxr: {
1071 unsigned BitWidth = Known.getBitWidth();
1072 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1073 unsigned MemBits = VT.getScalarSizeInBits();
1074 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1075 return;
1076 }
1077 }
1078 break;
1079 }
1080 case ISD::INTRINSIC_WO_CHAIN:
1081 case ISD::INTRINSIC_VOID: {
1082 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1083 switch (IntNo) {
1084 default:
1085 break;
1086 case Intrinsic::aarch64_neon_umaxv:
1087 case Intrinsic::aarch64_neon_uminv: {
1088 // Figure out the datatype of the vector operand. The UMINV instruction
1089 // will zero extend the result, so we can mark as known zero all the
1090 // bits larger than the element datatype. 32-bit or larget doesn't need
1091 // this as those are legal types and will be handled by isel directly.
1092 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1093 unsigned BitWidth = Known.getBitWidth();
1094 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1095 assert(BitWidth >= 8 && "Unexpected width!")((BitWidth >= 8 && "Unexpected width!") ? static_cast
<void> (0) : __assert_fail ("BitWidth >= 8 && \"Unexpected width!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1095, __PRETTY_FUNCTION__))
;
1096 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1097 Known.Zero |= Mask;
1098 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1099 assert(BitWidth >= 16 && "Unexpected width!")((BitWidth >= 16 && "Unexpected width!") ? static_cast
<void> (0) : __assert_fail ("BitWidth >= 16 && \"Unexpected width!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1099, __PRETTY_FUNCTION__))
;
1100 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1101 Known.Zero |= Mask;
1102 }
1103 break;
1104 } break;
1105 }
1106 }
1107 }
1108}
1109
1110MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1111 EVT) const {
1112 return MVT::i64;
1113}
1114
1115bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1116 EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1117 bool *Fast) const {
1118 if (Subtarget->requiresStrictAlign())
1119 return false;
1120
1121 if (Fast) {
1122 // Some CPUs are fine with unaligned stores except for 128-bit ones.
1123 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1124 // See comments in performSTORECombine() for more details about
1125 // these conditions.
1126
1127 // Code that uses clang vector extensions can mark that it
1128 // wants unaligned accesses to be treated as fast by
1129 // underspecifying alignment to be 1 or 2.
1130 Align <= 2 ||
1131
1132 // Disregard v2i64. Memcpy lowering produces those and splitting
1133 // them regresses performance on micro-benchmarks and olden/bh.
1134 VT == MVT::v2i64;
1135 }
1136 return true;
1137}
1138
1139// Same as above but handling LLTs instead.
1140bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1141 LLT Ty, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
1142 bool *Fast) const {
1143 if (Subtarget->requiresStrictAlign())
1144 return false;
1145
1146 if (Fast) {
1147 // Some CPUs are fine with unaligned stores except for 128-bit ones.
1148 *Fast = !Subtarget->isMisaligned128StoreSlow() ||
1149 Ty.getSizeInBytes() != 16 ||
1150 // See comments in performSTORECombine() for more details about
1151 // these conditions.
1152
1153 // Code that uses clang vector extensions can mark that it
1154 // wants unaligned accesses to be treated as fast by
1155 // underspecifying alignment to be 1 or 2.
1156 Align <= 2 ||
1157
1158 // Disregard v2i64. Memcpy lowering produces those and splitting
1159 // them regresses performance on micro-benchmarks and olden/bh.
1160 Ty == LLT::vector(2, 64);
1161 }
1162 return true;
1163}
1164
1165FastISel *
1166AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1167 const TargetLibraryInfo *libInfo) const {
1168 return AArch64::createFastISel(funcInfo, libInfo);
1169}
1170
1171const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1172 switch ((AArch64ISD::NodeType)Opcode) {
1173 case AArch64ISD::FIRST_NUMBER: break;
1174 case AArch64ISD::CALL: return "AArch64ISD::CALL";
1175 case AArch64ISD::ADRP: return "AArch64ISD::ADRP";
1176 case AArch64ISD::ADR: return "AArch64ISD::ADR";
1177 case AArch64ISD::ADDlow: return "AArch64ISD::ADDlow";
1178 case AArch64ISD::LOADgot: return "AArch64ISD::LOADgot";
1179 case AArch64ISD::RET_FLAG: return "AArch64ISD::RET_FLAG";
1180 case AArch64ISD::BRCOND: return "AArch64ISD::BRCOND";
1181 case AArch64ISD::CSEL: return "AArch64ISD::CSEL";
1182 case AArch64ISD::FCSEL: return "AArch64ISD::FCSEL";
1183 case AArch64ISD::CSINV: return "AArch64ISD::CSINV";
1184 case AArch64ISD::CSNEG: return "AArch64ISD::CSNEG";
1185 case AArch64ISD::CSINC: return "AArch64ISD::CSINC";
1186 case AArch64ISD::THREAD_POINTER: return "AArch64ISD::THREAD_POINTER";
1187 case AArch64ISD::TLSDESC_CALLSEQ: return "AArch64ISD::TLSDESC_CALLSEQ";
1188 case AArch64ISD::ADC: return "AArch64ISD::ADC";
1189 case AArch64ISD::SBC: return "AArch64ISD::SBC";
1190 case AArch64ISD::ADDS: return "AArch64ISD::ADDS";
1191 case AArch64ISD::SUBS: return "AArch64ISD::SUBS";
1192 case AArch64ISD::ADCS: return "AArch64ISD::ADCS";
1193 case AArch64ISD::SBCS: return "AArch64ISD::SBCS";
1194 case AArch64ISD::ANDS: return "AArch64ISD::ANDS";
1195 case AArch64ISD::CCMP: return "AArch64ISD::CCMP";
1196 case AArch64ISD::CCMN: return "AArch64ISD::CCMN";
1197 case AArch64ISD::FCCMP: return "AArch64ISD::FCCMP";
1198 case AArch64ISD::FCMP: return "AArch64ISD::FCMP";
1199 case AArch64ISD::DUP: return "AArch64ISD::DUP";
1200 case AArch64ISD::DUPLANE8: return "AArch64ISD::DUPLANE8";
1201 case AArch64ISD::DUPLANE16: return "AArch64ISD::DUPLANE16";
1202 case AArch64ISD::DUPLANE32: return "AArch64ISD::DUPLANE32";
1203 case AArch64ISD::DUPLANE64: return "AArch64ISD::DUPLANE64";
1204 case AArch64ISD::MOVI: return "AArch64ISD::MOVI";
1205 case AArch64ISD::MOVIshift: return "AArch64ISD::MOVIshift";
1206 case AArch64ISD::MOVIedit: return "AArch64ISD::MOVIedit";
1207 case AArch64ISD::MOVImsl: return "AArch64ISD::MOVImsl";
1208 case AArch64ISD::FMOV: return "AArch64ISD::FMOV";
1209 case AArch64ISD::MVNIshift: return "AArch64ISD::MVNIshift";
1210 case AArch64ISD::MVNImsl: return "AArch64ISD::MVNImsl";
1211 case AArch64ISD::BICi: return "AArch64ISD::BICi";
1212 case AArch64ISD::ORRi: return "AArch64ISD::ORRi";
1213 case AArch64ISD::BSL: return "AArch64ISD::BSL";
1214 case AArch64ISD::NEG: return "AArch64ISD::NEG";
1215 case AArch64ISD::EXTR: return "AArch64ISD::EXTR";
1216 case AArch64ISD::ZIP1: return "AArch64ISD::ZIP1";
1217 case AArch64ISD::ZIP2: return "AArch64ISD::ZIP2";
1218 case AArch64ISD::UZP1: return "AArch64ISD::UZP1";
1219 case AArch64ISD::UZP2: return "AArch64ISD::UZP2";
1220 case AArch64ISD::TRN1: return "AArch64ISD::TRN1";
1221 case AArch64ISD::TRN2: return "AArch64ISD::TRN2";
1222 case AArch64ISD::REV16: return "AArch64ISD::REV16";
1223 case AArch64ISD::REV32: return "AArch64ISD::REV32";
1224 case AArch64ISD::REV64: return "AArch64ISD::REV64";
1225 case AArch64ISD::EXT: return "AArch64ISD::EXT";
1226 case AArch64ISD::VSHL: return "AArch64ISD::VSHL";
1227 case AArch64ISD::VLSHR: return "AArch64ISD::VLSHR";
1228 case AArch64ISD::VASHR: return "AArch64ISD::VASHR";
1229 case AArch64ISD::CMEQ: return "AArch64ISD::CMEQ";
1230 case AArch64ISD::CMGE: return "AArch64ISD::CMGE";
1231 case AArch64ISD::CMGT: return "AArch64ISD::CMGT";
1232 case AArch64ISD::CMHI: return "AArch64ISD::CMHI";
1233 case AArch64ISD::CMHS: return "AArch64ISD::CMHS";
1234 case AArch64ISD::FCMEQ: return "AArch64ISD::FCMEQ";
1235 case AArch64ISD::FCMGE: return "AArch64ISD::FCMGE";
1236 case AArch64ISD::FCMGT: return "AArch64ISD::FCMGT";
1237 case AArch64ISD::CMEQz: return "AArch64ISD::CMEQz";
1238 case AArch64ISD::CMGEz: return "AArch64ISD::CMGEz";
1239 case AArch64ISD::CMGTz: return "AArch64ISD::CMGTz";
1240 case AArch64ISD::CMLEz: return "AArch64ISD::CMLEz";
1241 case AArch64ISD::CMLTz: return "AArch64ISD::CMLTz";
1242 case AArch64ISD::FCMEQz: return "AArch64ISD::FCMEQz";
1243 case AArch64ISD::FCMGEz: return "AArch64ISD::FCMGEz";
1244 case AArch64ISD::FCMGTz: return "AArch64ISD::FCMGTz";
1245 case AArch64ISD::FCMLEz: return "AArch64ISD::FCMLEz";
1246 case AArch64ISD::FCMLTz: return "AArch64ISD::FCMLTz";
1247 case AArch64ISD::SADDV: return "AArch64ISD::SADDV";
1248 case AArch64ISD::UADDV: return "AArch64ISD::UADDV";
1249 case AArch64ISD::SMINV: return "AArch64ISD::SMINV";
1250 case AArch64ISD::UMINV: return "AArch64ISD::UMINV";
1251 case AArch64ISD::SMAXV: return "AArch64ISD::SMAXV";
1252 case AArch64ISD::UMAXV: return "AArch64ISD::UMAXV";
1253 case AArch64ISD::NOT: return "AArch64ISD::NOT";
1254 case AArch64ISD::BIT: return "AArch64ISD::BIT";
1255 case AArch64ISD::CBZ: return "AArch64ISD::CBZ";
1256 case AArch64ISD::CBNZ: return "AArch64ISD::CBNZ";
1257 case AArch64ISD::TBZ: return "AArch64ISD::TBZ";
1258 case AArch64ISD::TBNZ: return "AArch64ISD::TBNZ";
1259 case AArch64ISD::TC_RETURN: return "AArch64ISD::TC_RETURN";
1260 case AArch64ISD::PREFETCH: return "AArch64ISD::PREFETCH";
1261 case AArch64ISD::SITOF: return "AArch64ISD::SITOF";
1262 case AArch64ISD::UITOF: return "AArch64ISD::UITOF";
1263 case AArch64ISD::NVCAST: return "AArch64ISD::NVCAST";
1264 case AArch64ISD::SQSHL_I: return "AArch64ISD::SQSHL_I";
1265 case AArch64ISD::UQSHL_I: return "AArch64ISD::UQSHL_I";
1266 case AArch64ISD::SRSHR_I: return "AArch64ISD::SRSHR_I";
1267 case AArch64ISD::URSHR_I: return "AArch64ISD::URSHR_I";
1268 case AArch64ISD::SQSHLU_I: return "AArch64ISD::SQSHLU_I";
1269 case AArch64ISD::WrapperLarge: return "AArch64ISD::WrapperLarge";
1270 case AArch64ISD::LD2post: return "AArch64ISD::LD2post";
1271 case AArch64ISD::LD3post: return "AArch64ISD::LD3post";
1272 case AArch64ISD::LD4post: return "AArch64ISD::LD4post";
1273 case AArch64ISD::ST2post: return "AArch64ISD::ST2post";
1274 case AArch64ISD::ST3post: return "AArch64ISD::ST3post";
1275 case AArch64ISD::ST4post: return "AArch64ISD::ST4post";
1276 case AArch64ISD::LD1x2post: return "AArch64ISD::LD1x2post";
1277 case AArch64ISD::LD1x3post: return "AArch64ISD::LD1x3post";
1278 case AArch64ISD::LD1x4post: return "AArch64ISD::LD1x4post";
1279 case AArch64ISD::ST1x2post: return "AArch64ISD::ST1x2post";
1280 case AArch64ISD::ST1x3post: return "AArch64ISD::ST1x3post";
1281 case AArch64ISD::ST1x4post: return "AArch64ISD::ST1x4post";
1282 case AArch64ISD::LD1DUPpost: return "AArch64ISD::LD1DUPpost";
1283 case AArch64ISD::LD2DUPpost: return "AArch64ISD::LD2DUPpost";
1284 case AArch64ISD::LD3DUPpost: return "AArch64ISD::LD3DUPpost";
1285 case AArch64ISD::LD4DUPpost: return "AArch64ISD::LD4DUPpost";
1286 case AArch64ISD::LD1LANEpost: return "AArch64ISD::LD1LANEpost";
1287 case AArch64ISD::LD2LANEpost: return "AArch64ISD::LD2LANEpost";
1288 case AArch64ISD::LD3LANEpost: return "AArch64ISD::LD3LANEpost";
1289 case AArch64ISD::LD4LANEpost: return "AArch64ISD::LD4LANEpost";
1290 case AArch64ISD::ST2LANEpost: return "AArch64ISD::ST2LANEpost";
1291 case AArch64ISD::ST3LANEpost: return "AArch64ISD::ST3LANEpost";
1292 case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost";
1293 case AArch64ISD::SMULL: return "AArch64ISD::SMULL";
1294 case AArch64ISD::UMULL: return "AArch64ISD::UMULL";
1295 case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE";
1296 case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS";
1297 case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE";
1298 case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS";
1299 case AArch64ISD::STG: return "AArch64ISD::STG";
1300 case AArch64ISD::STZG: return "AArch64ISD::STZG";
1301 case AArch64ISD::ST2G: return "AArch64ISD::ST2G";
1302 case AArch64ISD::STZ2G: return "AArch64ISD::STZ2G";
1303 }
1304 return nullptr;
1305}
1306
1307MachineBasicBlock *
1308AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
1309 MachineBasicBlock *MBB) const {
1310 // We materialise the F128CSEL pseudo-instruction as some control flow and a
1311 // phi node:
1312
1313 // OrigBB:
1314 // [... previous instrs leading to comparison ...]
1315 // b.ne TrueBB
1316 // b EndBB
1317 // TrueBB:
1318 // ; Fallthrough
1319 // EndBB:
1320 // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
1321
1322 MachineFunction *MF = MBB->getParent();
1323 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1324 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
1325 DebugLoc DL = MI.getDebugLoc();
1326 MachineFunction::iterator It = ++MBB->getIterator();
1327
1328 Register DestReg = MI.getOperand(0).getReg();
1329 Register IfTrueReg = MI.getOperand(1).getReg();
1330 Register IfFalseReg = MI.getOperand(2).getReg();
1331 unsigned CondCode = MI.getOperand(3).getImm();
1332 bool NZCVKilled = MI.getOperand(4).isKill();
1333
1334 MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
1335 MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
1336 MF->insert(It, TrueBB);
1337 MF->insert(It, EndBB);
1338
1339 // Transfer rest of current basic-block to EndBB
1340 EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
1341 MBB->end());
1342 EndBB->transferSuccessorsAndUpdatePHIs(MBB);
1343
1344 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
1345 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
1346 MBB->addSuccessor(TrueBB);
1347 MBB->addSuccessor(EndBB);
1348
1349 // TrueBB falls through to the end.
1350 TrueBB->addSuccessor(EndBB);
1351
1352 if (!NZCVKilled) {
1353 TrueBB->addLiveIn(AArch64::NZCV);
1354 EndBB->addLiveIn(AArch64::NZCV);
1355 }
1356
1357 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
1358 .addReg(IfTrueReg)
1359 .addMBB(TrueBB)
1360 .addReg(IfFalseReg)
1361 .addMBB(MBB);
1362
1363 MI.eraseFromParent();
1364 return EndBB;
1365}
1366
1367MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
1368 MachineInstr &MI, MachineBasicBlock *BB) const {
1369 assert(!isAsynchronousEHPersonality(classifyEHPersonality(((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1371, __PRETTY_FUNCTION__))
1370 BB->getParent()->getFunction().getPersonalityFn())) &&((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1371, __PRETTY_FUNCTION__))
1371 "SEH does not use catchret!")((!isAsynchronousEHPersonality(classifyEHPersonality( BB->
getParent()->getFunction().getPersonalityFn())) &&
"SEH does not use catchret!") ? static_cast<void> (0) :
__assert_fail ("!isAsynchronousEHPersonality(classifyEHPersonality( BB->getParent()->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1371, __PRETTY_FUNCTION__))
;
1372 return BB;
1373}
1374
1375MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchPad(
1376 MachineInstr &MI, MachineBasicBlock *BB) const {
1377 MI.eraseFromParent();
1378 return BB;
1379}
1380
1381MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
1382 MachineInstr &MI, MachineBasicBlock *BB) const {
1383 switch (MI.getOpcode()) {
1384 default:
1385#ifndef NDEBUG
1386 MI.dump();
1387#endif
1388 llvm_unreachable("Unexpected instruction for custom inserter!")::llvm::llvm_unreachable_internal("Unexpected instruction for custom inserter!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1388)
;
1389
1390 case AArch64::F128CSEL:
1391 return EmitF128CSEL(MI, BB);
1392
1393 case TargetOpcode::STACKMAP:
1394 case TargetOpcode::PATCHPOINT:
1395 return emitPatchPoint(MI, BB);
1396
1397 case AArch64::CATCHRET:
1398 return EmitLoweredCatchRet(MI, BB);
1399 case AArch64::CATCHPAD:
1400 return EmitLoweredCatchPad(MI, BB);
1401 }
1402}
1403
1404//===----------------------------------------------------------------------===//
1405// AArch64 Lowering private implementation.
1406//===----------------------------------------------------------------------===//
1407
1408//===----------------------------------------------------------------------===//
1409// Lowering Code
1410//===----------------------------------------------------------------------===//
1411
1412/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
1413/// CC
1414static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
1415 switch (CC) {
1416 default:
1417 llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1417)
;
1418 case ISD::SETNE:
1419 return AArch64CC::NE;
1420 case ISD::SETEQ:
1421 return AArch64CC::EQ;
1422 case ISD::SETGT:
1423 return AArch64CC::GT;
1424 case ISD::SETGE:
1425 return AArch64CC::GE;
1426 case ISD::SETLT:
1427 return AArch64CC::LT;
1428 case ISD::SETLE:
1429 return AArch64CC::LE;
1430 case ISD::SETUGT:
1431 return AArch64CC::HI;
1432 case ISD::SETUGE:
1433 return AArch64CC::HS;
1434 case ISD::SETULT:
1435 return AArch64CC::LO;
1436 case ISD::SETULE:
1437 return AArch64CC::LS;
1438 }
1439}
1440
1441/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
1442static void changeFPCCToAArch64CC(ISD::CondCode CC,
1443 AArch64CC::CondCode &CondCode,
1444 AArch64CC::CondCode &CondCode2) {
1445 CondCode2 = AArch64CC::AL;
1446 switch (CC) {
1447 default:
1448 llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1448)
;
1449 case ISD::SETEQ:
1450 case ISD::SETOEQ:
1451 CondCode = AArch64CC::EQ;
1452 break;
1453 case ISD::SETGT:
1454 case ISD::SETOGT:
1455 CondCode = AArch64CC::GT;
1456 break;
1457 case ISD::SETGE:
1458 case ISD::SETOGE:
1459 CondCode = AArch64CC::GE;
1460 break;
1461 case ISD::SETOLT:
1462 CondCode = AArch64CC::MI;
1463 break;
1464 case ISD::SETOLE:
1465 CondCode = AArch64CC::LS;
1466 break;
1467 case ISD::SETONE:
1468 CondCode = AArch64CC::MI;
1469 CondCode2 = AArch64CC::GT;
1470 break;
1471 case ISD::SETO:
1472 CondCode = AArch64CC::VC;
1473 break;
1474 case ISD::SETUO:
1475 CondCode = AArch64CC::VS;
1476 break;
1477 case ISD::SETUEQ:
1478 CondCode = AArch64CC::EQ;
1479 CondCode2 = AArch64CC::VS;
1480 break;
1481 case ISD::SETUGT:
1482 CondCode = AArch64CC::HI;
1483 break;
1484 case ISD::SETUGE:
1485 CondCode = AArch64CC::PL;
1486 break;
1487 case ISD::SETLT:
1488 case ISD::SETULT:
1489 CondCode = AArch64CC::LT;
1490 break;
1491 case ISD::SETLE:
1492 case ISD::SETULE:
1493 CondCode = AArch64CC::LE;
1494 break;
1495 case ISD::SETNE:
1496 case ISD::SETUNE:
1497 CondCode = AArch64CC::NE;
1498 break;
1499 }
1500}
1501
1502/// Convert a DAG fp condition code to an AArch64 CC.
1503/// This differs from changeFPCCToAArch64CC in that it returns cond codes that
1504/// should be AND'ed instead of OR'ed.
1505static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
1506 AArch64CC::CondCode &CondCode,
1507 AArch64CC::CondCode &CondCode2) {
1508 CondCode2 = AArch64CC::AL;
1509 switch (CC) {
1510 default:
1511 changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1512 assert(CondCode2 == AArch64CC::AL)((CondCode2 == AArch64CC::AL) ? static_cast<void> (0) :
__assert_fail ("CondCode2 == AArch64CC::AL", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1512, __PRETTY_FUNCTION__))
;
1513 break;
1514 case ISD::SETONE:
1515 // (a one b)
1516 // == ((a olt b) || (a ogt b))
1517 // == ((a ord b) && (a une b))
1518 CondCode = AArch64CC::VC;
1519 CondCode2 = AArch64CC::NE;
1520 break;
1521 case ISD::SETUEQ:
1522 // (a ueq b)
1523 // == ((a uno b) || (a oeq b))
1524 // == ((a ule b) && (a uge b))
1525 CondCode = AArch64CC::PL;
1526 CondCode2 = AArch64CC::LE;
1527 break;
1528 }
1529}
1530
1531/// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
1532/// CC usable with the vector instructions. Fewer operations are available
1533/// without a real NZCV register, so we have to use less efficient combinations
1534/// to get the same effect.
1535static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
1536 AArch64CC::CondCode &CondCode,
1537 AArch64CC::CondCode &CondCode2,
1538 bool &Invert) {
1539 Invert = false;
1540 switch (CC) {
1541 default:
1542 // Mostly the scalar mappings work fine.
1543 changeFPCCToAArch64CC(CC, CondCode, CondCode2);
1544 break;
1545 case ISD::SETUO:
1546 Invert = true;
1547 LLVM_FALLTHROUGH[[gnu::fallthrough]];
1548 case ISD::SETO:
1549 CondCode = AArch64CC::MI;
1550 CondCode2 = AArch64CC::GE;
1551 break;
1552 case ISD::SETUEQ:
1553 case ISD::SETULT:
1554 case ISD::SETULE:
1555 case ISD::SETUGT:
1556 case ISD::SETUGE:
1557 // All of the compare-mask comparisons are ordered, but we can switch
1558 // between the two by a double inversion. E.g. ULE == !OGT.
1559 Invert = true;
1560 changeFPCCToAArch64CC(getSetCCInverse(CC, false), CondCode, CondCode2);
1561 break;
1562 }
1563}
1564
1565static bool isLegalArithImmed(uint64_t C) {
1566 // Matches AArch64DAGToDAGISel::SelectArithImmed().
1567 bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
1568 LLVM_DEBUG(dbgs() << "Is imm " << Cdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Is imm " << C <<
" legal: " << (IsLegal ? "yes\n" : "no\n"); } } while (
false)
1569 << " legal: " << (IsLegal ? "yes\n" : "no\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Is imm " << C <<
" legal: " << (IsLegal ? "yes\n" : "no\n"); } } while (
false)
;
1570 return IsLegal;
1571}
1572
1573// Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
1574// the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
1575// can be set differently by this operation. It comes down to whether
1576// "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
1577// everything is fine. If not then the optimization is wrong. Thus general
1578// comparisons are only valid if op2 != 0.
1579//
1580// So, finally, the only LLVM-native comparisons that don't mention C and V
1581// are SETEQ and SETNE. They're the only ones we can safely use CMN for in
1582// the absence of information about op2.
1583static bool isCMN(SDValue Op, ISD::CondCode CC) {
1584 return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
1585 (CC == ISD::SETEQ || CC == ISD::SETNE);
1586}
1587
1588static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1589 const SDLoc &dl, SelectionDAG &DAG) {
1590 EVT VT = LHS.getValueType();
1591 const bool FullFP16 =
1592 static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1593
1594 if (VT.isFloatingPoint()) {
1595 assert(VT != MVT::f128)((VT != MVT::f128) ? static_cast<void> (0) : __assert_fail
("VT != MVT::f128", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1595, __PRETTY_FUNCTION__))
;
1596 if (VT == MVT::f16 && !FullFP16) {
1597 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
1598 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
1599 VT = MVT::f32;
1600 }
1601 return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
1602 }
1603
1604 // The CMP instruction is just an alias for SUBS, and representing it as
1605 // SUBS means that it's possible to get CSE with subtract operations.
1606 // A later phase can perform the optimization of setting the destination
1607 // register to WZR/XZR if it ends up being unused.
1608 unsigned Opcode = AArch64ISD::SUBS;
1609
1610 if (isCMN(RHS, CC)) {
1611 // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
1612 Opcode = AArch64ISD::ADDS;
1613 RHS = RHS.getOperand(1);
1614 } else if (isCMN(LHS, CC)) {
1615 // As we are looking for EQ/NE compares, the operands can be commuted ; can
1616 // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
1617 Opcode = AArch64ISD::ADDS;
1618 LHS = LHS.getOperand(1);
1619 } else if (LHS.getOpcode() == ISD::AND && isNullConstant(RHS) &&
1620 !isUnsignedIntSetCC(CC)) {
1621 // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
1622 // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
1623 // of the signed comparisons.
1624 Opcode = AArch64ISD::ANDS;
1625 RHS = LHS.getOperand(1);
1626 LHS = LHS.getOperand(0);
1627 }
1628
1629 return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
1630 .getValue(1);
1631}
1632
1633/// \defgroup AArch64CCMP CMP;CCMP matching
1634///
1635/// These functions deal with the formation of CMP;CCMP;... sequences.
1636/// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
1637/// a comparison. They set the NZCV flags to a predefined value if their
1638/// predicate is false. This allows to express arbitrary conjunctions, for
1639/// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
1640/// expressed as:
1641/// cmp A
1642/// ccmp B, inv(CB), CA
1643/// check for CB flags
1644///
1645/// This naturally lets us implement chains of AND operations with SETCC
1646/// operands. And we can even implement some other situations by transforming
1647/// them:
1648/// - We can implement (NEG SETCC) i.e. negating a single comparison by
1649/// negating the flags used in a CCMP/FCCMP operations.
1650/// - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
1651/// by negating the flags we test for afterwards. i.e.
1652/// NEG (CMP CCMP CCCMP ...) can be implemented.
1653/// - Note that we can only ever negate all previously processed results.
1654/// What we can not implement by flipping the flags to test is a negation
1655/// of two sub-trees (because the negation affects all sub-trees emitted so
1656/// far, so the 2nd sub-tree we emit would also affect the first).
1657/// With those tools we can implement some OR operations:
1658/// - (OR (SETCC A) (SETCC B)) can be implemented via:
1659/// NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
1660/// - After transforming OR to NEG/AND combinations we may be able to use NEG
1661/// elimination rules from earlier to implement the whole thing as a
1662/// CCMP/FCCMP chain.
1663///
1664/// As complete example:
1665/// or (or (setCA (cmp A)) (setCB (cmp B)))
1666/// (and (setCC (cmp C)) (setCD (cmp D)))"
1667/// can be reassociated to:
1668/// or (and (setCC (cmp C)) setCD (cmp D))
1669// (or (setCA (cmp A)) (setCB (cmp B)))
1670/// can be transformed to:
1671/// not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
1672/// (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
1673/// which can be implemented as:
1674/// cmp C
1675/// ccmp D, inv(CD), CC
1676/// ccmp A, CA, inv(CD)
1677/// ccmp B, CB, inv(CA)
1678/// check for CB flags
1679///
1680/// A counterexample is "or (and A B) (and C D)" which translates to
1681/// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
1682/// can only implement 1 of the inner (not) operations, but not both!
1683/// @{
1684
1685/// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
1686static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
1687 ISD::CondCode CC, SDValue CCOp,
1688 AArch64CC::CondCode Predicate,
1689 AArch64CC::CondCode OutCC,
1690 const SDLoc &DL, SelectionDAG &DAG) {
1691 unsigned Opcode = 0;
1692 const bool FullFP16 =
1693 static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
1694
1695 if (LHS.getValueType().isFloatingPoint()) {
1696 assert(LHS.getValueType() != MVT::f128)((LHS.getValueType() != MVT::f128) ? static_cast<void> (
0) : __assert_fail ("LHS.getValueType() != MVT::f128", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1696, __PRETTY_FUNCTION__))
;
1697 if (LHS.getValueType() == MVT::f16 && !FullFP16) {
1698 LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
1699 RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
1700 }
1701 Opcode = AArch64ISD::FCCMP;
1702 } else if (RHS.getOpcode() == ISD::SUB) {
1703 SDValue SubOp0 = RHS.getOperand(0);
1704 if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
1705 // See emitComparison() on why we can only do this for SETEQ and SETNE.
1706 Opcode = AArch64ISD::CCMN;
1707 RHS = RHS.getOperand(1);
1708 }
1709 }
1710 if (Opcode == 0)
1711 Opcode = AArch64ISD::CCMP;
1712
1713 SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
1714 AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
1715 unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
1716 SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
1717 return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
1718}
1719
1720/// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
1721/// expressed as a conjunction. See \ref AArch64CCMP.
1722/// \param CanNegate Set to true if we can negate the whole sub-tree just by
1723/// changing the conditions on the SETCC tests.
1724/// (this means we can call emitConjunctionRec() with
1725/// Negate==true on this sub-tree)
1726/// \param MustBeFirst Set to true if this subtree needs to be negated and we
1727/// cannot do the negation naturally. We are required to
1728/// emit the subtree first in this case.
1729/// \param WillNegate Is true if are called when the result of this
1730/// subexpression must be negated. This happens when the
1731/// outer expression is an OR. We can use this fact to know
1732/// that we have a double negation (or (or ...) ...) that
1733/// can be implemented for free.
1734static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
1735 bool &MustBeFirst, bool WillNegate,
1736 unsigned Depth = 0) {
1737 if (!Val.hasOneUse())
1738 return false;
1739 unsigned Opcode = Val->getOpcode();
1740 if (Opcode == ISD::SETCC) {
1741 if (Val->getOperand(0).getValueType() == MVT::f128)
1742 return false;
1743 CanNegate = true;
1744 MustBeFirst = false;
1745 return true;
1746 }
1747 // Protect against exponential runtime and stack overflow.
1748 if (Depth > 6)
1749 return false;
1750 if (Opcode == ISD::AND || Opcode == ISD::OR) {
1751 bool IsOR = Opcode == ISD::OR;
1752 SDValue O0 = Val->getOperand(0);
1753 SDValue O1 = Val->getOperand(1);
1754 bool CanNegateL;
1755 bool MustBeFirstL;
1756 if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
1757 return false;
1758 bool CanNegateR;
1759 bool MustBeFirstR;
1760 if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
1761 return false;
1762
1763 if (MustBeFirstL && MustBeFirstR)
1764 return false;
1765
1766 if (IsOR) {
1767 // For an OR expression we need to be able to naturally negate at least
1768 // one side or we cannot do the transformation at all.
1769 if (!CanNegateL && !CanNegateR)
1770 return false;
1771 // If we the result of the OR will be negated and we can naturally negate
1772 // the leafs, then this sub-tree as a whole negates naturally.
1773 CanNegate = WillNegate && CanNegateL && CanNegateR;
1774 // If we cannot naturally negate the whole sub-tree, then this must be
1775 // emitted first.
1776 MustBeFirst = !CanNegate;
1777 } else {
1778 assert(Opcode == ISD::AND && "Must be OR or AND")((Opcode == ISD::AND && "Must be OR or AND") ? static_cast
<void> (0) : __assert_fail ("Opcode == ISD::AND && \"Must be OR or AND\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1778, __PRETTY_FUNCTION__))
;
1779 // We cannot naturally negate an AND operation.
1780 CanNegate = false;
1781 MustBeFirst = MustBeFirstL || MustBeFirstR;
1782 }
1783 return true;
1784 }
1785 return false;
1786}
1787
1788/// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
1789/// of CCMP/CFCMP ops. See @ref AArch64CCMP.
1790/// Tries to transform the given i1 producing node @p Val to a series compare
1791/// and conditional compare operations. @returns an NZCV flags producing node
1792/// and sets @p OutCC to the flags that should be tested or returns SDValue() if
1793/// transformation was not possible.
1794/// \p Negate is true if we want this sub-tree being negated just by changing
1795/// SETCC conditions.
1796static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
1797 AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
1798 AArch64CC::CondCode Predicate) {
1799 // We're at a tree leaf, produce a conditional comparison operation.
1800 unsigned Opcode = Val->getOpcode();
1801 if (Opcode == ISD::SETCC) {
1802 SDValue LHS = Val->getOperand(0);
1803 SDValue RHS = Val->getOperand(1);
1804 ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
1805 bool isInteger = LHS.getValueType().isInteger();
1806 if (Negate)
1807 CC = getSetCCInverse(CC, isInteger);
1808 SDLoc DL(Val);
1809 // Determine OutCC and handle FP special case.
1810 if (isInteger) {
1811 OutCC = changeIntCCToAArch64CC(CC);
1812 } else {
1813 assert(LHS.getValueType().isFloatingPoint())((LHS.getValueType().isFloatingPoint()) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType().isFloatingPoint()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1813, __PRETTY_FUNCTION__))
;
1814 AArch64CC::CondCode ExtraCC;
1815 changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
1816 // Some floating point conditions can't be tested with a single condition
1817 // code. Construct an additional comparison in this case.
1818 if (ExtraCC != AArch64CC::AL) {
1819 SDValue ExtraCmp;
1820 if (!CCOp.getNode())
1821 ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
1822 else
1823 ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
1824 ExtraCC, DL, DAG);
1825 CCOp = ExtraCmp;
1826 Predicate = ExtraCC;
1827 }
1828 }
1829
1830 // Produce a normal comparison if we are first in the chain
1831 if (!CCOp)
1832 return emitComparison(LHS, RHS, CC, DL, DAG);
1833 // Otherwise produce a ccmp.
1834 return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
1835 DAG);
1836 }
1837 assert(Val->hasOneUse() && "Valid conjunction/disjunction tree")((Val->hasOneUse() && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("Val->hasOneUse() && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1837, __PRETTY_FUNCTION__))
;
1838
1839 bool IsOR = Opcode == ISD::OR;
1840
1841 SDValue LHS = Val->getOperand(0);
1842 bool CanNegateL;
1843 bool MustBeFirstL;
1844 bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
1845 assert(ValidL && "Valid conjunction/disjunction tree")((ValidL && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("ValidL && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1845, __PRETTY_FUNCTION__))
;
1846 (void)ValidL;
1847
1848 SDValue RHS = Val->getOperand(1);
1849 bool CanNegateR;
1850 bool MustBeFirstR;
1851 bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
1852 assert(ValidR && "Valid conjunction/disjunction tree")((ValidR && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("ValidR && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1852, __PRETTY_FUNCTION__))
;
1853 (void)ValidR;
1854
1855 // Swap sub-tree that must come first to the right side.
1856 if (MustBeFirstL) {
1857 assert(!MustBeFirstR && "Valid conjunction/disjunction tree")((!MustBeFirstR && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("!MustBeFirstR && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1857, __PRETTY_FUNCTION__))
;
1858 std::swap(LHS, RHS);
1859 std::swap(CanNegateL, CanNegateR);
1860 std::swap(MustBeFirstL, MustBeFirstR);
1861 }
1862
1863 bool NegateR;
1864 bool NegateAfterR;
1865 bool NegateL;
1866 bool NegateAfterAll;
1867 if (Opcode == ISD::OR) {
1868 // Swap the sub-tree that we can negate naturally to the left.
1869 if (!CanNegateL) {
1870 assert(CanNegateR && "at least one side must be negatable")((CanNegateR && "at least one side must be negatable"
) ? static_cast<void> (0) : __assert_fail ("CanNegateR && \"at least one side must be negatable\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1870, __PRETTY_FUNCTION__))
;
1871 assert(!MustBeFirstR && "invalid conjunction/disjunction tree")((!MustBeFirstR && "invalid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("!MustBeFirstR && \"invalid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1871, __PRETTY_FUNCTION__))
;
1872 assert(!Negate)((!Negate) ? static_cast<void> (0) : __assert_fail ("!Negate"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1872, __PRETTY_FUNCTION__))
;
1873 std::swap(LHS, RHS);
1874 NegateR = false;
1875 NegateAfterR = true;
1876 } else {
1877 // Negate the left sub-tree if possible, otherwise negate the result.
1878 NegateR = CanNegateR;
1879 NegateAfterR = !CanNegateR;
1880 }
1881 NegateL = true;
1882 NegateAfterAll = !Negate;
1883 } else {
1884 assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree")((Opcode == ISD::AND && "Valid conjunction/disjunction tree"
) ? static_cast<void> (0) : __assert_fail ("Opcode == ISD::AND && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1884, __PRETTY_FUNCTION__))
;
1885 assert(!Negate && "Valid conjunction/disjunction tree")((!Negate && "Valid conjunction/disjunction tree") ? static_cast
<void> (0) : __assert_fail ("!Negate && \"Valid conjunction/disjunction tree\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 1885, __PRETTY_FUNCTION__))
;
1886
1887 NegateL = false;
1888 NegateR = false;
1889 NegateAfterR = false;
1890 NegateAfterAll = false;
1891 }
1892
1893 // Emit sub-trees.
1894 AArch64CC::CondCode RHSCC;
1895 SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
1896 if (NegateAfterR)
1897 RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
1898 SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
1899 if (NegateAfterAll)
1900 OutCC = AArch64CC::getInvertedCondCode(OutCC);
1901 return CmpL;
1902}
1903
1904/// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
1905/// In some cases this is even possible with OR operations in the expression.
1906/// See \ref AArch64CCMP.
1907/// \see emitConjunctionRec().
1908static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
1909 AArch64CC::CondCode &OutCC) {
1910 bool DummyCanNegate;
1911 bool DummyMustBeFirst;
1912 if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
1913 return SDValue();
1914
1915 return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
1916}
1917
1918/// @}
1919
1920/// Returns how profitable it is to fold a comparison's operand's shift and/or
1921/// extension operations.
1922static unsigned getCmpOperandFoldingProfit(SDValue Op) {
1923 auto isSupportedExtend = [&](SDValue V) {
1924 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
1925 return true;
1926
1927 if (V.getOpcode() == ISD::AND)
1928 if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
1929 uint64_t Mask = MaskCst->getZExtValue();
1930 return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
1931 }
1932
1933 return false;
1934 };
1935
1936 if (!Op.hasOneUse())
1937 return 0;
1938
1939 if (isSupportedExtend(Op))
1940 return 1;
1941
1942 unsigned Opc = Op.getOpcode();
1943 if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
1944 if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
1945 uint64_t Shift = ShiftCst->getZExtValue();
1946 if (isSupportedExtend(Op.getOperand(0)))
1947 return (Shift <= 4) ? 2 : 1;
1948 EVT VT = Op.getValueType();
1949 if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
1950 return 1;
1951 }
1952
1953 return 0;
1954}
1955
1956static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
1957 SDValue &AArch64cc, SelectionDAG &DAG,
1958 const SDLoc &dl) {
1959 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
1960 EVT VT = RHS.getValueType();
1961 uint64_t C = RHSC->getZExtValue();
1962 if (!isLegalArithImmed(C)) {
1963 // Constant does not fit, try adjusting it by one?
1964 switch (CC) {
1965 default:
1966 break;
1967 case ISD::SETLT:
1968 case ISD::SETGE:
1969 if ((VT == MVT::i32 && C != 0x80000000 &&
1970 isLegalArithImmed((uint32_t)(C - 1))) ||
1971 (VT == MVT::i64 && C != 0x80000000ULL &&
1972 isLegalArithImmed(C - 1ULL))) {
1973 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
1974 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1975 RHS = DAG.getConstant(C, dl, VT);
1976 }
1977 break;
1978 case ISD::SETULT:
1979 case ISD::SETUGE:
1980 if ((VT == MVT::i32 && C != 0 &&
1981 isLegalArithImmed((uint32_t)(C - 1))) ||
1982 (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
1983 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
1984 C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
1985 RHS = DAG.getConstant(C, dl, VT);
1986 }
1987 break;
1988 case ISD::SETLE:
1989 case ISD::SETGT:
1990 if ((VT == MVT::i32 && C != INT32_MAX(2147483647) &&
1991 isLegalArithImmed((uint32_t)(C + 1))) ||
1992 (VT == MVT::i64 && C != INT64_MAX(9223372036854775807L) &&
1993 isLegalArithImmed(C + 1ULL))) {
1994 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
1995 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
1996 RHS = DAG.getConstant(C, dl, VT);
1997 }
1998 break;
1999 case ISD::SETULE:
2000 case ISD::SETUGT:
2001 if ((VT == MVT::i32 && C != UINT32_MAX(4294967295U) &&
2002 isLegalArithImmed((uint32_t)(C + 1))) ||
2003 (VT == MVT::i64 && C != UINT64_MAX(18446744073709551615UL) &&
2004 isLegalArithImmed(C + 1ULL))) {
2005 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2006 C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2007 RHS = DAG.getConstant(C, dl, VT);
2008 }
2009 break;
2010 }
2011 }
2012 }
2013
2014 // Comparisons are canonicalized so that the RHS operand is simpler than the
2015 // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2016 // can fold some shift+extend operations on the RHS operand, so swap the
2017 // operands if that can be done.
2018 //
2019 // For example:
2020 // lsl w13, w11, #1
2021 // cmp w13, w12
2022 // can be turned into:
2023 // cmp w12, w11, lsl #1
2024 if (!isa<ConstantSDNode>(RHS) ||
2025 !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
2026 SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2027
2028 if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
2029 std::swap(LHS, RHS);
2030 CC = ISD::getSetCCSwappedOperands(CC);
2031 }
2032 }
2033
2034 SDValue Cmp;
2035 AArch64CC::CondCode AArch64CC;
2036 if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
2037 const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
2038
2039 // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
2040 // For the i8 operand, the largest immediate is 255, so this can be easily
2041 // encoded in the compare instruction. For the i16 operand, however, the
2042 // largest immediate cannot be encoded in the compare.
2043 // Therefore, use a sign extending load and cmn to avoid materializing the
2044 // -1 constant. For example,
2045 // movz w1, #65535
2046 // ldrh w0, [x0, #0]
2047 // cmp w0, w1
2048 // >
2049 // ldrsh w0, [x0, #0]
2050 // cmn w0, #1
2051 // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
2052 // if and only if (sext LHS) == (sext RHS). The checks are in place to
2053 // ensure both the LHS and RHS are truly zero extended and to make sure the
2054 // transformation is profitable.
2055 if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
2056 cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
2057 cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
2058 LHS.getNode()->hasNUsesOfValue(1, 0)) {
2059 int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
2060 if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
2061 SDValue SExt =
2062 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
2063 DAG.getValueType(MVT::i16));
2064 Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
2065 RHS.getValueType()),
2066 CC, dl, DAG);
2067 AArch64CC = changeIntCCToAArch64CC(CC);
2068 }
2069 }
2070
2071 if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
2072 if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
2073 if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
2074 AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
2075 }
2076 }
2077 }
2078
2079 if (!Cmp) {
2080 Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2081 AArch64CC = changeIntCCToAArch64CC(CC);
2082 }
2083 AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
2084 return Cmp;
2085}
2086
2087static std::pair<SDValue, SDValue>
2088getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2089 assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&(((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::
i64) && "Unsupported value type") ? static_cast<void
> (0) : __assert_fail ("(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && \"Unsupported value type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2090, __PRETTY_FUNCTION__))
2090 "Unsupported value type")(((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::
i64) && "Unsupported value type") ? static_cast<void
> (0) : __assert_fail ("(Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) && \"Unsupported value type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2090, __PRETTY_FUNCTION__))
;
2091 SDValue Value, Overflow;
2092 SDLoc DL(Op);
2093 SDValue LHS = Op.getOperand(0);
2094 SDValue RHS = Op.getOperand(1);
2095 unsigned Opc = 0;
2096 switch (Op.getOpcode()) {
2097 default:
2098 llvm_unreachable("Unknown overflow instruction!")::llvm::llvm_unreachable_internal("Unknown overflow instruction!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2098)
;
2099 case ISD::SADDO:
2100 Opc = AArch64ISD::ADDS;
2101 CC = AArch64CC::VS;
2102 break;
2103 case ISD::UADDO:
2104 Opc = AArch64ISD::ADDS;
2105 CC = AArch64CC::HS;
2106 break;
2107 case ISD::SSUBO:
2108 Opc = AArch64ISD::SUBS;
2109 CC = AArch64CC::VS;
2110 break;
2111 case ISD::USUBO:
2112 Opc = AArch64ISD::SUBS;
2113 CC = AArch64CC::LO;
2114 break;
2115 // Multiply needs a little bit extra work.
2116 case ISD::SMULO:
2117 case ISD::UMULO: {
2118 CC = AArch64CC::NE;
2119 bool IsSigned = Op.getOpcode() == ISD::SMULO;
2120 if (Op.getValueType() == MVT::i32) {
2121 unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2122 // For a 32 bit multiply with overflow check we want the instruction
2123 // selector to generate a widening multiply (SMADDL/UMADDL). For that we
2124 // need to generate the following pattern:
2125 // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
2126 LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2127 RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2128 SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2129 SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2130 DAG.getConstant(0, DL, MVT::i64));
2131 // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2132 // operation. We need to clear out the upper 32 bits, because we used a
2133 // widening multiply that wrote all 64 bits. In the end this should be a
2134 // noop.
2135 Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2136 if (IsSigned) {
2137 // The signed overflow check requires more than just a simple check for
2138 // any bit set in the upper 32 bits of the result. These bits could be
2139 // just the sign bits of a negative number. To perform the overflow
2140 // check we have to arithmetic shift right the 32nd bit of the result by
2141 // 31 bits. Then we compare the result to the upper 32 bits.
2142 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2143 DAG.getConstant(32, DL, MVT::i64));
2144 UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2145 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2146 DAG.getConstant(31, DL, MVT::i64));
2147 // It is important that LowerBits is last, otherwise the arithmetic
2148 // shift will not be folded into the compare (SUBS).
2149 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2150 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2151 .getValue(1);
2152 } else {
2153 // The overflow check for unsigned multiply is easy. We only need to
2154 // check if any of the upper 32 bits are set. This can be done with a
2155 // CMP (shifted register). For that we need to generate the following
2156 // pattern:
2157 // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2158 SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2159 DAG.getConstant(32, DL, MVT::i64));
2160 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2161 Overflow =
2162 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2163 DAG.getConstant(0, DL, MVT::i64),
2164 UpperBits).getValue(1);
2165 }
2166 break;
2167 }
2168 assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type")((Op.getValueType() == MVT::i64 && "Expected an i64 value type"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == MVT::i64 && \"Expected an i64 value type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2168, __PRETTY_FUNCTION__))
;
2169 // For the 64 bit multiply
2170 Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2171 if (IsSigned) {
2172 SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2173 SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2174 DAG.getConstant(63, DL, MVT::i64));
2175 // It is important that LowerBits is last, otherwise the arithmetic
2176 // shift will not be folded into the compare (SUBS).
2177 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2178 Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2179 .getValue(1);
2180 } else {
2181 SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2182 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2183 Overflow =
2184 DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2185 DAG.getConstant(0, DL, MVT::i64),
2186 UpperBits).getValue(1);
2187 }
2188 break;
2189 }
2190 } // switch (...)
2191
2192 if (Opc) {
2193 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2194
2195 // Emit the AArch64 operation with overflow check.
2196 Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2197 Overflow = Value.getValue(1);
2198 }
2199 return std::make_pair(Value, Overflow);
2200}
2201
2202SDValue AArch64TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
2203 RTLIB::Libcall Call) const {
2204 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2205 MakeLibCallOptions CallOptions;
2206 return makeLibCall(DAG, Call, MVT::f128, Ops, CallOptions, SDLoc(Op)).first;
2207}
2208
2209// Returns true if the given Op is the overflow flag result of an overflow
2210// intrinsic operation.
2211static bool isOverflowIntrOpRes(SDValue Op) {
2212 unsigned Opc = Op.getOpcode();
2213 return (Op.getResNo() == 1 &&
2214 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2215 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2216}
2217
2218static SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) {
2219 SDValue Sel = Op.getOperand(0);
2220 SDValue Other = Op.getOperand(1);
2221 SDLoc dl(Sel);
2222
2223 // If the operand is an overflow checking operation, invert the condition
2224 // code and kill the Not operation. I.e., transform:
2225 // (xor (overflow_op_bool, 1))
2226 // -->
2227 // (csel 1, 0, invert(cc), overflow_op_bool)
2228 // ... which later gets transformed to just a cset instruction with an
2229 // inverted condition code, rather than a cset + eor sequence.
2230 if (isOneConstant(Other) && isOverflowIntrOpRes(Sel)) {
2231 // Only lower legal XALUO ops.
2232 if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
2233 return SDValue();
2234
2235 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2236 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2237 AArch64CC::CondCode CC;
2238 SDValue Value, Overflow;
2239 std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
2240 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2241 return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
2242 CCVal, Overflow);
2243 }
2244 // If neither operand is a SELECT_CC, give up.
2245 if (Sel.getOpcode() != ISD::SELECT_CC)
2246 std::swap(Sel, Other);
2247 if (Sel.getOpcode() != ISD::SELECT_CC)
2248 return Op;
2249
2250 // The folding we want to perform is:
2251 // (xor x, (select_cc a, b, cc, 0, -1) )
2252 // -->
2253 // (csel x, (xor x, -1), cc ...)
2254 //
2255 // The latter will get matched to a CSINV instruction.
2256
2257 ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
2258 SDValue LHS = Sel.getOperand(0);
2259 SDValue RHS = Sel.getOperand(1);
2260 SDValue TVal = Sel.getOperand(2);
2261 SDValue FVal = Sel.getOperand(3);
2262
2263 // FIXME: This could be generalized to non-integer comparisons.
2264 if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
2265 return Op;
2266
2267 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
2268 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
2269
2270 // The values aren't constants, this isn't the pattern we're looking for.
2271 if (!CFVal || !CTVal)
2272 return Op;
2273
2274 // We can commute the SELECT_CC by inverting the condition. This
2275 // might be needed to make this fit into a CSINV pattern.
2276 if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
2277 std::swap(TVal, FVal);
2278 std::swap(CTVal, CFVal);
2279 CC = ISD::getSetCCInverse(CC, true);
2280 }
2281
2282 // If the constants line up, perform the transform!
2283 if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
2284 SDValue CCVal;
2285 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
2286
2287 FVal = Other;
2288 TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
2289 DAG.getConstant(-1ULL, dl, Other.getValueType()));
2290
2291 return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
2292 CCVal, Cmp);
2293 }
2294
2295 return Op;
2296}
2297
2298static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
2299 EVT VT = Op.getValueType();
2300
2301 // Let legalize expand this if it isn't a legal type yet.
2302 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
2303 return SDValue();
2304
2305 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
2306
2307 unsigned Opc;
2308 bool ExtraOp = false;
2309 switch (Op.getOpcode()) {
2310 default:
2311 llvm_unreachable("Invalid code")::llvm::llvm_unreachable_internal("Invalid code", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2311)
;
2312 case ISD::ADDC:
2313 Opc = AArch64ISD::ADDS;
2314 break;
2315 case ISD::SUBC:
2316 Opc = AArch64ISD::SUBS;
2317 break;
2318 case ISD::ADDE:
2319 Opc = AArch64ISD::ADCS;
2320 ExtraOp = true;
2321 break;
2322 case ISD::SUBE:
2323 Opc = AArch64ISD::SBCS;
2324 ExtraOp = true;
2325 break;
2326 }
2327
2328 if (!ExtraOp)
2329 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
2330 return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
2331 Op.getOperand(2));
2332}
2333
2334static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
2335 // Let legalize expand this if it isn't a legal type yet.
2336 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
2337 return SDValue();
2338
2339 SDLoc dl(Op);
2340 AArch64CC::CondCode CC;
2341 // The actual operation that sets the overflow or carry flag.
2342 SDValue Value, Overflow;
2343 std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
2344
2345 // We use 0 and 1 as false and true values.
2346 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
2347 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
2348
2349 // We use an inverted condition, because the conditional select is inverted
2350 // too. This will allow it to be selected to a single instruction:
2351 // CSINC Wd, WZR, WZR, invert(cond).
2352 SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
2353 Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
2354 CCVal, Overflow);
2355
2356 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
2357 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
2358}
2359
2360// Prefetch operands are:
2361// 1: Address to prefetch
2362// 2: bool isWrite
2363// 3: int locality (0 = no locality ... 3 = extreme locality)
2364// 4: bool isDataCache
2365static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
2366 SDLoc DL(Op);
2367 unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
2368 unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
2369 unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
2370
2371 bool IsStream = !Locality;
2372 // When the locality number is set
2373 if (Locality) {
2374 // The front-end should have filtered out the out-of-range values
2375 assert(Locality <= 3 && "Prefetch locality out-of-range")((Locality <= 3 && "Prefetch locality out-of-range"
) ? static_cast<void> (0) : __assert_fail ("Locality <= 3 && \"Prefetch locality out-of-range\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2375, __PRETTY_FUNCTION__))
;
2376 // The locality degree is the opposite of the cache speed.
2377 // Put the number the other way around.
2378 // The encoding starts at 0 for level 1
2379 Locality = 3 - Locality;
2380 }
2381
2382 // built the mask value encoding the expected behavior.
2383 unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
2384 (!IsData << 3) | // IsDataCache bit
2385 (Locality << 1) | // Cache level bits
2386 (unsigned)IsStream; // Stream bit
2387 return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
2388 DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
2389}
2390
2391SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
2392 SelectionDAG &DAG) const {
2393 assert(Op.getValueType() == MVT::f128 && "Unexpected lowering")((Op.getValueType() == MVT::f128 && "Unexpected lowering"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType() == MVT::f128 && \"Unexpected lowering\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2393, __PRETTY_FUNCTION__))
;
2394
2395 RTLIB::Libcall LC;
2396 LC = RTLIB::getFPEXT(Op.getOperand(0).getValueType(), Op.getValueType());
2397
2398 return LowerF128Call(Op, DAG, LC);
2399}
2400
2401SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
2402 SelectionDAG &DAG) const {
2403 if (Op.getOperand(0).getValueType() != MVT::f128) {
2404 // It's legal except when f128 is involved
2405 return Op;
2406 }
2407
2408 RTLIB::Libcall LC;
2409 LC = RTLIB::getFPROUND(Op.getOperand(0).getValueType(), Op.getValueType());
2410
2411 // FP_ROUND node has a second operand indicating whether it is known to be
2412 // precise. That doesn't take part in the LibCall so we can't directly use
2413 // LowerF128Call.
2414 SDValue SrcVal = Op.getOperand(0);
2415 MakeLibCallOptions CallOptions;
2416 return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, CallOptions,
2417 SDLoc(Op)).first;
2418}
2419
2420SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
2421 SelectionDAG &DAG) const {
2422 // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2423 // Any additional optimization in this function should be recorded
2424 // in the cost tables.
2425 EVT InVT = Op.getOperand(0).getValueType();
2426 EVT VT = Op.getValueType();
2427 unsigned NumElts = InVT.getVectorNumElements();
2428
2429 // f16 conversions are promoted to f32 when full fp16 is not supported.
2430 if (InVT.getVectorElementType() == MVT::f16 &&
2431 !Subtarget->hasFullFP16()) {
2432 MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
2433 SDLoc dl(Op);
2434 return DAG.getNode(
2435 Op.getOpcode(), dl, Op.getValueType(),
2436 DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
2437 }
2438
2439 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2440 SDLoc dl(Op);
2441 SDValue Cv =
2442 DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
2443 Op.getOperand(0));
2444 return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
2445 }
2446
2447 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2448 SDLoc dl(Op);
2449 MVT ExtVT =
2450 MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
2451 VT.getVectorNumElements());
2452 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
2453 return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
2454 }
2455
2456 // Type changing conversions are illegal.
2457 return Op;
2458}
2459
2460SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
2461 SelectionDAG &DAG) const {
2462 if (Op.getOperand(0).getValueType().isVector())
2463 return LowerVectorFP_TO_INT(Op, DAG);
2464
2465 // f16 conversions are promoted to f32 when full fp16 is not supported.
2466 if (Op.getOperand(0).getValueType() == MVT::f16 &&
2467 !Subtarget->hasFullFP16()) {
2468 SDLoc dl(Op);
2469 return DAG.getNode(
2470 Op.getOpcode(), dl, Op.getValueType(),
2471 DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, Op.getOperand(0)));
2472 }
2473
2474 if (Op.getOperand(0).getValueType() != MVT::f128) {
2475 // It's legal except when f128 is involved
2476 return Op;
2477 }
2478
2479 RTLIB::Libcall LC;
2480 if (Op.getOpcode() == ISD::FP_TO_SINT)
2481 LC = RTLIB::getFPTOSINT(Op.getOperand(0).getValueType(), Op.getValueType());
2482 else
2483 LC = RTLIB::getFPTOUINT(Op.getOperand(0).getValueType(), Op.getValueType());
2484
2485 SmallVector<SDValue, 2> Ops(Op->op_begin(), Op->op_end());
2486 MakeLibCallOptions CallOptions;
2487 return makeLibCall(DAG, LC, Op.getValueType(), Ops, CallOptions, SDLoc(Op)).first;
2488}
2489
2490static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
2491 // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
2492 // Any additional optimization in this function should be recorded
2493 // in the cost tables.
2494 EVT VT = Op.getValueType();
2495 SDLoc dl(Op);
2496 SDValue In = Op.getOperand(0);
2497 EVT InVT = In.getValueType();
2498
2499 if (VT.getSizeInBits() < InVT.getSizeInBits()) {
2500 MVT CastVT =
2501 MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
2502 InVT.getVectorNumElements());
2503 In = DAG.getNode(Op.getOpcode(), dl, CastVT, In);
2504 return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
2505 }
2506
2507 if (VT.getSizeInBits() > InVT.getSizeInBits()) {
2508 unsigned CastOpc =
2509 Op.getOpcode() == ISD::SINT_TO_FP ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2510 EVT CastVT = VT.changeVectorElementTypeToInteger();
2511 In = DAG.getNode(CastOpc, dl, CastVT, In);
2512 return DAG.getNode(Op.getOpcode(), dl, VT, In);
2513 }
2514
2515 return Op;
2516}
2517
2518SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
2519 SelectionDAG &DAG) const {
2520 if (Op.getValueType().isVector())
2521 return LowerVectorINT_TO_FP(Op, DAG);
2522
2523 // f16 conversions are promoted to f32 when full fp16 is not supported.
2524 if (Op.getValueType() == MVT::f16 &&
2525 !Subtarget->hasFullFP16()) {
2526 SDLoc dl(Op);
2527 return DAG.getNode(
2528 ISD::FP_ROUND, dl, MVT::f16,
2529 DAG.getNode(Op.getOpcode(), dl, MVT::f32, Op.getOperand(0)),
2530 DAG.getIntPtrConstant(0, dl));
2531 }
2532
2533 // i128 conversions are libcalls.
2534 if (Op.getOperand(0).getValueType() == MVT::i128)
2535 return SDValue();
2536
2537 // Other conversions are legal, unless it's to the completely software-based
2538 // fp128.
2539 if (Op.getValueType() != MVT::f128)
2540 return Op;
2541
2542 RTLIB::Libcall LC;
2543 if (Op.getOpcode() == ISD::SINT_TO_FP)
2544 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2545 else
2546 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(), Op.getValueType());
2547
2548 return LowerF128Call(Op, DAG, LC);
2549}
2550
2551SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
2552 SelectionDAG &DAG) const {
2553 // For iOS, we want to call an alternative entry point: __sincos_stret,
2554 // which returns the values in two S / D registers.
2555 SDLoc dl(Op);
2556 SDValue Arg = Op.getOperand(0);
2557 EVT ArgVT = Arg.getValueType();
2558 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2559
2560 ArgListTy Args;
2561 ArgListEntry Entry;
2562
2563 Entry.Node = Arg;
2564 Entry.Ty = ArgTy;
2565 Entry.IsSExt = false;
2566 Entry.IsZExt = false;
2567 Args.push_back(Entry);
2568
2569 RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
2570 : RTLIB::SINCOS_STRET_F32;
2571 const char *LibcallName = getLibcallName(LC);
2572 SDValue Callee =
2573 DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
2574
2575 StructType *RetTy = StructType::get(ArgTy, ArgTy);
2576 TargetLowering::CallLoweringInfo CLI(DAG);
2577 CLI.setDebugLoc(dl)
2578 .setChain(DAG.getEntryNode())
2579 .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
2580
2581 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
2582 return CallResult.first;
2583}
2584
2585static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
2586 if (Op.getValueType() != MVT::f16)
2587 return SDValue();
2588
2589 assert(Op.getOperand(0).getValueType() == MVT::i16)((Op.getOperand(0).getValueType() == MVT::i16) ? static_cast<
void> (0) : __assert_fail ("Op.getOperand(0).getValueType() == MVT::i16"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2589, __PRETTY_FUNCTION__))
;
2590 SDLoc DL(Op);
2591
2592 Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
2593 Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
2594 return SDValue(
2595 DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, MVT::f16, Op,
2596 DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
2597 0);
2598}
2599
2600static EVT getExtensionTo64Bits(const EVT &OrigVT) {
2601 if (OrigVT.getSizeInBits() >= 64)
2602 return OrigVT;
2603
2604 assert(OrigVT.isSimple() && "Expecting a simple value type")((OrigVT.isSimple() && "Expecting a simple value type"
) ? static_cast<void> (0) : __assert_fail ("OrigVT.isSimple() && \"Expecting a simple value type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2604, __PRETTY_FUNCTION__))
;
2605
2606 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
2607 switch (OrigSimpleTy) {
2608 default: llvm_unreachable("Unexpected Vector Type")::llvm::llvm_unreachable_internal("Unexpected Vector Type", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2608)
;
2609 case MVT::v2i8:
2610 case MVT::v2i16:
2611 return MVT::v2i32;
2612 case MVT::v4i8:
2613 return MVT::v4i16;
2614 }
2615}
2616
2617static SDValue addRequiredExtensionForVectorMULL(SDValue N, SelectionDAG &DAG,
2618 const EVT &OrigTy,
2619 const EVT &ExtTy,
2620 unsigned ExtOpcode) {
2621 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
2622 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
2623 // 64-bits we need to insert a new extension so that it will be 64-bits.
2624 assert(ExtTy.is128BitVector() && "Unexpected extension size")((ExtTy.is128BitVector() && "Unexpected extension size"
) ? static_cast<void> (0) : __assert_fail ("ExtTy.is128BitVector() && \"Unexpected extension size\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2624, __PRETTY_FUNCTION__))
;
2625 if (OrigTy.getSizeInBits() >= 64)
2626 return N;
2627
2628 // Must extend size to at least 64 bits to be used as an operand for VMULL.
2629 EVT NewVT = getExtensionTo64Bits(OrigTy);
2630
2631 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
2632}
2633
2634static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
2635 bool isSigned) {
2636 EVT VT = N->getValueType(0);
2637
2638 if (N->getOpcode() != ISD::BUILD_VECTOR)
2639 return false;
2640
2641 for (const SDValue &Elt : N->op_values()) {
2642 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
2643 unsigned EltSize = VT.getScalarSizeInBits();
2644 unsigned HalfSize = EltSize / 2;
2645 if (isSigned) {
2646 if (!isIntN(HalfSize, C->getSExtValue()))
2647 return false;
2648 } else {
2649 if (!isUIntN(HalfSize, C->getZExtValue()))
2650 return false;
2651 }
2652 continue;
2653 }
2654 return false;
2655 }
2656
2657 return true;
2658}
2659
2660static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) {
2661 if (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND)
2662 return addRequiredExtensionForVectorMULL(N->getOperand(0), DAG,
2663 N->getOperand(0)->getValueType(0),
2664 N->getValueType(0),
2665 N->getOpcode());
2666
2667 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR")((N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::BUILD_VECTOR && \"expected BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2667, __PRETTY_FUNCTION__))
;
2668 EVT VT = N->getValueType(0);
2669 SDLoc dl(N);
2670 unsigned EltSize = VT.getScalarSizeInBits() / 2;
2671 unsigned NumElts = VT.getVectorNumElements();
2672 MVT TruncVT = MVT::getIntegerVT(EltSize);
2673 SmallVector<SDValue, 8> Ops;
2674 for (unsigned i = 0; i != NumElts; ++i) {
2675 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
2676 const APInt &CInt = C->getAPIntValue();
2677 // Element types smaller than 32 bits are not legal, so use i32 elements.
2678 // The values are implicitly truncated so sext vs. zext doesn't matter.
2679 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
2680 }
2681 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
2682}
2683
2684static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
2685 return N->getOpcode() == ISD::SIGN_EXTEND ||
2686 isExtendedBUILD_VECTOR(N, DAG, true);
2687}
2688
2689static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
2690 return N->getOpcode() == ISD::ZERO_EXTEND ||
2691 isExtendedBUILD_VECTOR(N, DAG, false);
2692}
2693
2694static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
2695 unsigned Opcode = N->getOpcode();
2696 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2697 SDNode *N0 = N->getOperand(0).getNode();
2698 SDNode *N1 = N->getOperand(1).getNode();
2699 return N0->hasOneUse() && N1->hasOneUse() &&
2700 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
2701 }
2702 return false;
2703}
2704
2705static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
2706 unsigned Opcode = N->getOpcode();
2707 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
2708 SDNode *N0 = N->getOperand(0).getNode();
2709 SDNode *N1 = N->getOperand(1).getNode();
2710 return N0->hasOneUse() && N1->hasOneUse() &&
2711 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
2712 }
2713 return false;
2714}
2715
2716SDValue AArch64TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
2717 SelectionDAG &DAG) const {
2718 // The rounding mode is in bits 23:22 of the FPSCR.
2719 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
2720 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
2721 // so that the shift + and get folded into a bitfield extract.
2722 SDLoc dl(Op);
2723
2724 SDValue FPCR_64 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::i64,
2725 DAG.getConstant(Intrinsic::aarch64_get_fpcr, dl,
2726 MVT::i64));
2727 SDValue FPCR_32 = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, FPCR_64);
2728 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPCR_32,
2729 DAG.getConstant(1U << 22, dl, MVT::i32));
2730 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
2731 DAG.getConstant(22, dl, MVT::i32));
2732 return DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
2733 DAG.getConstant(3, dl, MVT::i32));
2734}
2735
2736static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
2737 // Multiplications are only custom-lowered for 128-bit vectors so that
2738 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
2739 EVT VT = Op.getValueType();
2740 assert(VT.is128BitVector() && VT.isInteger() &&((VT.is128BitVector() && VT.isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2741, __PRETTY_FUNCTION__))
2741 "unexpected type for custom-lowering ISD::MUL")((VT.is128BitVector() && VT.isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2741, __PRETTY_FUNCTION__))
;
2742 SDNode *N0 = Op.getOperand(0).getNode();
2743 SDNode *N1 = Op.getOperand(1).getNode();
2744 unsigned NewOpc = 0;
2745 bool isMLA = false;
2746 bool isN0SExt = isSignExtended(N0, DAG);
2747 bool isN1SExt = isSignExtended(N1, DAG);
2748 if (isN0SExt && isN1SExt)
2749 NewOpc = AArch64ISD::SMULL;
2750 else {
2751 bool isN0ZExt = isZeroExtended(N0, DAG);
2752 bool isN1ZExt = isZeroExtended(N1, DAG);
2753 if (isN0ZExt && isN1ZExt)
2754 NewOpc = AArch64ISD::UMULL;
2755 else if (isN1SExt || isN1ZExt) {
2756 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
2757 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
2758 if (isN1SExt && isAddSubSExt(N0, DAG)) {
2759 NewOpc = AArch64ISD::SMULL;
2760 isMLA = true;
2761 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
2762 NewOpc = AArch64ISD::UMULL;
2763 isMLA = true;
2764 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
2765 std::swap(N0, N1);
2766 NewOpc = AArch64ISD::UMULL;
2767 isMLA = true;
2768 }
2769 }
2770
2771 if (!NewOpc) {
2772 if (VT == MVT::v2i64)
2773 // Fall through to expand this. It is not legal.
2774 return SDValue();
2775 else
2776 // Other vector multiplications are legal.
2777 return Op;
2778 }
2779 }
2780
2781 // Legalize to a S/UMULL instruction
2782 SDLoc DL(Op);
2783 SDValue Op0;
2784 SDValue Op1 = skipExtensionForVectorMULL(N1, DAG);
2785 if (!isMLA) {
2786 Op0 = skipExtensionForVectorMULL(N0, DAG);
2787 assert(Op0.getValueType().is64BitVector() &&((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2789, __PRETTY_FUNCTION__))
2788 Op1.getValueType().is64BitVector() &&((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2789, __PRETTY_FUNCTION__))
2789 "unexpected types for extended operands to VMULL")((Op0.getValueType().is64BitVector() && Op1.getValueType
().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2789, __PRETTY_FUNCTION__))
;
2790 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
2791 }
2792 // Optimizing (zext A + zext B) * C, to (S/UMULL A, C) + (S/UMULL B, C) during
2793 // isel lowering to take advantage of no-stall back to back s/umul + s/umla.
2794 // This is true for CPUs with accumulate forwarding such as Cortex-A53/A57
2795 SDValue N00 = skipExtensionForVectorMULL(N0->getOperand(0).getNode(), DAG);
2796 SDValue N01 = skipExtensionForVectorMULL(N0->getOperand(1).getNode(), DAG);
2797 EVT Op1VT = Op1.getValueType();
2798 return DAG.getNode(N0->getOpcode(), DL, VT,
2799 DAG.getNode(NewOpc, DL, VT,
2800 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
2801 DAG.getNode(NewOpc, DL, VT,
2802 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
2803}
2804
2805SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
2806 SelectionDAG &DAG) const {
2807 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
2808 SDLoc dl(Op);
2809 switch (IntNo) {
2810 default: return SDValue(); // Don't custom lower most intrinsics.
2811 case Intrinsic::thread_pointer: {
2812 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2813 return DAG.getNode(AArch64ISD::THREAD_POINTER, dl, PtrVT);
2814 }
2815 case Intrinsic::aarch64_neon_abs: {
2816 EVT Ty = Op.getValueType();
2817 if (Ty == MVT::i64) {
2818 SDValue Result = DAG.getNode(ISD::BITCAST, dl, MVT::v1i64,
2819 Op.getOperand(1));
2820 Result = DAG.getNode(ISD::ABS, dl, MVT::v1i64, Result);
2821 return DAG.getNode(ISD::BITCAST, dl, MVT::i64, Result);
2822 } else if (Ty.isVector() && Ty.isInteger() && isTypeLegal(Ty)) {
2823 return DAG.getNode(ISD::ABS, dl, Ty, Op.getOperand(1));
2824 } else {
2825 report_fatal_error("Unexpected type for AArch64 NEON intrinic");
2826 }
2827 }
2828 case Intrinsic::aarch64_neon_smax:
2829 return DAG.getNode(ISD::SMAX, dl, Op.getValueType(),
2830 Op.getOperand(1), Op.getOperand(2));
2831 case Intrinsic::aarch64_neon_umax:
2832 return DAG.getNode(ISD::UMAX, dl, Op.getValueType(),
2833 Op.getOperand(1), Op.getOperand(2));
2834 case Intrinsic::aarch64_neon_smin:
2835 return DAG.getNode(ISD::SMIN, dl, Op.getValueType(),
2836 Op.getOperand(1), Op.getOperand(2));
2837 case Intrinsic::aarch64_neon_umin:
2838 return DAG.getNode(ISD::UMIN, dl, Op.getValueType(),
2839 Op.getOperand(1), Op.getOperand(2));
2840
2841 case Intrinsic::localaddress: {
2842 const auto &MF = DAG.getMachineFunction();
2843 const auto *RegInfo = Subtarget->getRegisterInfo();
2844 unsigned Reg = RegInfo->getLocalAddressRegister(MF);
2845 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg,
2846 Op.getSimpleValueType());
2847 }
2848
2849 case Intrinsic::eh_recoverfp: {
2850 // FIXME: This needs to be implemented to correctly handle highly aligned
2851 // stack objects. For now we simply return the incoming FP. Refer D53541
2852 // for more details.
2853 SDValue FnOp = Op.getOperand(1);
2854 SDValue IncomingFPOp = Op.getOperand(2);
2855 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
2856 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
2857 if (!Fn)
2858 report_fatal_error(
2859 "llvm.eh.recoverfp must take a function as the first argument");
2860 return IncomingFPOp;
2861 }
2862 }
2863}
2864
2865// Custom lower trunc store for v4i8 vectors, since it is promoted to v4i16.
2866static SDValue LowerTruncateVectorStore(SDLoc DL, StoreSDNode *ST,
2867 EVT VT, EVT MemVT,
2868 SelectionDAG &DAG) {
2869 assert(VT.isVector() && "VT should be a vector type")((VT.isVector() && "VT should be a vector type") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"VT should be a vector type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2869, __PRETTY_FUNCTION__))
;
2870 assert(MemVT == MVT::v4i8 && VT == MVT::v4i16)((MemVT == MVT::v4i8 && VT == MVT::v4i16) ? static_cast
<void> (0) : __assert_fail ("MemVT == MVT::v4i8 && VT == MVT::v4i16"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2870, __PRETTY_FUNCTION__))
;
2871
2872 SDValue Value = ST->getValue();
2873
2874 // It first extend the promoted v4i16 to v8i16, truncate to v8i8, and extract
2875 // the word lane which represent the v4i8 subvector. It optimizes the store
2876 // to:
2877 //
2878 // xtn v0.8b, v0.8h
2879 // str s0, [x0]
2880
2881 SDValue Undef = DAG.getUNDEF(MVT::i16);
2882 SDValue UndefVec = DAG.getBuildVector(MVT::v4i16, DL,
2883 {Undef, Undef, Undef, Undef});
2884
2885 SDValue TruncExt = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i16,
2886 Value, UndefVec);
2887 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i8, TruncExt);
2888
2889 Trunc = DAG.getNode(ISD::BITCAST, DL, MVT::v2i32, Trunc);
2890 SDValue ExtractTrunc = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32,
2891 Trunc, DAG.getConstant(0, DL, MVT::i64));
2892
2893 return DAG.getStore(ST->getChain(), DL, ExtractTrunc,
2894 ST->getBasePtr(), ST->getMemOperand());
2895}
2896
2897// Custom lowering for any store, vector or scalar and/or default or with
2898// a truncate operations. Currently only custom lower truncate operation
2899// from vector v4i16 to v4i8.
2900SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
2901 SelectionDAG &DAG) const {
2902 SDLoc Dl(Op);
2903 StoreSDNode *StoreNode = cast<StoreSDNode>(Op);
2904 assert (StoreNode && "Can only custom lower store nodes")((StoreNode && "Can only custom lower store nodes") ?
static_cast<void> (0) : __assert_fail ("StoreNode && \"Can only custom lower store nodes\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2904, __PRETTY_FUNCTION__))
;
2905
2906 SDValue Value = StoreNode->getValue();
2907
2908 EVT VT = Value.getValueType();
2909 EVT MemVT = StoreNode->getMemoryVT();
2910
2911 assert (VT.isVector() && "Can only custom lower vector store types")((VT.isVector() && "Can only custom lower vector store types"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Can only custom lower vector store types\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2911, __PRETTY_FUNCTION__))
;
2912
2913 unsigned AS = StoreNode->getAddressSpace();
2914 unsigned Align = StoreNode->getAlignment();
2915 if (Align < MemVT.getStoreSize() &&
2916 !allowsMisalignedMemoryAccesses(
2917 MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) {
2918 return scalarizeVectorStore(StoreNode, DAG);
2919 }
2920
2921 if (StoreNode->isTruncatingStore()) {
2922 return LowerTruncateVectorStore(Dl, StoreNode, VT, MemVT, DAG);
2923 }
2924
2925 return SDValue();
2926}
2927
2928SDValue AArch64TargetLowering::LowerOperation(SDValue Op,
2929 SelectionDAG &DAG) const {
2930 LLVM_DEBUG(dbgs() << "Custom lowering: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "Custom lowering: "; } }
while (false)
;
2931 LLVM_DEBUG(Op.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { Op.dump(); } } while (false)
;
2932
2933 switch (Op.getOpcode()) {
2934 default:
2935 llvm_unreachable("unimplemented operand")::llvm::llvm_unreachable_internal("unimplemented operand", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 2935)
;
2936 return SDValue();
2937 case ISD::BITCAST:
2938 return LowerBITCAST(Op, DAG);
2939 case ISD::GlobalAddress:
2940 return LowerGlobalAddress(Op, DAG);
2941 case ISD::GlobalTLSAddress:
2942 return LowerGlobalTLSAddress(Op, DAG);
2943 case ISD::SETCC:
2944 return LowerSETCC(Op, DAG);
2945 case ISD::BR_CC:
2946 return LowerBR_CC(Op, DAG);
2947 case ISD::SELECT:
2948 return LowerSELECT(Op, DAG);
2949 case ISD::SELECT_CC:
2950 return LowerSELECT_CC(Op, DAG);
2951 case ISD::JumpTable:
2952 return LowerJumpTable(Op, DAG);
2953 case ISD::BR_JT:
2954 return LowerBR_JT(Op, DAG);
2955 case ISD::ConstantPool:
2956 return LowerConstantPool(Op, DAG);
2957 case ISD::BlockAddress:
2958 return LowerBlockAddress(Op, DAG);
2959 case ISD::VASTART:
2960 return LowerVASTART(Op, DAG);
2961 case ISD::VACOPY:
2962 return LowerVACOPY(Op, DAG);
2963 case ISD::VAARG:
2964 return LowerVAARG(Op, DAG);
2965 case ISD::ADDC:
2966 case ISD::ADDE:
2967 case ISD::SUBC:
2968 case ISD::SUBE:
2969 return LowerADDC_ADDE_SUBC_SUBE(Op, DAG);
2970 case ISD::SADDO:
2971 case ISD::UADDO:
2972 case ISD::SSUBO:
2973 case ISD::USUBO:
2974 case ISD::SMULO:
2975 case ISD::UMULO:
2976 return LowerXALUO(Op, DAG);
2977 case ISD::FADD:
2978 return LowerF128Call(Op, DAG, RTLIB::ADD_F128);
2979 case ISD::FSUB:
2980 return LowerF128Call(Op, DAG, RTLIB::SUB_F128);
2981 case ISD::FMUL:
2982 return LowerF128Call(Op, DAG, RTLIB::MUL_F128);
2983 case ISD::FDIV:
2984 return LowerF128Call(Op, DAG, RTLIB::DIV_F128);
2985 case ISD::FP_ROUND:
2986 return LowerFP_ROUND(Op, DAG);
2987 case ISD::FP_EXTEND:
2988 return LowerFP_EXTEND(Op, DAG);
2989 case ISD::FRAMEADDR:
2990 return LowerFRAMEADDR(Op, DAG);
2991 case ISD::SPONENTRY:
2992 return LowerSPONENTRY(Op, DAG);
2993 case ISD::RETURNADDR:
2994 return LowerRETURNADDR(Op, DAG);
2995 case ISD::ADDROFRETURNADDR:
2996 return LowerADDROFRETURNADDR(Op, DAG);
2997 case ISD::INSERT_VECTOR_ELT:
2998 return LowerINSERT_VECTOR_ELT(Op, DAG);
2999 case ISD::EXTRACT_VECTOR_ELT:
3000 return LowerEXTRACT_VECTOR_ELT(Op, DAG);
3001 case ISD::BUILD_VECTOR:
3002 return LowerBUILD_VECTOR(Op, DAG);
3003 case ISD::VECTOR_SHUFFLE:
3004 return LowerVECTOR_SHUFFLE(Op, DAG);
3005 case ISD::EXTRACT_SUBVECTOR:
3006 return LowerEXTRACT_SUBVECTOR(Op, DAG);
3007 case ISD::SRA:
3008 case ISD::SRL:
3009 case ISD::SHL:
3010 return LowerVectorSRA_SRL_SHL(Op, DAG);
3011 case ISD::SHL_PARTS:
3012 return LowerShiftLeftParts(Op, DAG);
3013 case ISD::SRL_PARTS:
3014 case ISD::SRA_PARTS:
3015 return LowerShiftRightParts(Op, DAG);
3016 case ISD::CTPOP:
3017 return LowerCTPOP(Op, DAG);
3018 case ISD::FCOPYSIGN:
3019 return LowerFCOPYSIGN(Op, DAG);
3020 case ISD::OR:
3021 return LowerVectorOR(Op, DAG);
3022 case ISD::XOR:
3023 return LowerXOR(Op, DAG);
3024 case ISD::PREFETCH:
3025 return LowerPREFETCH(Op, DAG);
3026 case ISD::SINT_TO_FP:
3027 case ISD::UINT_TO_FP:
3028 return LowerINT_TO_FP(Op, DAG);
3029 case ISD::FP_TO_SINT:
3030 case ISD::FP_TO_UINT:
3031 return LowerFP_TO_INT(Op, DAG);
3032 case ISD::FSINCOS:
3033 return LowerFSINCOS(Op, DAG);
3034 case ISD::FLT_ROUNDS_:
3035 return LowerFLT_ROUNDS_(Op, DAG);
3036 case ISD::MUL:
3037 return LowerMUL(Op, DAG);
3038 case ISD::INTRINSIC_WO_CHAIN:
3039 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
3040 case ISD::STORE:
3041 return LowerSTORE(Op, DAG);
3042 case ISD::VECREDUCE_ADD:
3043 case ISD::VECREDUCE_SMAX:
3044 case ISD::VECREDUCE_SMIN:
3045 case ISD::VECREDUCE_UMAX:
3046 case ISD::VECREDUCE_UMIN:
3047 case ISD::VECREDUCE_FMAX:
3048 case ISD::VECREDUCE_FMIN:
3049 return LowerVECREDUCE(Op, DAG);
3050 case ISD::ATOMIC_LOAD_SUB:
3051 return LowerATOMIC_LOAD_SUB(Op, DAG);
3052 case ISD::ATOMIC_LOAD_AND:
3053 return LowerATOMIC_LOAD_AND(Op, DAG);
3054 case ISD::DYNAMIC_STACKALLOC:
3055 return LowerDYNAMIC_STACKALLOC(Op, DAG);
3056 }
3057}
3058
3059//===----------------------------------------------------------------------===//
3060// Calling Convention Implementation
3061//===----------------------------------------------------------------------===//
3062
3063/// Selects the correct CCAssignFn for a given CallingConvention value.
3064CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC,
3065 bool IsVarArg) const {
3066 switch (CC) {
3067 default:
3068 report_fatal_error("Unsupported calling convention.");
3069 case CallingConv::WebKit_JS:
3070 return CC_AArch64_WebKit_JS;
3071 case CallingConv::GHC:
3072 return CC_AArch64_GHC;
3073 case CallingConv::C:
3074 case CallingConv::Fast:
3075 case CallingConv::PreserveMost:
3076 case CallingConv::CXX_FAST_TLS:
3077 case CallingConv::Swift:
3078 if (Subtarget->isTargetWindows() && IsVarArg)
3079 return CC_AArch64_Win64_VarArg;
3080 if (!Subtarget->isTargetDarwin())
3081 return CC_AArch64_AAPCS;
3082 if (!IsVarArg)
3083 return CC_AArch64_DarwinPCS;
3084 return Subtarget->isTargetILP32() ? CC_AArch64_DarwinPCS_ILP32_VarArg
3085 : CC_AArch64_DarwinPCS_VarArg;
3086 case CallingConv::Win64:
3087 return IsVarArg ? CC_AArch64_Win64_VarArg : CC_AArch64_AAPCS;
3088 case CallingConv::AArch64_VectorCall:
3089 return CC_AArch64_AAPCS;
3090 }
3091}
3092
3093CCAssignFn *
3094AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const {
3095 return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS
3096 : RetCC_AArch64_AAPCS;
3097}
3098
3099SDValue AArch64TargetLowering::LowerFormalArguments(
3100 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3101 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3102 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3103 MachineFunction &MF = DAG.getMachineFunction();
3104 MachineFrameInfo &MFI = MF.getFrameInfo();
3105 bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
3106
3107 // Assign locations to all of the incoming arguments.
3108 SmallVector<CCValAssign, 16> ArgLocs;
3109 DenseMap<unsigned, SDValue> CopiedRegs;
3110 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
3111 *DAG.getContext());
3112
3113 // At this point, Ins[].VT may already be promoted to i32. To correctly
3114 // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
3115 // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
3116 // Since AnalyzeFormalArguments uses Ins[].VT for both ValVT and LocVT, here
3117 // we use a special version of AnalyzeFormalArguments to pass in ValVT and
3118 // LocVT.
3119 unsigned NumArgs = Ins.size();
3120 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
3121 unsigned CurArgIdx = 0;
3122 for (unsigned i = 0; i != NumArgs; ++i) {
3123 MVT ValVT = Ins[i].VT;
3124 if (Ins[i].isOrigArg()) {
3125 std::advance(CurOrigArg, Ins[i].getOrigArgIndex() - CurArgIdx);
3126 CurArgIdx = Ins[i].getOrigArgIndex();
3127
3128 // Get type of the original argument.
3129 EVT ActualVT = getValueType(DAG.getDataLayout(), CurOrigArg->getType(),
3130 /*AllowUnknown*/ true);
3131 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : MVT::Other;
3132 // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3133 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3134 ValVT = MVT::i8;
3135 else if (ActualMVT == MVT::i16)
3136 ValVT = MVT::i16;
3137 }
3138 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3139 bool Res =
3140 AssignFn(i, ValVT, ValVT, CCValAssign::Full, Ins[i].Flags, CCInfo);
3141 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3141, __PRETTY_FUNCTION__))
;
3142 (void)Res;
3143 }
3144 assert(ArgLocs.size() == Ins.size())((ArgLocs.size() == Ins.size()) ? static_cast<void> (0)
: __assert_fail ("ArgLocs.size() == Ins.size()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3144, __PRETTY_FUNCTION__))
;
3145 SmallVector<SDValue, 16> ArgValues;
3146 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
3147 CCValAssign &VA = ArgLocs[i];
3148
3149 if (Ins[i].Flags.isByVal()) {
3150 // Byval is used for HFAs in the PCS, but the system should work in a
3151 // non-compliant manner for larger structs.
3152 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3153 int Size = Ins[i].Flags.getByValSize();
3154 unsigned NumRegs = (Size + 7) / 8;
3155
3156 // FIXME: This works on big-endian for composite byvals, which are the common
3157 // case. It should also work for fundamental types too.
3158 unsigned FrameIdx =
3159 MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false);
3160 SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT);
3161 InVals.push_back(FrameIdxN);
3162
3163 continue;
3164 }
3165
3166 SDValue ArgValue;
3167 if (VA.isRegLoc()) {
3168 // Arguments stored in registers.
3169 EVT RegVT = VA.getLocVT();
3170 const TargetRegisterClass *RC;
3171
3172 if (RegVT == MVT::i32)
3173 RC = &AArch64::GPR32RegClass;
3174 else if (RegVT == MVT::i64)
3175 RC = &AArch64::GPR64RegClass;
3176 else if (RegVT == MVT::f16)
3177 RC = &AArch64::FPR16RegClass;
3178 else if (RegVT == MVT::f32)
3179 RC = &AArch64::FPR32RegClass;
3180 else if (RegVT == MVT::f64 || RegVT.is64BitVector())
3181 RC = &AArch64::FPR64RegClass;
3182 else if (RegVT == MVT::f128 || RegVT.is128BitVector())
3183 RC = &AArch64::FPR128RegClass;
3184 else if (RegVT.isScalableVector() &&
3185 RegVT.getVectorElementType() == MVT::i1)
3186 RC = &AArch64::PPRRegClass;
3187 else if (RegVT.isScalableVector())
3188 RC = &AArch64::ZPRRegClass;
3189 else
3190 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering")::llvm::llvm_unreachable_internal("RegVT not supported by FORMAL_ARGUMENTS Lowering"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3190)
;
3191
3192 // Transform the arguments in physical registers into virtual ones.
3193 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3194 ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegVT);
3195
3196 // If this is an 8, 16 or 32-bit value, it is really passed promoted
3197 // to 64 bits. Insert an assert[sz]ext to capture this, then
3198 // truncate to the right size.
3199 switch (VA.getLocInfo()) {
3200 default:
3201 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3201)
;
3202 case CCValAssign::Full:
3203 break;
3204 case CCValAssign::Indirect:
3205 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3206, __PRETTY_FUNCTION__))
3206 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3206, __PRETTY_FUNCTION__))
;
3207 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3207)
;
3208 case CCValAssign::BCvt:
3209 ArgValue = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), ArgValue);
3210 break;
3211 case CCValAssign::AExt:
3212 case CCValAssign::SExt:
3213 case CCValAssign::ZExt:
3214 break;
3215 case CCValAssign::AExtUpper:
3216 ArgValue = DAG.getNode(ISD::SRL, DL, RegVT, ArgValue,
3217 DAG.getConstant(32, DL, RegVT));
3218 ArgValue = DAG.getZExtOrTrunc(ArgValue, DL, VA.getValVT());
3219 break;
3220 }
3221 } else { // VA.isRegLoc()
3222 assert(VA.isMemLoc() && "CCValAssign is neither reg nor mem")((VA.isMemLoc() && "CCValAssign is neither reg nor mem"
) ? static_cast<void> (0) : __assert_fail ("VA.isMemLoc() && \"CCValAssign is neither reg nor mem\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3222, __PRETTY_FUNCTION__))
;
3223 unsigned ArgOffset = VA.getLocMemOffset();
3224 unsigned ArgSize = VA.getValVT().getSizeInBits() / 8;
3225
3226 uint32_t BEAlign = 0;
3227 if (!Subtarget->isLittleEndian() && ArgSize < 8 &&
3228 !Ins[i].Flags.isInConsecutiveRegs())
3229 BEAlign = 8 - ArgSize;
3230
3231 int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true);
3232
3233 // Create load nodes to retrieve arguments from the stack.
3234 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
3235
3236 // For NON_EXTLOAD, generic code in getLoad assert(ValVT == MemVT)
3237 ISD::LoadExtType ExtType = ISD::NON_EXTLOAD;
3238 MVT MemVT = VA.getValVT();
3239
3240 switch (VA.getLocInfo()) {
3241 default:
3242 break;
3243 case CCValAssign::Trunc:
3244 case CCValAssign::BCvt:
3245 MemVT = VA.getLocVT();
3246 break;
3247 case CCValAssign::Indirect:
3248 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3249, __PRETTY_FUNCTION__))
3249 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3249, __PRETTY_FUNCTION__))
;
3250 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3250)
;
3251 case CCValAssign::SExt:
3252 ExtType = ISD::SEXTLOAD;
3253 break;
3254 case CCValAssign::ZExt:
3255 ExtType = ISD::ZEXTLOAD;
3256 break;
3257 case CCValAssign::AExt:
3258 ExtType = ISD::EXTLOAD;
3259 break;
3260 }
3261
3262 ArgValue = DAG.getExtLoad(
3263 ExtType, DL, VA.getLocVT(), Chain, FIN,
3264 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI),
3265 MemVT);
3266
3267 }
3268 if (Subtarget->isTargetILP32() && Ins[i].Flags.isPointer())
3269 ArgValue = DAG.getNode(ISD::AssertZext, DL, ArgValue.getValueType(),
3270 ArgValue, DAG.getValueType(MVT::i32));
3271 InVals.push_back(ArgValue);
3272 }
3273
3274 // varargs
3275 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3276 if (isVarArg) {
3277 if (!Subtarget->isTargetDarwin() || IsWin64) {
3278 // The AAPCS variadic function ABI is identical to the non-variadic
3279 // one. As a result there may be more arguments in registers and we should
3280 // save them for future reference.
3281 // Win64 variadic functions also pass arguments in registers, but all float
3282 // arguments are passed in integer registers.
3283 saveVarArgRegisters(CCInfo, DAG, DL, Chain);
3284 }
3285
3286 // This will point to the next argument passed via stack.
3287 unsigned StackOffset = CCInfo.getNextStackOffset();
3288 // We currently pass all varargs at 8-byte alignment, or 4 for ILP32
3289 StackOffset = alignTo(StackOffset, Subtarget->isTargetILP32() ? 4 : 8);
3290 FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true));
3291
3292 if (MFI.hasMustTailInVarArgFunc()) {
3293 SmallVector<MVT, 2> RegParmTypes;
3294 RegParmTypes.push_back(MVT::i64);
3295 RegParmTypes.push_back(MVT::f128);
3296 // Compute the set of forwarded registers. The rest are scratch.
3297 SmallVectorImpl<ForwardedRegister> &Forwards =
3298 FuncInfo->getForwardedMustTailRegParms();
3299 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes,
3300 CC_AArch64_AAPCS);
3301
3302 // Conservatively forward X8, since it might be used for aggregate return.
3303 if (!CCInfo.isAllocated(AArch64::X8)) {
3304 unsigned X8VReg = MF.addLiveIn(AArch64::X8, &AArch64::GPR64RegClass);
3305 Forwards.push_back(ForwardedRegister(X8VReg, AArch64::X8, MVT::i64));
3306 }
3307 }
3308 }
3309
3310 // On Windows, InReg pointers must be returned, so record the pointer in a
3311 // virtual register at the start of the function so it can be returned in the
3312 // epilogue.
3313 if (IsWin64) {
3314 for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
3315 if (Ins[I].Flags.isInReg()) {
3316 assert(!FuncInfo->getSRetReturnReg())((!FuncInfo->getSRetReturnReg()) ? static_cast<void>
(0) : __assert_fail ("!FuncInfo->getSRetReturnReg()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3316, __PRETTY_FUNCTION__))
;
3317
3318 MVT PtrTy = getPointerTy(DAG.getDataLayout());
3319 Register Reg =
3320 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
3321 FuncInfo->setSRetReturnReg(Reg);
3322
3323 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), DL, Reg, InVals[I]);
3324 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Copy, Chain);
3325 break;
3326 }
3327 }
3328 }
3329
3330 unsigned StackArgSize = CCInfo.getNextStackOffset();
3331 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3332 if (DoesCalleeRestoreStack(CallConv, TailCallOpt)) {
3333 // This is a non-standard ABI so by fiat I say we're allowed to make full
3334 // use of the stack area to be popped, which must be aligned to 16 bytes in
3335 // any case:
3336 StackArgSize = alignTo(StackArgSize, 16);
3337
3338 // If we're expected to restore the stack (e.g. fastcc) then we'll be adding
3339 // a multiple of 16.
3340 FuncInfo->setArgumentStackToRestore(StackArgSize);
3341
3342 // This realignment carries over to the available bytes below. Our own
3343 // callers will guarantee the space is free by giving an aligned value to
3344 // CALLSEQ_START.
3345 }
3346 // Even if we're not expected to free up the space, it's useful to know how
3347 // much is there while considering tail calls (because we can reuse it).
3348 FuncInfo->setBytesInStackArgArea(StackArgSize);
3349
3350 if (Subtarget->hasCustomCallingConv())
3351 Subtarget->getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF);
3352
3353 return Chain;
3354}
3355
3356void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo,
3357 SelectionDAG &DAG,
3358 const SDLoc &DL,
3359 SDValue &Chain) const {
3360 MachineFunction &MF = DAG.getMachineFunction();
3361 MachineFrameInfo &MFI = MF.getFrameInfo();
3362 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3363 auto PtrVT = getPointerTy(DAG.getDataLayout());
3364 bool IsWin64 = Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv());
3365
3366 SmallVector<SDValue, 8> MemOps;
3367
3368 static const MCPhysReg GPRArgRegs[] = { AArch64::X0, AArch64::X1, AArch64::X2,
3369 AArch64::X3, AArch64::X4, AArch64::X5,
3370 AArch64::X6, AArch64::X7 };
3371 static const unsigned NumGPRArgRegs = array_lengthof(GPRArgRegs);
3372 unsigned FirstVariadicGPR = CCInfo.getFirstUnallocated(GPRArgRegs);
3373
3374 unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR);
3375 int GPRIdx = 0;
3376 if (GPRSaveSize != 0) {
3377 if (IsWin64) {
3378 GPRIdx = MFI.CreateFixedObject(GPRSaveSize, -(int)GPRSaveSize, false);
3379 if (GPRSaveSize & 15)
3380 // The extra size here, if triggered, will always be 8.
3381 MFI.CreateFixedObject(16 - (GPRSaveSize & 15), -(int)alignTo(GPRSaveSize, 16), false);
3382 } else
3383 GPRIdx = MFI.CreateStackObject(GPRSaveSize, 8, false);
3384
3385 SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT);
3386
3387 for (unsigned i = FirstVariadicGPR; i < NumGPRArgRegs; ++i) {
3388 unsigned VReg = MF.addLiveIn(GPRArgRegs[i], &AArch64::GPR64RegClass);
3389 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::i64);
3390 SDValue Store = DAG.getStore(
3391 Val.getValue(1), DL, Val, FIN,
3392 IsWin64
3393 ? MachinePointerInfo::getFixedStack(DAG.getMachineFunction(),
3394 GPRIdx,
3395 (i - FirstVariadicGPR) * 8)
3396 : MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 8));
3397 MemOps.push_back(Store);
3398 FIN =
3399 DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getConstant(8, DL, PtrVT));
3400 }
3401 }
3402 FuncInfo->setVarArgsGPRIndex(GPRIdx);
3403 FuncInfo->setVarArgsGPRSize(GPRSaveSize);
3404
3405 if (Subtarget->hasFPARMv8() && !IsWin64) {
3406 static const MCPhysReg FPRArgRegs[] = {
3407 AArch64::Q0, AArch64::Q1, AArch64::Q2, AArch64::Q3,
3408 AArch64::Q4, AArch64::Q5, AArch64::Q6, AArch64::Q7};
3409 static const unsigned NumFPRArgRegs = array_lengthof(FPRArgRegs);
3410 unsigned FirstVariadicFPR = CCInfo.getFirstUnallocated(FPRArgRegs);
3411
3412 unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR);
3413 int FPRIdx = 0;
3414 if (FPRSaveSize != 0) {
3415 FPRIdx = MFI.CreateStackObject(FPRSaveSize, 16, false);
3416
3417 SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT);
3418
3419 for (unsigned i = FirstVariadicFPR; i < NumFPRArgRegs; ++i) {
3420 unsigned VReg = MF.addLiveIn(FPRArgRegs[i], &AArch64::FPR128RegClass);
3421 SDValue Val = DAG.getCopyFromReg(Chain, DL, VReg, MVT::f128);
3422
3423 SDValue Store = DAG.getStore(
3424 Val.getValue(1), DL, Val, FIN,
3425 MachinePointerInfo::getStack(DAG.getMachineFunction(), i * 16));
3426 MemOps.push_back(Store);
3427 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN,
3428 DAG.getConstant(16, DL, PtrVT));
3429 }
3430 }
3431 FuncInfo->setVarArgsFPRIndex(FPRIdx);
3432 FuncInfo->setVarArgsFPRSize(FPRSaveSize);
3433 }
3434
3435 if (!MemOps.empty()) {
3436 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
3437 }
3438}
3439
3440/// LowerCallResult - Lower the result values of a call into the
3441/// appropriate copies out of appropriate physical registers.
3442SDValue AArch64TargetLowering::LowerCallResult(
3443 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
3444 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
3445 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
3446 SDValue ThisVal) const {
3447 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
3448 ? RetCC_AArch64_WebKit_JS
3449 : RetCC_AArch64_AAPCS;
3450 // Assign locations to each value returned by this call.
3451 SmallVector<CCValAssign, 16> RVLocs;
3452 DenseMap<unsigned, SDValue> CopiedRegs;
3453 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3454 *DAG.getContext());
3455 CCInfo.AnalyzeCallResult(Ins, RetCC);
3456
3457 // Copy all of the result registers out of their specified physreg.
3458 for (unsigned i = 0; i != RVLocs.size(); ++i) {
3459 CCValAssign VA = RVLocs[i];
3460
3461 // Pass 'this' value directly from the argument to return value, to avoid
3462 // reg unit interference
3463 if (i == 0 && isThisReturn) {
3464 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&((!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
"unexpected return calling convention register assignment") ?
static_cast<void> (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i64 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3465, __PRETTY_FUNCTION__))
3465 "unexpected return calling convention register assignment")((!VA.needsCustom() && VA.getLocVT() == MVT::i64 &&
"unexpected return calling convention register assignment") ?
static_cast<void> (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i64 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3465, __PRETTY_FUNCTION__))
;
3466 InVals.push_back(ThisVal);
3467 continue;
3468 }
3469
3470 // Avoid copying a physreg twice since RegAllocFast is incompetent and only
3471 // allows one use of a physreg per block.
3472 SDValue Val = CopiedRegs.lookup(VA.getLocReg());
3473 if (!Val) {
3474 Val =
3475 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), InFlag);
3476 Chain = Val.getValue(1);
3477 InFlag = Val.getValue(2);
3478 CopiedRegs[VA.getLocReg()] = Val;
3479 }
3480
3481 switch (VA.getLocInfo()) {
3482 default:
3483 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3483)
;
3484 case CCValAssign::Full:
3485 break;
3486 case CCValAssign::BCvt:
3487 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
3488 break;
3489 case CCValAssign::AExtUpper:
3490 Val = DAG.getNode(ISD::SRL, DL, VA.getLocVT(), Val,
3491 DAG.getConstant(32, DL, VA.getLocVT()));
3492 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3493 case CCValAssign::AExt:
3494 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3495 case CCValAssign::ZExt:
3496 Val = DAG.getZExtOrTrunc(Val, DL, VA.getValVT());
3497 break;
3498 }
3499
3500 InVals.push_back(Val);
3501 }
3502
3503 return Chain;
3504}
3505
3506/// Return true if the calling convention is one that we can guarantee TCO for.
3507static bool canGuaranteeTCO(CallingConv::ID CC) {
3508 return CC == CallingConv::Fast;
3509}
3510
3511/// Return true if we might ever do TCO for calls with this calling convention.
3512static bool mayTailCallThisCC(CallingConv::ID CC) {
3513 switch (CC) {
3514 case CallingConv::C:
3515 case CallingConv::PreserveMost:
3516 case CallingConv::Swift:
3517 return true;
3518 default:
3519 return canGuaranteeTCO(CC);
3520 }
3521}
3522
3523bool AArch64TargetLowering::isEligibleForTailCallOptimization(
3524 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
3525 const SmallVectorImpl<ISD::OutputArg> &Outs,
3526 const SmallVectorImpl<SDValue> &OutVals,
3527 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
3528 if (!mayTailCallThisCC(CalleeCC))
3529 return false;
3530
3531 MachineFunction &MF = DAG.getMachineFunction();
3532 const Function &CallerF = MF.getFunction();
3533 CallingConv::ID CallerCC = CallerF.getCallingConv();
3534 bool CCMatch = CallerCC == CalleeCC;
3535
3536 // Byval parameters hand the function a pointer directly into the stack area
3537 // we want to reuse during a tail call. Working around this *is* possible (see
3538 // X86) but less efficient and uglier in LowerCall.
3539 for (Function::const_arg_iterator i = CallerF.arg_begin(),
3540 e = CallerF.arg_end();
3541 i != e; ++i) {
3542 if (i->hasByValAttr())
3543 return false;
3544
3545 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
3546 // In this case, it is necessary to save/restore X0 in the callee. Tail
3547 // call opt interferes with this. So we disable tail call opt when the
3548 // caller has an argument with "inreg" attribute.
3549
3550 // FIXME: Check whether the callee also has an "inreg" argument.
3551 if (i->hasInRegAttr())
3552 return false;
3553 }
3554
3555 if (getTargetMachine().Options.GuaranteedTailCallOpt)
3556 return canGuaranteeTCO(CalleeCC) && CCMatch;
3557
3558 // Externally-defined functions with weak linkage should not be
3559 // tail-called on AArch64 when the OS does not support dynamic
3560 // pre-emption of symbols, as the AAELF spec requires normal calls
3561 // to undefined weak functions to be replaced with a NOP or jump to the
3562 // next instruction. The behaviour of branch instructions in this
3563 // situation (as used for tail calls) is implementation-defined, so we
3564 // cannot rely on the linker replacing the tail call with a return.
3565 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3566 const GlobalValue *GV = G->getGlobal();
3567 const Triple &TT = getTargetMachine().getTargetTriple();
3568 if (GV->hasExternalWeakLinkage() &&
3569 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
3570 return false;
3571 }
3572
3573 // Now we search for cases where we can use a tail call without changing the
3574 // ABI. Sibcall is used in some places (particularly gcc) to refer to this
3575 // concept.
3576
3577 // I want anyone implementing a new calling convention to think long and hard
3578 // about this assert.
3579 assert((!isVarArg || CalleeCC == CallingConv::C) &&(((!isVarArg || CalleeCC == CallingConv::C) && "Unexpected variadic calling convention"
) ? static_cast<void> (0) : __assert_fail ("(!isVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3580, __PRETTY_FUNCTION__))
3580 "Unexpected variadic calling convention")(((!isVarArg || CalleeCC == CallingConv::C) && "Unexpected variadic calling convention"
) ? static_cast<void> (0) : __assert_fail ("(!isVarArg || CalleeCC == CallingConv::C) && \"Unexpected variadic calling convention\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3580, __PRETTY_FUNCTION__))
;
3581
3582 LLVMContext &C = *DAG.getContext();
3583 if (isVarArg && !Outs.empty()) {
3584 // At least two cases here: if caller is fastcc then we can't have any
3585 // memory arguments (we'd be expected to clean up the stack afterwards). If
3586 // caller is C then we could potentially use its argument area.
3587
3588 // FIXME: for now we take the most conservative of these in both cases:
3589 // disallow all variadic memory operands.
3590 SmallVector<CCValAssign, 16> ArgLocs;
3591 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3592
3593 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, true));
3594 for (const CCValAssign &ArgLoc : ArgLocs)
3595 if (!ArgLoc.isRegLoc())
3596 return false;
3597 }
3598
3599 // Check that the call results are passed in the same way.
3600 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
3601 CCAssignFnForCall(CalleeCC, isVarArg),
3602 CCAssignFnForCall(CallerCC, isVarArg)))
3603 return false;
3604 // The callee has to preserve all registers the caller needs to preserve.
3605 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
3606 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
3607 if (!CCMatch) {
3608 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
3609 if (Subtarget->hasCustomCallingConv()) {
3610 TRI->UpdateCustomCallPreservedMask(MF, &CallerPreserved);
3611 TRI->UpdateCustomCallPreservedMask(MF, &CalleePreserved);
3612 }
3613 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
3614 return false;
3615 }
3616
3617 // Nothing more to check if the callee is taking no arguments
3618 if (Outs.empty())
3619 return true;
3620
3621 SmallVector<CCValAssign, 16> ArgLocs;
3622 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
3623
3624 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
3625
3626 const AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3627
3628 // If the stack arguments for this call do not fit into our own save area then
3629 // the call cannot be made tail.
3630 if (CCInfo.getNextStackOffset() > FuncInfo->getBytesInStackArgArea())
3631 return false;
3632
3633 const MachineRegisterInfo &MRI = MF.getRegInfo();
3634 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
3635 return false;
3636
3637 return true;
3638}
3639
3640SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain,
3641 SelectionDAG &DAG,
3642 MachineFrameInfo &MFI,
3643 int ClobberedFI) const {
3644 SmallVector<SDValue, 8> ArgChains;
3645 int64_t FirstByte = MFI.getObjectOffset(ClobberedFI);
3646 int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1;
3647
3648 // Include the original chain at the beginning of the list. When this is
3649 // used by target LowerCall hooks, this helps legalize find the
3650 // CALLSEQ_BEGIN node.
3651 ArgChains.push_back(Chain);
3652
3653 // Add a chain value for each stack argument corresponding
3654 for (SDNode::use_iterator U = DAG.getEntryNode().getNode()->use_begin(),
3655 UE = DAG.getEntryNode().getNode()->use_end();
3656 U != UE; ++U)
3657 if (LoadSDNode *L = dyn_cast<LoadSDNode>(*U))
3658 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
3659 if (FI->getIndex() < 0) {
3660 int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex());
3661 int64_t InLastByte = InFirstByte;
3662 InLastByte += MFI.getObjectSize(FI->getIndex()) - 1;
3663
3664 if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) ||
3665 (FirstByte <= InFirstByte && InFirstByte <= LastByte))
3666 ArgChains.push_back(SDValue(L, 1));
3667 }
3668
3669 // Build a tokenfactor for all the chains.
3670 return DAG.getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
3671}
3672
3673bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC,
3674 bool TailCallOpt) const {
3675 return CallCC == CallingConv::Fast && TailCallOpt;
3676}
3677
3678/// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain,
3679/// and add input and output parameter nodes.
3680SDValue
3681AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI,
3682 SmallVectorImpl<SDValue> &InVals) const {
3683 SelectionDAG &DAG = CLI.DAG;
3684 SDLoc &DL = CLI.DL;
3685 SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs;
3686 SmallVector<SDValue, 32> &OutVals = CLI.OutVals;
3687 SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins;
3688 SDValue Chain = CLI.Chain;
3689 SDValue Callee = CLI.Callee;
3690 bool &IsTailCall = CLI.IsTailCall;
3691 CallingConv::ID CallConv = CLI.CallConv;
3692 bool IsVarArg = CLI.IsVarArg;
3693
3694 MachineFunction &MF = DAG.getMachineFunction();
3695 MachineFunction::CallSiteInfo CSInfo;
3696 bool IsThisReturn = false;
3697
3698 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
3699 bool TailCallOpt = MF.getTarget().Options.GuaranteedTailCallOpt;
3700 bool IsSibCall = false;
3701
3702 if (IsTailCall) {
3703 // Check if it's really possible to do a tail call.
3704 IsTailCall = isEligibleForTailCallOptimization(
3705 Callee, CallConv, IsVarArg, Outs, OutVals, Ins, DAG);
3706 if (!IsTailCall && CLI.CS && CLI.CS.isMustTailCall())
3707 report_fatal_error("failed to perform tail call elimination on a call "
3708 "site marked musttail");
3709
3710 // A sibling call is one where we're under the usual C ABI and not planning
3711 // to change that but can still do a tail call:
3712 if (!TailCallOpt && IsTailCall)
3713 IsSibCall = true;
3714
3715 if (IsTailCall)
3716 ++NumTailCalls;
3717 }
3718
3719 // Analyze operands of the call, assigning locations to each operand.
3720 SmallVector<CCValAssign, 16> ArgLocs;
3721 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), ArgLocs,
3722 *DAG.getContext());
3723
3724 if (IsVarArg) {
3725 // Handle fixed and variable vector arguments differently.
3726 // Variable vector arguments always go into memory.
3727 unsigned NumArgs = Outs.size();
3728
3729 for (unsigned i = 0; i != NumArgs; ++i) {
3730 MVT ArgVT = Outs[i].VT;
3731 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3732 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv,
3733 /*IsVarArg=*/ !Outs[i].IsFixed);
3734 bool Res = AssignFn(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
3735 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3735, __PRETTY_FUNCTION__))
;
3736 (void)Res;
3737 }
3738 } else {
3739 // At this point, Outs[].VT may already be promoted to i32. To correctly
3740 // handle passing i8 as i8 instead of i32 on stack, we pass in both i32 and
3741 // i8 to CC_AArch64_AAPCS with i32 being ValVT and i8 being LocVT.
3742 // Since AnalyzeCallOperands uses Ins[].VT for both ValVT and LocVT, here
3743 // we use a special version of AnalyzeCallOperands to pass in ValVT and
3744 // LocVT.
3745 unsigned NumArgs = Outs.size();
3746 for (unsigned i = 0; i != NumArgs; ++i) {
3747 MVT ValVT = Outs[i].VT;
3748 // Get type of the original argument.
3749 EVT ActualVT = getValueType(DAG.getDataLayout(),
3750 CLI.getArgs()[Outs[i].OrigArgIndex].Ty,
3751 /*AllowUnknown*/ true);
3752 MVT ActualMVT = ActualVT.isSimple() ? ActualVT.getSimpleVT() : ValVT;
3753 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
3754 // If ActualMVT is i1/i8/i16, we should set LocVT to i8/i8/i16.
3755 if (ActualMVT == MVT::i1 || ActualMVT == MVT::i8)
3756 ValVT = MVT::i8;
3757 else if (ActualMVT == MVT::i16)
3758 ValVT = MVT::i16;
3759
3760 CCAssignFn *AssignFn = CCAssignFnForCall(CallConv, /*IsVarArg=*/false);
3761 bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full, ArgFlags, CCInfo);
3762 assert(!Res && "Call operand has unhandled type")((!Res && "Call operand has unhandled type") ? static_cast
<void> (0) : __assert_fail ("!Res && \"Call operand has unhandled type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3762, __PRETTY_FUNCTION__))
;
3763 (void)Res;
3764 }
3765 }
3766
3767 // Get a count of how many bytes are to be pushed on the stack.
3768 unsigned NumBytes = CCInfo.getNextStackOffset();
3769
3770 if (IsSibCall) {
3771 // Since we're not changing the ABI to make this a tail call, the memory
3772 // operands are already available in the caller's incoming argument space.
3773 NumBytes = 0;
3774 }
3775
3776 // FPDiff is the byte offset of the call's argument area from the callee's.
3777 // Stores to callee stack arguments will be placed in FixedStackSlots offset
3778 // by this amount for a tail call. In a sibling call it must be 0 because the
3779 // caller will deallocate the entire stack and the callee still expects its
3780 // arguments to begin at SP+0. Completely unused for non-tail calls.
3781 int FPDiff = 0;
3782
3783 if (IsTailCall && !IsSibCall) {
3784 unsigned NumReusableBytes = FuncInfo->getBytesInStackArgArea();
3785
3786 // Since callee will pop argument stack as a tail call, we must keep the
3787 // popped size 16-byte aligned.
3788 NumBytes = alignTo(NumBytes, 16);
3789
3790 // FPDiff will be negative if this tail call requires more space than we
3791 // would automatically have in our incoming argument space. Positive if we
3792 // can actually shrink the stack.
3793 FPDiff = NumReusableBytes - NumBytes;
3794
3795 // The stack pointer must be 16-byte aligned at all times it's used for a
3796 // memory operation, which in practice means at *all* times and in
3797 // particular across call boundaries. Therefore our own arguments started at
3798 // a 16-byte aligned SP and the delta applied for the tail call should
3799 // satisfy the same constraint.
3800 assert(FPDiff % 16 == 0 && "unaligned stack on tail call")((FPDiff % 16 == 0 && "unaligned stack on tail call")
? static_cast<void> (0) : __assert_fail ("FPDiff % 16 == 0 && \"unaligned stack on tail call\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3800, __PRETTY_FUNCTION__))
;
3801 }
3802
3803 // Adjust the stack pointer for the new arguments...
3804 // These operations are automatically eliminated by the prolog/epilog pass
3805 if (!IsSibCall)
3806 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, DL);
3807
3808 SDValue StackPtr = DAG.getCopyFromReg(Chain, DL, AArch64::SP,
3809 getPointerTy(DAG.getDataLayout()));
3810
3811 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3812 SmallSet<unsigned, 8> RegsUsed;
3813 SmallVector<SDValue, 8> MemOpChains;
3814 auto PtrVT = getPointerTy(DAG.getDataLayout());
3815
3816 if (IsVarArg && CLI.CS && CLI.CS.isMustTailCall()) {
3817 const auto &Forwards = FuncInfo->getForwardedMustTailRegParms();
3818 for (const auto &F : Forwards) {
3819 SDValue Val = DAG.getCopyFromReg(Chain, DL, F.VReg, F.VT);
3820 RegsToPass.emplace_back(F.PReg, Val);
3821 }
3822 }
3823
3824 // Walk the register/memloc assignments, inserting copies/loads.
3825 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e;
3826 ++i, ++realArgIdx) {
3827 CCValAssign &VA = ArgLocs[i];
3828 SDValue Arg = OutVals[realArgIdx];
3829 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
3830
3831 // Promote the value if needed.
3832 switch (VA.getLocInfo()) {
3833 default:
3834 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3834)
;
3835 case CCValAssign::Full:
3836 break;
3837 case CCValAssign::SExt:
3838 Arg = DAG.getNode(ISD::SIGN_EXTEND, DL, VA.getLocVT(), Arg);
3839 break;
3840 case CCValAssign::ZExt:
3841 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
3842 break;
3843 case CCValAssign::AExt:
3844 if (Outs[realArgIdx].ArgVT == MVT::i1) {
3845 // AAPCS requires i1 to be zero-extended to 8-bits by the caller.
3846 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
3847 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i8, Arg);
3848 }
3849 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3850 break;
3851 case CCValAssign::AExtUpper:
3852 assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits")((VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::i32 && \"only expect 32 -> 64 upper bits\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3852, __PRETTY_FUNCTION__))
;
3853 Arg = DAG.getNode(ISD::ANY_EXTEND, DL, VA.getLocVT(), Arg);
3854 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
3855 DAG.getConstant(32, DL, VA.getLocVT()));
3856 break;
3857 case CCValAssign::BCvt:
3858 Arg = DAG.getBitcast(VA.getLocVT(), Arg);
3859 break;
3860 case CCValAssign::Trunc:
3861 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
3862 break;
3863 case CCValAssign::FPExt:
3864 Arg = DAG.getNode(ISD::FP_EXTEND, DL, VA.getLocVT(), Arg);
3865 break;
3866 case CCValAssign::Indirect:
3867 assert(VA.getValVT().isScalableVector() &&((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3868, __PRETTY_FUNCTION__))
3868 "Only scalable vectors can be passed indirectly")((VA.getValVT().isScalableVector() && "Only scalable vectors can be passed indirectly"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT().isScalableVector() && \"Only scalable vectors can be passed indirectly\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3868, __PRETTY_FUNCTION__))
;
3869 llvm_unreachable("Spilling of SVE vectors not yet implemented")::llvm::llvm_unreachable_internal("Spilling of SVE vectors not yet implemented"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3869)
;
3870 }
3871
3872 if (VA.isRegLoc()) {
3873 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
3874 Outs[0].VT == MVT::i64) {
3875 assert(VA.getLocVT() == MVT::i64 &&((VA.getLocVT() == MVT::i64 && "unexpected calling convention register assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocVT() == MVT::i64 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3876, __PRETTY_FUNCTION__))
3876 "unexpected calling convention register assignment")((VA.getLocVT() == MVT::i64 && "unexpected calling convention register assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocVT() == MVT::i64 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3876, __PRETTY_FUNCTION__))
;
3877 assert(!Ins.empty() && Ins[0].VT == MVT::i64 &&((!Ins.empty() && Ins[0].VT == MVT::i64 && "unexpected use of 'returned'"
) ? static_cast<void> (0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i64 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3878, __PRETTY_FUNCTION__))
3878 "unexpected use of 'returned'")((!Ins.empty() && Ins[0].VT == MVT::i64 && "unexpected use of 'returned'"
) ? static_cast<void> (0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i64 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3878, __PRETTY_FUNCTION__))
;
3879 IsThisReturn = true;
3880 }
3881 if (RegsUsed.count(VA.getLocReg())) {
3882 // If this register has already been used then we're trying to pack
3883 // parts of an [N x i32] into an X-register. The extension type will
3884 // take care of putting the two halves in the right place but we have to
3885 // combine them.
3886 SDValue &Bits =
3887 std::find_if(RegsToPass.begin(), RegsToPass.end(),
3888 [=](const std::pair<unsigned, SDValue> &Elt) {
3889 return Elt.first == VA.getLocReg();
3890 })
3891 ->second;
3892 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
3893 // Call site info is used for function's parameter entry value
3894 // tracking. For now we track only simple cases when parameter
3895 // is transferred through whole register.
3896 CSInfo.erase(std::remove_if(CSInfo.begin(), CSInfo.end(),
3897 [&VA](MachineFunction::ArgRegPair ArgReg) {
3898 return ArgReg.Reg == VA.getLocReg();
3899 }),
3900 CSInfo.end());
3901 } else {
3902 RegsToPass.emplace_back(VA.getLocReg(), Arg);
3903 RegsUsed.insert(VA.getLocReg());
3904 const TargetOptions &Options = DAG.getTarget().Options;
3905 if (Options.EnableDebugEntryValues)
3906 CSInfo.emplace_back(VA.getLocReg(), i);
3907 }
3908 } else {
3909 assert(VA.isMemLoc())((VA.isMemLoc()) ? static_cast<void> (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3909, __PRETTY_FUNCTION__))
;
3910
3911 SDValue DstAddr;
3912 MachinePointerInfo DstInfo;
3913
3914 // FIXME: This works on big-endian for composite byvals, which are the
3915 // common case. It should also work for fundamental types too.
3916 uint32_t BEAlign = 0;
3917 unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() * 8
3918 : VA.getValVT().getSizeInBits();
3919 OpSize = (OpSize + 7) / 8;
3920 if (!Subtarget->isLittleEndian() && !Flags.isByVal() &&
3921 !Flags.isInConsecutiveRegs()) {
3922 if (OpSize < 8)
3923 BEAlign = 8 - OpSize;
3924 }
3925 unsigned LocMemOffset = VA.getLocMemOffset();
3926 int32_t Offset = LocMemOffset + BEAlign;
3927 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3928 PtrOff = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
3929
3930 if (IsTailCall) {
3931 Offset = Offset + FPDiff;
3932 int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
3933
3934 DstAddr = DAG.getFrameIndex(FI, PtrVT);
3935 DstInfo =
3936 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
3937
3938 // Make sure any stack arguments overlapping with where we're storing
3939 // are loaded before this eventual operation. Otherwise they'll be
3940 // clobbered.
3941 Chain = addTokenForArgument(Chain, DAG, MF.getFrameInfo(), FI);
3942 } else {
3943 SDValue PtrOff = DAG.getIntPtrConstant(Offset, DL);
3944
3945 DstAddr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, PtrOff);
3946 DstInfo = MachinePointerInfo::getStack(DAG.getMachineFunction(),
3947 LocMemOffset);
3948 }
3949
3950 if (Outs[i].Flags.isByVal()) {
3951 SDValue SizeNode =
3952 DAG.getConstant(Outs[i].Flags.getByValSize(), DL, MVT::i64);
3953 SDValue Cpy = DAG.getMemcpy(
3954 Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(),
3955 /*isVol = */ false, /*AlwaysInline = */ false,
3956 /*isTailCall = */ false,
3957 DstInfo, MachinePointerInfo());
3958
3959 MemOpChains.push_back(Cpy);
3960 } else {
3961 // Since we pass i1/i8/i16 as i1/i8/i16 on stack and Arg is already
3962 // promoted to a legal register type i32, we should truncate Arg back to
3963 // i1/i8/i16.
3964 if (VA.getValVT() == MVT::i1 || VA.getValVT() == MVT::i8 ||
3965 VA.getValVT() == MVT::i16)
3966 Arg = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), Arg);
3967
3968 SDValue Store = DAG.getStore(Chain, DL, Arg, DstAddr, DstInfo);
3969 MemOpChains.push_back(Store);
3970 }
3971 }
3972 }
3973
3974 if (!MemOpChains.empty())
3975 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
3976
3977 // Build a sequence of copy-to-reg nodes chained together with token chain
3978 // and flag operands which copy the outgoing args into the appropriate regs.
3979 SDValue InFlag;
3980 for (auto &RegToPass : RegsToPass) {
3981 Chain = DAG.getCopyToReg(Chain, DL, RegToPass.first,
3982 RegToPass.second, InFlag);
3983 InFlag = Chain.getValue(1);
3984 }
3985
3986 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
3987 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
3988 // node so that legalize doesn't hack it.
3989 if (auto *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
3990 auto GV = G->getGlobal();
3991 if (Subtarget->classifyGlobalFunctionReference(GV, getTargetMachine()) ==
3992 AArch64II::MO_GOT) {
3993 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_GOT);
3994 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
3995 } else if (Subtarget->isTargetCOFF() && GV->hasDLLImportStorageClass()) {
3996 assert(Subtarget->isTargetWindows() &&((Subtarget->isTargetWindows() && "Windows is the only supported COFF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3997, __PRETTY_FUNCTION__))
3997 "Windows is the only supported COFF target")((Subtarget->isTargetWindows() && "Windows is the only supported COFF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 3997, __PRETTY_FUNCTION__))
;
3998 Callee = getGOT(G, DAG, AArch64II::MO_DLLIMPORT);
3999 } else {
4000 const GlobalValue *GV = G->getGlobal();
4001 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, 0);
4002 }
4003 } else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
4004 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
4005 Subtarget->isTargetMachO()) {
4006 const char *Sym = S->getSymbol();
4007 Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, AArch64II::MO_GOT);
4008 Callee = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, Callee);
4009 } else {
4010 const char *Sym = S->getSymbol();
4011 Callee = DAG.getTargetExternalSymbol(Sym, PtrVT, 0);
4012 }
4013 }
4014
4015 // We don't usually want to end the call-sequence here because we would tidy
4016 // the frame up *after* the call, however in the ABI-changing tail-call case
4017 // we've carefully laid out the parameters so that when sp is reset they'll be
4018 // in the correct location.
4019 if (IsTailCall && !IsSibCall) {
4020 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4021 DAG.getIntPtrConstant(0, DL, true), InFlag, DL);
4022 InFlag = Chain.getValue(1);
4023 }
4024
4025 std::vector<SDValue> Ops;
4026 Ops.push_back(Chain);
4027 Ops.push_back(Callee);
4028
4029 if (IsTailCall) {
4030 // Each tail call may have to adjust the stack by a different amount, so
4031 // this information must travel along with the operation for eventual
4032 // consumption by emitEpilogue.
4033 Ops.push_back(DAG.getTargetConstant(FPDiff, DL, MVT::i32));
4034 }
4035
4036 // Add argument registers to the end of the list so that they are known live
4037 // into the call.
4038 for (auto &RegToPass : RegsToPass)
4039 Ops.push_back(DAG.getRegister(RegToPass.first,
4040 RegToPass.second.getValueType()));
4041
4042 // Check callee args/returns for SVE registers and set calling convention
4043 // accordingly.
4044 if (CallConv == CallingConv::C) {
4045 bool CalleeOutSVE = any_of(Outs, [](ISD::OutputArg &Out){
4046 return Out.VT.isScalableVector();
4047 });
4048 bool CalleeInSVE = any_of(Ins, [](ISD::InputArg &In){
4049 return In.VT.isScalableVector();
4050 });
4051
4052 if (CalleeInSVE || CalleeOutSVE)
4053 CallConv = CallingConv::AArch64_SVE_VectorCall;
4054 }
4055
4056 // Add a register mask operand representing the call-preserved registers.
4057 const uint32_t *Mask;
4058 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4059 if (IsThisReturn) {
4060 // For 'this' returns, use the X0-preserving mask if applicable
4061 Mask = TRI->getThisReturnPreservedMask(MF, CallConv);
4062 if (!Mask) {
4063 IsThisReturn = false;
4064 Mask = TRI->getCallPreservedMask(MF, CallConv);
4065 }
4066 } else
4067 Mask = TRI->getCallPreservedMask(MF, CallConv);
4068
4069 if (Subtarget->hasCustomCallingConv())
4070 TRI->UpdateCustomCallPreservedMask(MF, &Mask);
4071
4072 if (TRI->isAnyArgRegReserved(MF))
4073 TRI->emitReservedArgRegCallError(MF);
4074
4075 assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention"
) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4075, __PRETTY_FUNCTION__))
;
4076 Ops.push_back(DAG.getRegisterMask(Mask));
4077
4078 if (InFlag.getNode())
4079 Ops.push_back(InFlag);
4080
4081 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4082
4083 // If we're doing a tall call, use a TC_RETURN here rather than an
4084 // actual call instruction.
4085 if (IsTailCall) {
4086 MF.getFrameInfo().setHasTailCall();
4087 SDValue Ret = DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops);
4088 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
4089 return Ret;
4090 }
4091
4092 // Returns a chain and a flag for retval copy to use.
4093 Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops);
4094 InFlag = Chain.getValue(1);
4095 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
4096
4097 uint64_t CalleePopBytes =
4098 DoesCalleeRestoreStack(CallConv, TailCallOpt) ? alignTo(NumBytes, 16) : 0;
4099
4100 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, DL, true),
4101 DAG.getIntPtrConstant(CalleePopBytes, DL, true),
4102 InFlag, DL);
4103 if (!Ins.empty())
4104 InFlag = Chain.getValue(1);
4105
4106 // Handle result values, copying them out of physregs into vregs that we
4107 // return.
4108 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, DL, DAG,
4109 InVals, IsThisReturn,
4110 IsThisReturn ? OutVals[0] : SDValue());
4111}
4112
4113bool AArch64TargetLowering::CanLowerReturn(
4114 CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
4115 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
4116 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
4117 ? RetCC_AArch64_WebKit_JS
4118 : RetCC_AArch64_AAPCS;
4119 SmallVector<CCValAssign, 16> RVLocs;
4120 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
4121 return CCInfo.CheckReturn(Outs, RetCC);
4122}
4123
4124SDValue
4125AArch64TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
4126 bool isVarArg,
4127 const SmallVectorImpl<ISD::OutputArg> &Outs,
4128 const SmallVectorImpl<SDValue> &OutVals,
4129 const SDLoc &DL, SelectionDAG &DAG) const {
4130 auto &MF = DAG.getMachineFunction();
4131 auto *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
4132
4133 CCAssignFn *RetCC = CallConv == CallingConv::WebKit_JS
4134 ? RetCC_AArch64_WebKit_JS
4135 : RetCC_AArch64_AAPCS;
4136 SmallVector<CCValAssign, 16> RVLocs;
4137 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
4138 *DAG.getContext());
4139 CCInfo.AnalyzeReturn(Outs, RetCC);
4140
4141 // Copy the result values into the output registers.
4142 SDValue Flag;
4143 SmallVector<std::pair<unsigned, SDValue>, 4> RetVals;
4144 SmallSet<unsigned, 4> RegsUsed;
4145 for (unsigned i = 0, realRVLocIdx = 0; i != RVLocs.size();
4146 ++i, ++realRVLocIdx) {
4147 CCValAssign &VA = RVLocs[i];
4148 assert(VA.isRegLoc() && "Can only return in registers!")((VA.isRegLoc() && "Can only return in registers!") ?
static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4148, __PRETTY_FUNCTION__))
;
4149 SDValue Arg = OutVals[realRVLocIdx];
4150
4151 switch (VA.getLocInfo()) {
4152 default:
4153 llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4153)
;
4154 case CCValAssign::Full:
4155 if (Outs[i].ArgVT == MVT::i1) {
4156 // AAPCS requires i1 to be zero-extended to i8 by the producer of the
4157 // value. This is strictly redundant on Darwin (which uses "zeroext
4158 // i1"), but will be optimised out before ISel.
4159 Arg = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, Arg);
4160 Arg = DAG.getNode(ISD::ZERO_EXTEND, DL, VA.getLocVT(), Arg);
4161 }
4162 break;
4163 case CCValAssign::BCvt:
4164 Arg = DAG.getNode(ISD::BITCAST, DL, VA.getLocVT(), Arg);
4165 break;
4166 case CCValAssign::AExt:
4167 case CCValAssign::ZExt:
4168 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4169 break;
4170 case CCValAssign::AExtUpper:
4171 assert(VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits")((VA.getValVT() == MVT::i32 && "only expect 32 -> 64 upper bits"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::i32 && \"only expect 32 -> 64 upper bits\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4171, __PRETTY_FUNCTION__))
;
4172 Arg = DAG.getZExtOrTrunc(Arg, DL, VA.getLocVT());
4173 Arg = DAG.getNode(ISD::SHL, DL, VA.getLocVT(), Arg,
4174 DAG.getConstant(32, DL, VA.getLocVT()));
4175 break;
4176 }
4177
4178 if (RegsUsed.count(VA.getLocReg())) {
4179 SDValue &Bits =
4180 std::find_if(RetVals.begin(), RetVals.end(),
4181 [=](const std::pair<unsigned, SDValue> &Elt) {
4182 return Elt.first == VA.getLocReg();
4183 })
4184 ->second;
4185 Bits = DAG.getNode(ISD::OR, DL, Bits.getValueType(), Bits, Arg);
4186 } else {
4187 RetVals.emplace_back(VA.getLocReg(), Arg);
4188 RegsUsed.insert(VA.getLocReg());
4189 }
4190 }
4191
4192 SmallVector<SDValue, 4> RetOps(1, Chain);
4193 for (auto &RetVal : RetVals) {
4194 Chain = DAG.getCopyToReg(Chain, DL, RetVal.first, RetVal.second, Flag);
4195 Flag = Chain.getValue(1);
4196 RetOps.push_back(
4197 DAG.getRegister(RetVal.first, RetVal.second.getValueType()));
4198 }
4199
4200 // Windows AArch64 ABIs require that for returning structs by value we copy
4201 // the sret argument into X0 for the return.
4202 // We saved the argument into a virtual register in the entry block,
4203 // so now we copy the value out and into X0.
4204 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
4205 SDValue Val = DAG.getCopyFromReg(RetOps[0], DL, SRetReg,
4206 getPointerTy(MF.getDataLayout()));
4207
4208 unsigned RetValReg = AArch64::X0;
4209 Chain = DAG.getCopyToReg(Chain, DL, RetValReg, Val, Flag);
4210 Flag = Chain.getValue(1);
4211
4212 RetOps.push_back(
4213 DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
4214 }
4215
4216 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4217 const MCPhysReg *I =
4218 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
4219 if (I) {
4220 for (; *I; ++I) {
4221 if (AArch64::GPR64RegClass.contains(*I))
4222 RetOps.push_back(DAG.getRegister(*I, MVT::i64));
4223 else if (AArch64::FPR64RegClass.contains(*I))
4224 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
4225 else
4226 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4226)
;
4227 }
4228 }
4229
4230 RetOps[0] = Chain; // Update chain.
4231
4232 // Add the flag if we have it.
4233 if (Flag.getNode())
4234 RetOps.push_back(Flag);
4235
4236 return DAG.getNode(AArch64ISD::RET_FLAG, DL, MVT::Other, RetOps);
4237}
4238
4239//===----------------------------------------------------------------------===//
4240// Other Lowering Code
4241//===----------------------------------------------------------------------===//
4242
4243SDValue AArch64TargetLowering::getTargetNode(GlobalAddressSDNode *N, EVT Ty,
4244 SelectionDAG &DAG,
4245 unsigned Flag) const {
4246 return DAG.getTargetGlobalAddress(N->getGlobal(), SDLoc(N), Ty,
4247 N->getOffset(), Flag);
4248}
4249
4250SDValue AArch64TargetLowering::getTargetNode(JumpTableSDNode *N, EVT Ty,
4251 SelectionDAG &DAG,
4252 unsigned Flag) const {
4253 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flag);
4254}
4255
4256SDValue AArch64TargetLowering::getTargetNode(ConstantPoolSDNode *N, EVT Ty,
4257 SelectionDAG &DAG,
4258 unsigned Flag) const {
4259 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
4260 N->getOffset(), Flag);
4261}
4262
4263SDValue AArch64TargetLowering::getTargetNode(BlockAddressSDNode* N, EVT Ty,
4264 SelectionDAG &DAG,
4265 unsigned Flag) const {
4266 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, 0, Flag);
4267}
4268
4269// (loadGOT sym)
4270template <class NodeTy>
4271SDValue AArch64TargetLowering::getGOT(NodeTy *N, SelectionDAG &DAG,
4272 unsigned Flags) const {
4273 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getGOT\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getGOT\n"
; } } while (false)
;
4274 SDLoc DL(N);
4275 EVT Ty = getPointerTy(DAG.getDataLayout());
4276 SDValue GotAddr = getTargetNode(N, Ty, DAG, AArch64II::MO_GOT | Flags);
4277 // FIXME: Once remat is capable of dealing with instructions with register
4278 // operands, expand this into two nodes instead of using a wrapper node.
4279 return DAG.getNode(AArch64ISD::LOADgot, DL, Ty, GotAddr);
4280}
4281
4282// (wrapper %highest(sym), %higher(sym), %hi(sym), %lo(sym))
4283template <class NodeTy>
4284SDValue AArch64TargetLowering::getAddrLarge(NodeTy *N, SelectionDAG &DAG,
4285 unsigned Flags) const {
4286 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrLarge\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddrLarge\n"
; } } while (false)
;
4287 SDLoc DL(N);
4288 EVT Ty = getPointerTy(DAG.getDataLayout());
4289 const unsigned char MO_NC = AArch64II::MO_NC;
4290 return DAG.getNode(
4291 AArch64ISD::WrapperLarge, DL, Ty,
4292 getTargetNode(N, Ty, DAG, AArch64II::MO_G3 | Flags),
4293 getTargetNode(N, Ty, DAG, AArch64II::MO_G2 | MO_NC | Flags),
4294 getTargetNode(N, Ty, DAG, AArch64II::MO_G1 | MO_NC | Flags),
4295 getTargetNode(N, Ty, DAG, AArch64II::MO_G0 | MO_NC | Flags));
4296}
4297
4298// (addlow (adrp %hi(sym)) %lo(sym))
4299template <class NodeTy>
4300SDValue AArch64TargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4301 unsigned Flags) const {
4302 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddr\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddr\n"
; } } while (false)
;
4303 SDLoc DL(N);
4304 EVT Ty = getPointerTy(DAG.getDataLayout());
4305 SDValue Hi = getTargetNode(N, Ty, DAG, AArch64II::MO_PAGE | Flags);
4306 SDValue Lo = getTargetNode(N, Ty, DAG,
4307 AArch64II::MO_PAGEOFF | AArch64II::MO_NC | Flags);
4308 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, Ty, Hi);
4309 return DAG.getNode(AArch64ISD::ADDlow, DL, Ty, ADRP, Lo);
4310}
4311
4312// (adr sym)
4313template <class NodeTy>
4314SDValue AArch64TargetLowering::getAddrTiny(NodeTy *N, SelectionDAG &DAG,
4315 unsigned Flags) const {
4316 LLVM_DEBUG(dbgs() << "AArch64TargetLowering::getAddrTiny\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << "AArch64TargetLowering::getAddrTiny\n"
; } } while (false)
;
4317 SDLoc DL(N);
4318 EVT Ty = getPointerTy(DAG.getDataLayout());
4319 SDValue Sym = getTargetNode(N, Ty, DAG, Flags);
4320 return DAG.getNode(AArch64ISD::ADR, DL, Ty, Sym);
4321}
4322
4323SDValue AArch64TargetLowering::LowerGlobalAddress(SDValue Op,
4324 SelectionDAG &DAG) const {
4325 GlobalAddressSDNode *GN = cast<GlobalAddressSDNode>(Op);
4326 const GlobalValue *GV = GN->getGlobal();
4327 unsigned OpFlags = Subtarget->ClassifyGlobalReference(GV, getTargetMachine());
4328
4329 if (OpFlags != AArch64II::MO_NO_FLAG)
4330 assert(cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&((cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
"unexpected offset in global node") ? static_cast<void>
(0) : __assert_fail ("cast<GlobalAddressSDNode>(Op)->getOffset() == 0 && \"unexpected offset in global node\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4331, __PRETTY_FUNCTION__))
4331 "unexpected offset in global node")((cast<GlobalAddressSDNode>(Op)->getOffset() == 0 &&
"unexpected offset in global node") ? static_cast<void>
(0) : __assert_fail ("cast<GlobalAddressSDNode>(Op)->getOffset() == 0 && \"unexpected offset in global node\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4331, __PRETTY_FUNCTION__))
;
4332
4333 // This also catches the large code model case for Darwin, and tiny code
4334 // model with got relocations.
4335 if ((OpFlags & AArch64II::MO_GOT) != 0) {
4336 return getGOT(GN, DAG, OpFlags);
4337 }
4338
4339 SDValue Result;
4340 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
4341 Result = getAddrLarge(GN, DAG, OpFlags);
4342 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
4343 Result = getAddrTiny(GN, DAG, OpFlags);
4344 } else {
4345 Result = getAddr(GN, DAG, OpFlags);
4346 }
4347 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4348 SDLoc DL(GN);
4349 if (OpFlags & (AArch64II::MO_DLLIMPORT | AArch64II::MO_COFFSTUB))
4350 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
4351 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
4352 return Result;
4353}
4354
4355/// Convert a TLS address reference into the correct sequence of loads
4356/// and calls to compute the variable's address (for Darwin, currently) and
4357/// return an SDValue containing the final node.
4358
4359/// Darwin only has one TLS scheme which must be capable of dealing with the
4360/// fully general situation, in the worst case. This means:
4361/// + "extern __thread" declaration.
4362/// + Defined in a possibly unknown dynamic library.
4363///
4364/// The general system is that each __thread variable has a [3 x i64] descriptor
4365/// which contains information used by the runtime to calculate the address. The
4366/// only part of this the compiler needs to know about is the first xword, which
4367/// contains a function pointer that must be called with the address of the
4368/// entire descriptor in "x0".
4369///
4370/// Since this descriptor may be in a different unit, in general even the
4371/// descriptor must be accessed via an indirect load. The "ideal" code sequence
4372/// is:
4373/// adrp x0, _var@TLVPPAGE
4374/// ldr x0, [x0, _var@TLVPPAGEOFF] ; x0 now contains address of descriptor
4375/// ldr x1, [x0] ; x1 contains 1st entry of descriptor,
4376/// ; the function pointer
4377/// blr x1 ; Uses descriptor address in x0
4378/// ; Address of _var is now in x0.
4379///
4380/// If the address of _var's descriptor *is* known to the linker, then it can
4381/// change the first "ldr" instruction to an appropriate "add x0, x0, #imm" for
4382/// a slight efficiency gain.
4383SDValue
4384AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op,
4385 SelectionDAG &DAG) const {
4386 assert(Subtarget->isTargetDarwin() &&((Subtarget->isTargetDarwin() && "This function expects a Darwin target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4387, __PRETTY_FUNCTION__))
4387 "This function expects a Darwin target")((Subtarget->isTargetDarwin() && "This function expects a Darwin target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4387, __PRETTY_FUNCTION__))
;
4388
4389 SDLoc DL(Op);
4390 MVT PtrVT = getPointerTy(DAG.getDataLayout());
4391 MVT PtrMemVT = getPointerMemTy(DAG.getDataLayout());
4392 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
4393
4394 SDValue TLVPAddr =
4395 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4396 SDValue DescAddr = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TLVPAddr);
4397
4398 // The first entry in the descriptor is a function pointer that we must call
4399 // to obtain the address of the variable.
4400 SDValue Chain = DAG.getEntryNode();
4401 SDValue FuncTLVGet = DAG.getLoad(
4402 PtrMemVT, DL, Chain, DescAddr,
4403 MachinePointerInfo::getGOT(DAG.getMachineFunction()),
4404 /* Alignment = */ PtrMemVT.getSizeInBits() / 8,
4405 MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
4406 Chain = FuncTLVGet.getValue(1);
4407
4408 // Extend loaded pointer if necessary (i.e. if ILP32) to DAG pointer.
4409 FuncTLVGet = DAG.getZExtOrTrunc(FuncTLVGet, DL, PtrVT);
4410
4411 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
4412 MFI.setAdjustsStack(true);
4413
4414 // TLS calls preserve all registers except those that absolutely must be
4415 // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be
4416 // silly).
4417 const AArch64RegisterInfo *TRI = Subtarget->getRegisterInfo();
4418 const uint32_t *Mask = TRI->getTLSCallPreservedMask();
4419 if (Subtarget->hasCustomCallingConv())
4420 TRI->UpdateCustomCallPreservedMask(DAG.getMachineFunction(), &Mask);
4421
4422 // Finally, we can make the call. This is just a degenerate version of a
4423 // normal AArch64 call node: x0 takes the address of the descriptor, and
4424 // returns the address of the variable in this thread.
4425 Chain = DAG.getCopyToReg(Chain, DL, AArch64::X0, DescAddr, SDValue());
4426 Chain =
4427 DAG.getNode(AArch64ISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
4428 Chain, FuncTLVGet, DAG.getRegister(AArch64::X0, MVT::i64),
4429 DAG.getRegisterMask(Mask), Chain.getValue(1));
4430 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Chain.getValue(1));
4431}
4432
4433/// When accessing thread-local variables under either the general-dynamic or
4434/// local-dynamic system, we make a "TLS-descriptor" call. The variable will
4435/// have a descriptor, accessible via a PC-relative ADRP, and whose first entry
4436/// is a function pointer to carry out the resolution.
4437///
4438/// The sequence is:
4439/// adrp x0, :tlsdesc:var
4440/// ldr x1, [x0, #:tlsdesc_lo12:var]
4441/// add x0, x0, #:tlsdesc_lo12:var
4442/// .tlsdesccall var
4443/// blr x1
4444/// (TPIDR_EL0 offset now in x0)
4445///
4446/// The above sequence must be produced unscheduled, to enable the linker to
4447/// optimize/relax this sequence.
4448/// Therefore, a pseudo-instruction (TLSDESC_CALLSEQ) is used to represent the
4449/// above sequence, and expanded really late in the compilation flow, to ensure
4450/// the sequence is produced as per above.
4451SDValue AArch64TargetLowering::LowerELFTLSDescCallSeq(SDValue SymAddr,
4452 const SDLoc &DL,
4453 SelectionDAG &DAG) const {
4454 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4455
4456 SDValue Chain = DAG.getEntryNode();
4457 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4458
4459 Chain =
4460 DAG.getNode(AArch64ISD::TLSDESC_CALLSEQ, DL, NodeTys, {Chain, SymAddr});
4461 SDValue Glue = Chain.getValue(1);
4462
4463 return DAG.getCopyFromReg(Chain, DL, AArch64::X0, PtrVT, Glue);
4464}
4465
4466SDValue
4467AArch64TargetLowering::LowerELFGlobalTLSAddress(SDValue Op,
4468 SelectionDAG &DAG) const {
4469 assert(Subtarget->isTargetELF() && "This function expects an ELF target")((Subtarget->isTargetELF() && "This function expects an ELF target"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetELF() && \"This function expects an ELF target\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4469, __PRETTY_FUNCTION__))
;
4470 if (getTargetMachine().getCodeModel() == CodeModel::Large)
4471 report_fatal_error("ELF TLS only supported in small memory model");
4472 // Different choices can be made for the maximum size of the TLS area for a
4473 // module. For the small address model, the default TLS size is 16MiB and the
4474 // maximum TLS size is 4GiB.
4475 // FIXME: add -mtls-size command line option and make it control the 16MiB
4476 // vs. 4GiB code sequence generation.
4477 // FIXME: add tiny codemodel support. We currently generate the same code as
4478 // small, which may be larger than needed.
4479 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4480
4481 TLSModel::Model Model = getTargetMachine().getTLSModel(GA->getGlobal());
4482
4483 if (!EnableAArch64ELFLocalDynamicTLSGeneration) {
4484 if (Model == TLSModel::LocalDynamic)
4485 Model = TLSModel::GeneralDynamic;
4486 }
4487
4488 SDValue TPOff;
4489 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4490 SDLoc DL(Op);
4491 const GlobalValue *GV = GA->getGlobal();
4492
4493 SDValue ThreadBase = DAG.getNode(AArch64ISD::THREAD_POINTER, DL, PtrVT);
4494
4495 if (Model == TLSModel::LocalExec) {
4496 SDValue HiVar = DAG.getTargetGlobalAddress(
4497 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4498 SDValue LoVar = DAG.getTargetGlobalAddress(
4499 GV, DL, PtrVT, 0,
4500 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4501
4502 SDValue TPWithOff_lo =
4503 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, ThreadBase,
4504 HiVar,
4505 DAG.getTargetConstant(0, DL, MVT::i32)),
4506 0);
4507 SDValue TPWithOff =
4508 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPWithOff_lo,
4509 LoVar,
4510 DAG.getTargetConstant(0, DL, MVT::i32)),
4511 0);
4512 return TPWithOff;
4513 } else if (Model == TLSModel::InitialExec) {
4514 TPOff = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4515 TPOff = DAG.getNode(AArch64ISD::LOADgot, DL, PtrVT, TPOff);
4516 } else if (Model == TLSModel::LocalDynamic) {
4517 // Local-dynamic accesses proceed in two phases. A general-dynamic TLS
4518 // descriptor call against the special symbol _TLS_MODULE_BASE_ to calculate
4519 // the beginning of the module's TLS region, followed by a DTPREL offset
4520 // calculation.
4521
4522 // These accesses will need deduplicating if there's more than one.
4523 AArch64FunctionInfo *MFI =
4524 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
4525 MFI->incNumLocalDynamicTLSAccesses();
4526
4527 // The call needs a relocation too for linker relaxation. It doesn't make
4528 // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4529 // the address.
4530 SDValue SymAddr = DAG.getTargetExternalSymbol("_TLS_MODULE_BASE_", PtrVT,
4531 AArch64II::MO_TLS);
4532
4533 // Now we can calculate the offset from TPIDR_EL0 to this module's
4534 // thread-local area.
4535 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4536
4537 // Now use :dtprel_whatever: operations to calculate this variable's offset
4538 // in its thread-storage area.
4539 SDValue HiVar = DAG.getTargetGlobalAddress(
4540 GV, DL, MVT::i64, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4541 SDValue LoVar = DAG.getTargetGlobalAddress(
4542 GV, DL, MVT::i64, 0,
4543 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4544
4545 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, HiVar,
4546 DAG.getTargetConstant(0, DL, MVT::i32)),
4547 0);
4548 TPOff = SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TPOff, LoVar,
4549 DAG.getTargetConstant(0, DL, MVT::i32)),
4550 0);
4551 } else if (Model == TLSModel::GeneralDynamic) {
4552 // The call needs a relocation too for linker relaxation. It doesn't make
4553 // sense to call it MO_PAGE or MO_PAGEOFF though so we need another copy of
4554 // the address.
4555 SDValue SymAddr =
4556 DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, AArch64II::MO_TLS);
4557
4558 // Finally we can make a call to calculate the offset from tpidr_el0.
4559 TPOff = LowerELFTLSDescCallSeq(SymAddr, DL, DAG);
4560 } else
4561 llvm_unreachable("Unsupported ELF TLS access model")::llvm::llvm_unreachable_internal("Unsupported ELF TLS access model"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4561)
;
4562
4563 return DAG.getNode(ISD::ADD, DL, PtrVT, ThreadBase, TPOff);
4564}
4565
4566SDValue
4567AArch64TargetLowering::LowerWindowsGlobalTLSAddress(SDValue Op,
4568 SelectionDAG &DAG) const {
4569 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering")((Subtarget->isTargetWindows() && "Windows specific TLS lowering"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetWindows() && \"Windows specific TLS lowering\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4569, __PRETTY_FUNCTION__))
;
4570
4571 SDValue Chain = DAG.getEntryNode();
4572 EVT PtrVT = getPointerTy(DAG.getDataLayout());
4573 SDLoc DL(Op);
4574
4575 SDValue TEB = DAG.getRegister(AArch64::X18, MVT::i64);
4576
4577 // Load the ThreadLocalStoragePointer from the TEB
4578 // A pointer to the TLS array is located at offset 0x58 from the TEB.
4579 SDValue TLSArray =
4580 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x58, DL));
4581 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
4582 Chain = TLSArray.getValue(1);
4583
4584 // Load the TLS index from the C runtime;
4585 // This does the same as getAddr(), but without having a GlobalAddressSDNode.
4586 // This also does the same as LOADgot, but using a generic i32 load,
4587 // while LOADgot only loads i64.
4588 SDValue TLSIndexHi =
4589 DAG.getTargetExternalSymbol("_tls_index", PtrVT, AArch64II::MO_PAGE);
4590 SDValue TLSIndexLo = DAG.getTargetExternalSymbol(
4591 "_tls_index", PtrVT, AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4592 SDValue ADRP = DAG.getNode(AArch64ISD::ADRP, DL, PtrVT, TLSIndexHi);
4593 SDValue TLSIndex =
4594 DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, ADRP, TLSIndexLo);
4595 TLSIndex = DAG.getLoad(MVT::i32, DL, Chain, TLSIndex, MachinePointerInfo());
4596 Chain = TLSIndex.getValue(1);
4597
4598 // The pointer to the thread's TLS data area is at the TLS Index scaled by 8
4599 // offset into the TLSArray.
4600 TLSIndex = DAG.getNode(ISD::ZERO_EXTEND, DL, PtrVT, TLSIndex);
4601 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
4602 DAG.getConstant(3, DL, PtrVT));
4603 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
4604 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
4605 MachinePointerInfo());
4606 Chain = TLS.getValue(1);
4607
4608 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4609 const GlobalValue *GV = GA->getGlobal();
4610 SDValue TGAHi = DAG.getTargetGlobalAddress(
4611 GV, DL, PtrVT, 0, AArch64II::MO_TLS | AArch64II::MO_HI12);
4612 SDValue TGALo = DAG.getTargetGlobalAddress(
4613 GV, DL, PtrVT, 0,
4614 AArch64II::MO_TLS | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
4615
4616 // Add the offset from the start of the .tls section (section base).
4617 SDValue Addr =
4618 SDValue(DAG.getMachineNode(AArch64::ADDXri, DL, PtrVT, TLS, TGAHi,
4619 DAG.getTargetConstant(0, DL, MVT::i32)),
4620 0);
4621 Addr = DAG.getNode(AArch64ISD::ADDlow, DL, PtrVT, Addr, TGALo);
4622 return Addr;
4623}
4624
4625SDValue AArch64TargetLowering::LowerGlobalTLSAddress(SDValue Op,
4626 SelectionDAG &DAG) const {
4627 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
4628 if (DAG.getTarget().useEmulatedTLS())
4629 return LowerToTLSEmulatedModel(GA, DAG);
4630
4631 if (Subtarget->isTargetDarwin())
4632 return LowerDarwinGlobalTLSAddress(Op, DAG);
4633 if (Subtarget->isTargetELF())
4634 return LowerELFGlobalTLSAddress(Op, DAG);
4635 if (Subtarget->isTargetWindows())
4636 return LowerWindowsGlobalTLSAddress(Op, DAG);
4637
4638 llvm_unreachable("Unexpected platform trying to use TLS")::llvm::llvm_unreachable_internal("Unexpected platform trying to use TLS"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4638)
;
4639}
4640
4641SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
4642 SDValue Chain = Op.getOperand(0);
4643 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
4644 SDValue LHS = Op.getOperand(2);
4645 SDValue RHS = Op.getOperand(3);
4646 SDValue Dest = Op.getOperand(4);
4647 SDLoc dl(Op);
4648
4649 MachineFunction &MF = DAG.getMachineFunction();
4650 // Speculation tracking/SLH assumes that optimized TB(N)Z/CB(N)Z instructions
4651 // will not be produced, as they are conditional branch instructions that do
4652 // not set flags.
4653 bool ProduceNonFlagSettingCondBr =
4654 !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening);
4655
4656 // Handle f128 first, since lowering it will result in comparing the return
4657 // value of a libcall against zero, which is just what the rest of LowerBR_CC
4658 // is expecting to deal with.
4659 if (LHS.getValueType() == MVT::f128) {
4660 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
4661
4662 // If softenSetCCOperands returned a scalar, we need to compare the result
4663 // against zero to select between true and false values.
4664 if (!RHS.getNode()) {
4665 RHS = DAG.getConstant(0, dl, LHS.getValueType());
4666 CC = ISD::SETNE;
4667 }
4668 }
4669
4670 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
4671 // instruction.
4672 if (isOverflowIntrOpRes(LHS) && isOneConstant(RHS) &&
4673 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
4674 // Only lower legal XALUO ops.
4675 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
4676 return SDValue();
4677
4678 // The actual operation with overflow check.
4679 AArch64CC::CondCode OFCC;
4680 SDValue Value, Overflow;
4681 std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, LHS.getValue(0), DAG);
4682
4683 if (CC == ISD::SETNE)
4684 OFCC = getInvertedCondCode(OFCC);
4685 SDValue CCVal = DAG.getConstant(OFCC, dl, MVT::i32);
4686
4687 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4688 Overflow);
4689 }
4690
4691 if (LHS.getValueType().isInteger()) {
4692 assert((LHS.getValueType() == RHS.getValueType()) &&(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4693, __PRETTY_FUNCTION__))
4693 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64))(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4693, __PRETTY_FUNCTION__))
;
4694
4695 // If the RHS of the comparison is zero, we can potentially fold this
4696 // to a specialized branch.
4697 const ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS);
4698 if (RHSC && RHSC->getZExtValue() == 0 && ProduceNonFlagSettingCondBr) {
4699 if (CC == ISD::SETEQ) {
4700 // See if we can use a TBZ to fold in an AND as well.
4701 // TBZ has a smaller branch displacement than CBZ. If the offset is
4702 // out of bounds, a late MI-layer pass rewrites branches.
4703 // 403.gcc is an example that hits this case.
4704 if (LHS.getOpcode() == ISD::AND &&
4705 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4706 isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4707 SDValue Test = LHS.getOperand(0);
4708 uint64_t Mask = LHS.getConstantOperandVal(1);
4709 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, Test,
4710 DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4711 Dest);
4712 }
4713
4714 return DAG.getNode(AArch64ISD::CBZ, dl, MVT::Other, Chain, LHS, Dest);
4715 } else if (CC == ISD::SETNE) {
4716 // See if we can use a TBZ to fold in an AND as well.
4717 // TBZ has a smaller branch displacement than CBZ. If the offset is
4718 // out of bounds, a late MI-layer pass rewrites branches.
4719 // 403.gcc is an example that hits this case.
4720 if (LHS.getOpcode() == ISD::AND &&
4721 isa<ConstantSDNode>(LHS.getOperand(1)) &&
4722 isPowerOf2_64(LHS.getConstantOperandVal(1))) {
4723 SDValue Test = LHS.getOperand(0);
4724 uint64_t Mask = LHS.getConstantOperandVal(1);
4725 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, Test,
4726 DAG.getConstant(Log2_64(Mask), dl, MVT::i64),
4727 Dest);
4728 }
4729
4730 return DAG.getNode(AArch64ISD::CBNZ, dl, MVT::Other, Chain, LHS, Dest);
4731 } else if (CC == ISD::SETLT && LHS.getOpcode() != ISD::AND) {
4732 // Don't combine AND since emitComparison converts the AND to an ANDS
4733 // (a.k.a. TST) and the test in the test bit and branch instruction
4734 // becomes redundant. This would also increase register pressure.
4735 uint64_t Mask = LHS.getValueSizeInBits() - 1;
4736 return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS,
4737 DAG.getConstant(Mask, dl, MVT::i64), Dest);
4738 }
4739 }
4740 if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT &&
4741 LHS.getOpcode() != ISD::AND && ProduceNonFlagSettingCondBr) {
4742 // Don't combine AND since emitComparison converts the AND to an ANDS
4743 // (a.k.a. TST) and the test in the test bit and branch instruction
4744 // becomes redundant. This would also increase register pressure.
4745 uint64_t Mask = LHS.getValueSizeInBits() - 1;
4746 return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS,
4747 DAG.getConstant(Mask, dl, MVT::i64), Dest);
4748 }
4749
4750 SDValue CCVal;
4751 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
4752 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal,
4753 Cmp);
4754 }
4755
4756 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4757, __PRETTY_FUNCTION__))
4757 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4757, __PRETTY_FUNCTION__))
;
4758
4759 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
4760 // clean. Some of them require two branches to implement.
4761 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4762 AArch64CC::CondCode CC1, CC2;
4763 changeFPCCToAArch64CC(CC, CC1, CC2);
4764 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4765 SDValue BR1 =
4766 DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp);
4767 if (CC2 != AArch64CC::AL) {
4768 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4769 return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val,
4770 Cmp);
4771 }
4772
4773 return BR1;
4774}
4775
4776SDValue AArch64TargetLowering::LowerFCOPYSIGN(SDValue Op,
4777 SelectionDAG &DAG) const {
4778 EVT VT = Op.getValueType();
4779 SDLoc DL(Op);
4780
4781 SDValue In1 = Op.getOperand(0);
4782 SDValue In2 = Op.getOperand(1);
4783 EVT SrcVT = In2.getValueType();
4784
4785 if (SrcVT.bitsLT(VT))
4786 In2 = DAG.getNode(ISD::FP_EXTEND, DL, VT, In2);
4787 else if (SrcVT.bitsGT(VT))
4788 In2 = DAG.getNode(ISD::FP_ROUND, DL, VT, In2, DAG.getIntPtrConstant(0, DL));
4789
4790 EVT VecVT;
4791 uint64_t EltMask;
4792 SDValue VecVal1, VecVal2;
4793
4794 auto setVecVal = [&] (int Idx) {
4795 if (!VT.isVector()) {
4796 VecVal1 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4797 DAG.getUNDEF(VecVT), In1);
4798 VecVal2 = DAG.getTargetInsertSubreg(Idx, DL, VecVT,
4799 DAG.getUNDEF(VecVT), In2);
4800 } else {
4801 VecVal1 = DAG.getNode(ISD::BITCAST, DL, VecVT, In1);
4802 VecVal2 = DAG.getNode(ISD::BITCAST, DL, VecVT, In2);
4803 }
4804 };
4805
4806 if (VT == MVT::f32 || VT == MVT::v2f32 || VT == MVT::v4f32) {
4807 VecVT = (VT == MVT::v2f32 ? MVT::v2i32 : MVT::v4i32);
4808 EltMask = 0x80000000ULL;
4809 setVecVal(AArch64::ssub);
4810 } else if (VT == MVT::f64 || VT == MVT::v2f64) {
4811 VecVT = MVT::v2i64;
4812
4813 // We want to materialize a mask with the high bit set, but the AdvSIMD
4814 // immediate moves cannot materialize that in a single instruction for
4815 // 64-bit elements. Instead, materialize zero and then negate it.
4816 EltMask = 0;
4817
4818 setVecVal(AArch64::dsub);
4819 } else if (VT == MVT::f16 || VT == MVT::v4f16 || VT == MVT::v8f16) {
4820 VecVT = (VT == MVT::v4f16 ? MVT::v4i16 : MVT::v8i16);
4821 EltMask = 0x8000ULL;
4822 setVecVal(AArch64::hsub);
4823 } else {
4824 llvm_unreachable("Invalid type for copysign!")::llvm::llvm_unreachable_internal("Invalid type for copysign!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4824)
;
4825 }
4826
4827 SDValue BuildVec = DAG.getConstant(EltMask, DL, VecVT);
4828
4829 // If we couldn't materialize the mask above, then the mask vector will be
4830 // the zero vector, and we need to negate it here.
4831 if (VT == MVT::f64 || VT == MVT::v2f64) {
4832 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2f64, BuildVec);
4833 BuildVec = DAG.getNode(ISD::FNEG, DL, MVT::v2f64, BuildVec);
4834 BuildVec = DAG.getNode(ISD::BITCAST, DL, MVT::v2i64, BuildVec);
4835 }
4836
4837 SDValue Sel =
4838 DAG.getNode(AArch64ISD::BIT, DL, VecVT, VecVal1, VecVal2, BuildVec);
4839
4840 if (VT == MVT::f16)
4841 return DAG.getTargetExtractSubreg(AArch64::hsub, DL, VT, Sel);
4842 if (VT == MVT::f32)
4843 return DAG.getTargetExtractSubreg(AArch64::ssub, DL, VT, Sel);
4844 else if (VT == MVT::f64)
4845 return DAG.getTargetExtractSubreg(AArch64::dsub, DL, VT, Sel);
4846 else
4847 return DAG.getNode(ISD::BITCAST, DL, VT, Sel);
4848}
4849
4850SDValue AArch64TargetLowering::LowerCTPOP(SDValue Op, SelectionDAG &DAG) const {
4851 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
4852 Attribute::NoImplicitFloat))
4853 return SDValue();
4854
4855 if (!Subtarget->hasNEON())
4856 return SDValue();
4857
4858 // While there is no integer popcount instruction, it can
4859 // be more efficiently lowered to the following sequence that uses
4860 // AdvSIMD registers/instructions as long as the copies to/from
4861 // the AdvSIMD registers are cheap.
4862 // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
4863 // CNT V0.8B, V0.8B // 8xbyte pop-counts
4864 // ADDV B0, V0.8B // sum 8xbyte pop-counts
4865 // UMOV X0, V0.B[0] // copy byte result back to integer reg
4866 SDValue Val = Op.getOperand(0);
4867 SDLoc DL(Op);
4868 EVT VT = Op.getValueType();
4869
4870 if (VT == MVT::i32 || VT == MVT::i64) {
4871 if (VT == MVT::i32)
4872 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Val);
4873 Val = DAG.getNode(ISD::BITCAST, DL, MVT::v8i8, Val);
4874
4875 SDValue CtPop = DAG.getNode(ISD::CTPOP, DL, MVT::v8i8, Val);
4876 SDValue UaddLV = DAG.getNode(
4877 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
4878 DAG.getConstant(Intrinsic::aarch64_neon_uaddlv, DL, MVT::i32), CtPop);
4879
4880 if (VT == MVT::i64)
4881 UaddLV = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, UaddLV);
4882 return UaddLV;
4883 }
4884
4885 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4887, __PRETTY_FUNCTION__))
4886 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4887, __PRETTY_FUNCTION__))
4887 "Unexpected type for custom ctpop lowering")(((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||
VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&
"Unexpected type for custom ctpop lowering") ? static_cast<
void> (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4887, __PRETTY_FUNCTION__))
;
4888
4889 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
4890 Val = DAG.getBitcast(VT8Bit, Val);
4891 Val = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Val);
4892
4893 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
4894 unsigned EltSize = 8;
4895 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
4896 while (EltSize != VT.getScalarSizeInBits()) {
4897 EltSize *= 2;
4898 NumElts /= 2;
4899 MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
4900 Val = DAG.getNode(
4901 ISD::INTRINSIC_WO_CHAIN, DL, WidenVT,
4902 DAG.getConstant(Intrinsic::aarch64_neon_uaddlp, DL, MVT::i32), Val);
4903 }
4904
4905 return Val;
4906}
4907
4908SDValue AArch64TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
4909
4910 if (Op.getValueType().isVector())
4911 return LowerVSETCC(Op, DAG);
4912
4913 SDValue LHS = Op.getOperand(0);
4914 SDValue RHS = Op.getOperand(1);
4915 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(2))->get();
4916 SDLoc dl(Op);
4917
4918 // We chose ZeroOrOneBooleanContents, so use zero and one.
4919 EVT VT = Op.getValueType();
4920 SDValue TVal = DAG.getConstant(1, dl, VT);
4921 SDValue FVal = DAG.getConstant(0, dl, VT);
4922
4923 // Handle f128 first, since one possible outcome is a normal integer
4924 // comparison which gets picked up by the next if statement.
4925 if (LHS.getValueType() == MVT::f128) {
4926 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
4927
4928 // If softenSetCCOperands returned a scalar, use it.
4929 if (!RHS.getNode()) {
4930 assert(LHS.getValueType() == Op.getValueType() &&((LHS.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4931, __PRETTY_FUNCTION__))
4931 "Unexpected setcc expansion!")((LHS.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4931, __PRETTY_FUNCTION__))
;
4932 return LHS;
4933 }
4934 }
4935
4936 if (LHS.getValueType().isInteger()) {
4937 SDValue CCVal;
4938 SDValue Cmp =
4939 getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl);
4940
4941 // Note that we inverted the condition above, so we reverse the order of
4942 // the true and false operands here. This will allow the setcc to be
4943 // matched to a single CSINC instruction.
4944 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp);
4945 }
4946
4947 // Now we know we're dealing with FP values.
4948 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4949, __PRETTY_FUNCTION__))
4949 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 4949, __PRETTY_FUNCTION__))
;
4950
4951 // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead
4952 // and do the comparison.
4953 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
4954
4955 AArch64CC::CondCode CC1, CC2;
4956 changeFPCCToAArch64CC(CC, CC1, CC2);
4957 if (CC2 == AArch64CC::AL) {
4958 changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2);
4959 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4960
4961 // Note that we inverted the condition above, so we reverse the order of
4962 // the true and false operands here. This will allow the setcc to be
4963 // matched to a single CSINC instruction.
4964 return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CC1Val, Cmp);
4965 } else {
4966 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't
4967 // totally clean. Some of them require two CSELs to implement. As is in
4968 // this case, we emit the first CSEL and then emit a second using the output
4969 // of the first as the RHS. We're effectively OR'ing the two CC's together.
4970
4971 // FIXME: It would be nice if we could match the two CSELs to two CSINCs.
4972 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
4973 SDValue CS1 =
4974 DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
4975
4976 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
4977 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
4978 }
4979}
4980
4981SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS,
4982 SDValue RHS, SDValue TVal,
4983 SDValue FVal, const SDLoc &dl,
4984 SelectionDAG &DAG) const {
4985 // Handle f128 first, because it will result in a comparison of some RTLIB
4986 // call result against zero.
4987 if (LHS.getValueType() == MVT::f128) {
1
Taking true branch
4988 softenSetCCOperands(DAG, MVT::f128, LHS, RHS, CC, dl, LHS, RHS);
4989
4990 // If softenSetCCOperands returned a scalar, we need to compare the result
4991 // against zero to select between true and false values.
4992 if (!RHS.getNode()) {
2
Assuming the condition is false
3
Taking false branch
4993 RHS = DAG.getConstant(0, dl, LHS.getValueType());
4994 CC = ISD::SETNE;
4995 }
4996 }
4997
4998 // Also handle f16, for which we need to do a f32 comparison.
4999 if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
4
Taking false branch
5000 LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
5001 RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
5002 }
5003
5004 // Next, handle integers.
5005 if (LHS.getValueType().isInteger()) {
5
Taking true branch
5006 assert((LHS.getValueType() == RHS.getValueType()) &&(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5007, __PRETTY_FUNCTION__))
5007 (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64))(((LHS.getValueType() == RHS.getValueType()) && (LHS.
getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)
) ? static_cast<void> (0) : __assert_fail ("(LHS.getValueType() == RHS.getValueType()) && (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5007, __PRETTY_FUNCTION__))
;
5008
5009 unsigned Opcode = AArch64ISD::CSEL;
5010
5011 // If both the TVal and the FVal are constants, see if we can swap them in
5012 // order to for a CSINV or CSINC out of them.
5013 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
5014 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
6
Calling 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
21
Returning from 'dyn_cast<llvm::ConstantSDNode, llvm::SDValue>'
5015
5016 if (CTVal && CFVal && CTVal->isAllOnesValue() && CFVal->isNullValue()) {
22
Assuming 'CTVal' is null
5017 std::swap(TVal, FVal);
5018 std::swap(CTVal, CFVal);
5019 CC = ISD::getSetCCInverse(CC, true);
5020 } else if (CTVal
22.1
'CTVal' is null
22.1
'CTVal' is null
22.1
'CTVal' is null
&& CFVal && CTVal->isOne() && CFVal->isNullValue()) {
5021 std::swap(TVal, FVal);
5022 std::swap(CTVal, CFVal);
5023 CC = ISD::getSetCCInverse(CC, true);
5024 } else if (TVal.getOpcode() == ISD::XOR) {
23
Calling 'SDValue::getOpcode'
5025 // If TVal is a NOT we want to swap TVal and FVal so that we can match
5026 // with a CSINV rather than a CSEL.
5027 if (isAllOnesConstant(TVal.getOperand(1))) {
5028 std::swap(TVal, FVal);
5029 std::swap(CTVal, CFVal);
5030 CC = ISD::getSetCCInverse(CC, true);
5031 }
5032 } else if (TVal.getOpcode() == ISD::SUB) {
5033 // If TVal is a negation (SUB from 0) we want to swap TVal and FVal so
5034 // that we can match with a CSNEG rather than a CSEL.
5035 if (isNullConstant(TVal.getOperand(0))) {
5036 std::swap(TVal, FVal);
5037 std::swap(CTVal, CFVal);
5038 CC = ISD::getSetCCInverse(CC, true);
5039 }
5040 } else if (CTVal && CFVal) {
5041 const int64_t TrueVal = CTVal->getSExtValue();
5042 const int64_t FalseVal = CFVal->getSExtValue();
5043 bool Swap = false;
5044
5045 // If both TVal and FVal are constants, see if FVal is the
5046 // inverse/negation/increment of TVal and generate a CSINV/CSNEG/CSINC
5047 // instead of a CSEL in that case.
5048 if (TrueVal == ~FalseVal) {
5049 Opcode = AArch64ISD::CSINV;
5050 } else if (TrueVal == -FalseVal) {
5051 Opcode = AArch64ISD::CSNEG;
5052 } else if (TVal.getValueType() == MVT::i32) {
5053 // If our operands are only 32-bit wide, make sure we use 32-bit
5054 // arithmetic for the check whether we can use CSINC. This ensures that
5055 // the addition in the check will wrap around properly in case there is
5056 // an overflow (which would not be the case if we do the check with
5057 // 64-bit arithmetic).
5058 const uint32_t TrueVal32 = CTVal->getZExtValue();
5059 const uint32_t FalseVal32 = CFVal->getZExtValue();
5060
5061 if ((TrueVal32 == FalseVal32 + 1) || (TrueVal32 + 1 == FalseVal32)) {
5062 Opcode = AArch64ISD::CSINC;
5063
5064 if (TrueVal32 > FalseVal32) {
5065 Swap = true;
5066 }
5067 }
5068 // 64-bit check whether we can use CSINC.
5069 } else if ((TrueVal == FalseVal + 1) || (TrueVal + 1 == FalseVal)) {
5070 Opcode = AArch64ISD::CSINC;
5071
5072 if (TrueVal > FalseVal) {
5073 Swap = true;
5074 }
5075 }
5076
5077 // Swap TVal and FVal if necessary.
5078 if (Swap) {
5079 std::swap(TVal, FVal);
5080 std::swap(CTVal, CFVal);
5081 CC = ISD::getSetCCInverse(CC, true);
5082 }
5083
5084 if (Opcode != AArch64ISD::CSEL) {
5085 // Drop FVal since we can get its value by simply inverting/negating
5086 // TVal.
5087 FVal = TVal;
5088 }
5089 }
5090
5091 // Avoid materializing a constant when possible by reusing a known value in
5092 // a register. However, don't perform this optimization if the known value
5093 // is one, zero or negative one in the case of a CSEL. We can always
5094 // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the
5095 // FVal, respectively.
5096 ConstantSDNode *RHSVal = dyn_cast<ConstantSDNode>(RHS);
5097 if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() &&
5098 !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) {
5099 AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
5100 // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to
5101 // "a != C ? x : a" to avoid materializing C.
5102 if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ)
5103 TVal = LHS;
5104 else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE)
5105 FVal = LHS;
5106 } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) {
5107 assert (CTVal && CFVal && "Expected constant operands for CSNEG.")((CTVal && CFVal && "Expected constant operands for CSNEG."
) ? static_cast<void> (0) : __assert_fail ("CTVal && CFVal && \"Expected constant operands for CSNEG.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5107, __PRETTY_FUNCTION__))
;
5108 // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to
5109 // avoid materializing C.
5110 AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC);
5111 if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) {
5112 Opcode = AArch64ISD::CSINV;
5113 TVal = LHS;
5114 FVal = DAG.getConstant(0, dl, FVal.getValueType());
5115 }
5116 }
5117
5118 SDValue CCVal;
5119 SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
5120 EVT VT = TVal.getValueType();
5121 return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp);
5122 }
5123
5124 // Now we know we're dealing with FP values.
5125 assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 ||((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5126, __PRETTY_FUNCTION__))
5126 LHS.getValueType() == MVT::f64)((LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT
::f32 || LHS.getValueType() == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5126, __PRETTY_FUNCTION__))
;
5127 assert(LHS.getValueType() == RHS.getValueType())((LHS.getValueType() == RHS.getValueType()) ? static_cast<
void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5127, __PRETTY_FUNCTION__))
;
5128 EVT VT = TVal.getValueType();
5129 SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
5130
5131 // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally
5132 // clean. Some of them require two CSELs to implement.
5133 AArch64CC::CondCode CC1, CC2;
5134 changeFPCCToAArch64CC(CC, CC1, CC2);
5135
5136 if (DAG.getTarget().Options.UnsafeFPMath) {
5137 // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and
5138 // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0.
5139 ConstantFPSDNode *RHSVal = dyn_cast<ConstantFPSDNode>(RHS);
5140 if (RHSVal && RHSVal->isZero()) {
5141 ConstantFPSDNode *CFVal = dyn_cast<ConstantFPSDNode>(FVal);
5142 ConstantFPSDNode *CTVal = dyn_cast<ConstantFPSDNode>(TVal);
5143
5144 if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) &&
5145 CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType())
5146 TVal = LHS;
5147 else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) &&
5148 CFVal && CFVal->isZero() &&
5149 FVal.getValueType() == LHS.getValueType())
5150 FVal = LHS;
5151 }
5152 }
5153
5154 // Emit first, and possibly only, CSEL.
5155 SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32);
5156 SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp);
5157
5158 // If we need a second CSEL, emit it, using the output of the first as the
5159 // RHS. We're effectively OR'ing the two CC's together.
5160 if (CC2 != AArch64CC::AL) {
5161 SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32);
5162 return DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, CS1, CC2Val, Cmp);
5163 }
5164
5165 // Otherwise, return the output of the first CSEL.
5166 return CS1;
5167}
5168
5169SDValue AArch64TargetLowering::LowerSELECT_CC(SDValue Op,
5170 SelectionDAG &DAG) const {
5171 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5172 SDValue LHS = Op.getOperand(0);
5173 SDValue RHS = Op.getOperand(1);
5174 SDValue TVal = Op.getOperand(2);
5175 SDValue FVal = Op.getOperand(3);
5176 SDLoc DL(Op);
5177 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5178}
5179
5180SDValue AArch64TargetLowering::LowerSELECT(SDValue Op,
5181 SelectionDAG &DAG) const {
5182 SDValue CCVal = Op->getOperand(0);
5183 SDValue TVal = Op->getOperand(1);
5184 SDValue FVal = Op->getOperand(2);
5185 SDLoc DL(Op);
5186
5187 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a select
5188 // instruction.
5189 if (isOverflowIntrOpRes(CCVal)) {
5190 // Only lower legal XALUO ops.
5191 if (!DAG.getTargetLoweringInfo().isTypeLegal(CCVal->getValueType(0)))
5192 return SDValue();
5193
5194 AArch64CC::CondCode OFCC;
5195 SDValue Value, Overflow;
5196 std::tie(Value, Overflow) = getAArch64XALUOOp(OFCC, CCVal.getValue(0), DAG);
5197 SDValue CCVal = DAG.getConstant(OFCC, DL, MVT::i32);
5198
5199 return DAG.getNode(AArch64ISD::CSEL, DL, Op.getValueType(), TVal, FVal,
5200 CCVal, Overflow);
5201 }
5202
5203 // Lower it the same way as we would lower a SELECT_CC node.
5204 ISD::CondCode CC;
5205 SDValue LHS, RHS;
5206 if (CCVal.getOpcode() == ISD::SETCC) {
5207 LHS = CCVal.getOperand(0);
5208 RHS = CCVal.getOperand(1);
5209 CC = cast<CondCodeSDNode>(CCVal->getOperand(2))->get();
5210 } else {
5211 LHS = CCVal;
5212 RHS = DAG.getConstant(0, DL, CCVal.getValueType());
5213 CC = ISD::SETNE;
5214 }
5215 return LowerSELECT_CC(CC, LHS, RHS, TVal, FVal, DL, DAG);
5216}
5217
5218SDValue AArch64TargetLowering::LowerJumpTable(SDValue Op,
5219 SelectionDAG &DAG) const {
5220 // Jump table entries as PC relative offsets. No additional tweaking
5221 // is necessary here. Just get the address of the jump table.
5222 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
5223
5224 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5225 !Subtarget->isTargetMachO()) {
5226 return getAddrLarge(JT, DAG);
5227 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5228 return getAddrTiny(JT, DAG);
5229 }
5230 return getAddr(JT, DAG);
5231}
5232
5233SDValue AArch64TargetLowering::LowerBR_JT(SDValue Op,
5234 SelectionDAG &DAG) const {
5235 // Jump table entries as PC relative offsets. No additional tweaking
5236 // is necessary here. Just get the address of the jump table.
5237 SDLoc DL(Op);
5238 SDValue JT = Op.getOperand(1);
5239 SDValue Entry = Op.getOperand(2);
5240 int JTI = cast<JumpTableSDNode>(JT.getNode())->getIndex();
5241
5242 SDNode *Dest =
5243 DAG.getMachineNode(AArch64::JumpTableDest32, DL, MVT::i64, MVT::i64, JT,
5244 Entry, DAG.getTargetJumpTable(JTI, MVT::i32));
5245 return DAG.getNode(ISD::BRIND, DL, MVT::Other, Op.getOperand(0),
5246 SDValue(Dest, 0));
5247}
5248
5249SDValue AArch64TargetLowering::LowerConstantPool(SDValue Op,
5250 SelectionDAG &DAG) const {
5251 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
5252
5253 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
5254 // Use the GOT for the large code model on iOS.
5255 if (Subtarget->isTargetMachO()) {
5256 return getGOT(CP, DAG);
5257 }
5258 return getAddrLarge(CP, DAG);
5259 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5260 return getAddrTiny(CP, DAG);
5261 } else {
5262 return getAddr(CP, DAG);
5263 }
5264}
5265
5266SDValue AArch64TargetLowering::LowerBlockAddress(SDValue Op,
5267 SelectionDAG &DAG) const {
5268 BlockAddressSDNode *BA = cast<BlockAddressSDNode>(Op);
5269 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
5270 !Subtarget->isTargetMachO()) {
5271 return getAddrLarge(BA, DAG);
5272 } else if (getTargetMachine().getCodeModel() == CodeModel::Tiny) {
5273 return getAddrTiny(BA, DAG);
5274 }
5275 return getAddr(BA, DAG);
5276}
5277
5278SDValue AArch64TargetLowering::LowerDarwin_VASTART(SDValue Op,
5279 SelectionDAG &DAG) const {
5280 AArch64FunctionInfo *FuncInfo =
5281 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
5282
5283 SDLoc DL(Op);
5284 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(),
5285 getPointerTy(DAG.getDataLayout()));
5286 FR = DAG.getZExtOrTrunc(FR, DL, getPointerMemTy(DAG.getDataLayout()));
5287 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5288 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
5289 MachinePointerInfo(SV));
5290}
5291
5292SDValue AArch64TargetLowering::LowerWin64_VASTART(SDValue Op,
5293 SelectionDAG &DAG) const {
5294 AArch64FunctionInfo *FuncInfo =
5295 DAG.getMachineFunction().getInfo<AArch64FunctionInfo>();
5296
5297 SDLoc DL(Op);
5298 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsGPRSize() > 0
5299 ? FuncInfo->getVarArgsGPRIndex()
5300 : FuncInfo->getVarArgsStackIndex(),
5301 getPointerTy(DAG.getDataLayout()));
5302 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5303 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
5304 MachinePointerInfo(SV));
5305}
5306
5307SDValue AArch64TargetLowering::LowerAAPCS_VASTART(SDValue Op,
5308 SelectionDAG &DAG) const {
5309 // The layout of the va_list struct is specified in the AArch64 Procedure Call
5310 // Standard, section B.3.
5311 MachineFunction &MF = DAG.getMachineFunction();
5312 AArch64FunctionInfo *FuncInfo = MF.getInfo<AArch64FunctionInfo>();
5313 auto PtrVT = getPointerTy(DAG.getDataLayout());
5314 SDLoc DL(Op);
5315
5316 SDValue Chain = Op.getOperand(0);
5317 SDValue VAList = Op.getOperand(1);
5318 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5319 SmallVector<SDValue, 4> MemOps;
5320
5321 // void *__stack at offset 0
5322 SDValue Stack = DAG.getFrameIndex(FuncInfo->getVarArgsStackIndex(), PtrVT);
5323 MemOps.push_back(DAG.getStore(Chain, DL, Stack, VAList,
5324 MachinePointerInfo(SV), /* Alignment = */ 8));
5325
5326 // void *__gr_top at offset 8
5327 int GPRSize = FuncInfo->getVarArgsGPRSize();
5328 if (GPRSize > 0) {
5329 SDValue GRTop, GRTopAddr;
5330
5331 GRTopAddr =
5332 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(8, DL, PtrVT));
5333
5334 GRTop = DAG.getFrameIndex(FuncInfo->getVarArgsGPRIndex(), PtrVT);
5335 GRTop = DAG.getNode(ISD::ADD, DL, PtrVT, GRTop,
5336 DAG.getConstant(GPRSize, DL, PtrVT));
5337
5338 MemOps.push_back(DAG.getStore(Chain, DL, GRTop, GRTopAddr,
5339 MachinePointerInfo(SV, 8),
5340 /* Alignment = */ 8));
5341 }
5342
5343 // void *__vr_top at offset 16
5344 int FPRSize = FuncInfo->getVarArgsFPRSize();
5345 if (FPRSize > 0) {
5346 SDValue VRTop, VRTopAddr;
5347 VRTopAddr = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5348 DAG.getConstant(16, DL, PtrVT));
5349
5350 VRTop = DAG.getFrameIndex(FuncInfo->getVarArgsFPRIndex(), PtrVT);
5351 VRTop = DAG.getNode(ISD::ADD, DL, PtrVT, VRTop,
5352 DAG.getConstant(FPRSize, DL, PtrVT));
5353
5354 MemOps.push_back(DAG.getStore(Chain, DL, VRTop, VRTopAddr,
5355 MachinePointerInfo(SV, 16),
5356 /* Alignment = */ 8));
5357 }
5358
5359 // int __gr_offs at offset 24
5360 SDValue GROffsAddr =
5361 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(24, DL, PtrVT));
5362 MemOps.push_back(DAG.getStore(
5363 Chain, DL, DAG.getConstant(-GPRSize, DL, MVT::i32), GROffsAddr,
5364 MachinePointerInfo(SV, 24), /* Alignment = */ 4));
5365
5366 // int __vr_offs at offset 28
5367 SDValue VROffsAddr =
5368 DAG.getNode(ISD::ADD, DL, PtrVT, VAList, DAG.getConstant(28, DL, PtrVT));
5369 MemOps.push_back(DAG.getStore(
5370 Chain, DL, DAG.getConstant(-FPRSize, DL, MVT::i32), VROffsAddr,
5371 MachinePointerInfo(SV, 28), /* Alignment = */ 4));
5372
5373 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
5374}
5375
5376SDValue AArch64TargetLowering::LowerVASTART(SDValue Op,
5377 SelectionDAG &DAG) const {
5378 MachineFunction &MF = DAG.getMachineFunction();
5379
5380 if (Subtarget->isCallingConvWin64(MF.getFunction().getCallingConv()))
5381 return LowerWin64_VASTART(Op, DAG);
5382 else if (Subtarget->isTargetDarwin())
5383 return LowerDarwin_VASTART(Op, DAG);
5384 else
5385 return LowerAAPCS_VASTART(Op, DAG);
5386}
5387
5388SDValue AArch64TargetLowering::LowerVACOPY(SDValue Op,
5389 SelectionDAG &DAG) const {
5390 // AAPCS has three pointers and two ints (= 32 bytes), Darwin has single
5391 // pointer.
5392 SDLoc DL(Op);
5393 unsigned PtrSize = Subtarget->isTargetILP32() ? 4 : 8;
5394 unsigned VaListSize = (Subtarget->isTargetDarwin() ||
5395 Subtarget->isTargetWindows()) ? PtrSize : 32;
5396 const Value *DestSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
5397 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
5398
5399 return DAG.getMemcpy(Op.getOperand(0), DL, Op.getOperand(1), Op.getOperand(2),
5400 DAG.getConstant(VaListSize, DL, MVT::i32), PtrSize,
5401 false, false, false, MachinePointerInfo(DestSV),
5402 MachinePointerInfo(SrcSV));
5403}
5404
5405SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
5406 assert(Subtarget->isTargetDarwin() &&((Subtarget->isTargetDarwin() && "automatic va_arg instruction only works on Darwin"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"automatic va_arg instruction only works on Darwin\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5407, __PRETTY_FUNCTION__))
5407 "automatic va_arg instruction only works on Darwin")((Subtarget->isTargetDarwin() && "automatic va_arg instruction only works on Darwin"
) ? static_cast<void> (0) : __assert_fail ("Subtarget->isTargetDarwin() && \"automatic va_arg instruction only works on Darwin\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5407, __PRETTY_FUNCTION__))
;
5408
5409 const Value *V = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
5410 EVT VT = Op.getValueType();
5411 SDLoc DL(Op);
5412 SDValue Chain = Op.getOperand(0);
5413 SDValue Addr = Op.getOperand(1);
5414 unsigned Align = Op.getConstantOperandVal(3);
5415 unsigned MinSlotSize = Subtarget->isTargetILP32() ? 4 : 8;
5416 auto PtrVT = getPointerTy(DAG.getDataLayout());
5417 auto PtrMemVT = getPointerMemTy(DAG.getDataLayout());
5418 SDValue VAList =
5419 DAG.getLoad(PtrMemVT, DL, Chain, Addr, MachinePointerInfo(V));
5420 Chain = VAList.getValue(1);
5421 VAList = DAG.getZExtOrTrunc(VAList, DL, PtrVT);
5422
5423 if (Align > MinSlotSize) {
5424 assert(((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2")((((Align & (Align - 1)) == 0) && "Expected Align to be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("((Align & (Align - 1)) == 0) && \"Expected Align to be a power of 2\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5424, __PRETTY_FUNCTION__))
;
5425 VAList = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5426 DAG.getConstant(Align - 1, DL, PtrVT));
5427 VAList = DAG.getNode(ISD::AND, DL, PtrVT, VAList,
5428 DAG.getConstant(-(int64_t)Align, DL, PtrVT));
5429 }
5430
5431 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
5432 unsigned ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
5433
5434 // Scalar integer and FP values smaller than 64 bits are implicitly extended
5435 // up to 64 bits. At the very least, we have to increase the striding of the
5436 // vaargs list to match this, and for FP values we need to introduce
5437 // FP_ROUND nodes as well.
5438 if (VT.isInteger() && !VT.isVector())
5439 ArgSize = std::max(ArgSize, MinSlotSize);
5440 bool NeedFPTrunc = false;
5441 if (VT.isFloatingPoint() && !VT.isVector() && VT != MVT::f64) {
5442 ArgSize = 8;
5443 NeedFPTrunc = true;
5444 }
5445
5446 // Increment the pointer, VAList, to the next vaarg
5447 SDValue VANext = DAG.getNode(ISD::ADD, DL, PtrVT, VAList,
5448 DAG.getConstant(ArgSize, DL, PtrVT));
5449 VANext = DAG.getZExtOrTrunc(VANext, DL, PtrMemVT);
5450
5451 // Store the incremented VAList to the legalized pointer
5452 SDValue APStore =
5453 DAG.getStore(Chain, DL, VANext, Addr, MachinePointerInfo(V));
5454
5455 // Load the actual argument out of the pointer VAList
5456 if (NeedFPTrunc) {
5457 // Load the value as an f64.
5458 SDValue WideFP =
5459 DAG.getLoad(MVT::f64, DL, APStore, VAList, MachinePointerInfo());
5460 // Round the value down to an f32.
5461 SDValue NarrowFP = DAG.getNode(ISD::FP_ROUND, DL, VT, WideFP.getValue(0),
5462 DAG.getIntPtrConstant(1, DL));
5463 SDValue Ops[] = { NarrowFP, WideFP.getValue(1) };
5464 // Merge the rounded value with the chain output of the load.
5465 return DAG.getMergeValues(Ops, DL);
5466 }
5467
5468 return DAG.getLoad(VT, DL, APStore, VAList, MachinePointerInfo());
5469}
5470
5471SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op,
5472 SelectionDAG &DAG) const {
5473 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5474 MFI.setFrameAddressIsTaken(true);
5475
5476 EVT VT = Op.getValueType();
5477 SDLoc DL(Op);
5478 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5479 SDValue FrameAddr =
5480 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, MVT::i64);
5481 while (Depth--)
5482 FrameAddr = DAG.getLoad(VT, DL, DAG.getEntryNode(), FrameAddr,
5483 MachinePointerInfo());
5484
5485 if (Subtarget->isTargetILP32())
5486 FrameAddr = DAG.getNode(ISD::AssertZext, DL, MVT::i64, FrameAddr,
5487 DAG.getValueType(VT));
5488
5489 return FrameAddr;
5490}
5491
5492SDValue AArch64TargetLowering::LowerSPONENTRY(SDValue Op,
5493 SelectionDAG &DAG) const {
5494 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5495
5496 EVT VT = getPointerTy(DAG.getDataLayout());
5497 SDLoc DL(Op);
5498 int FI = MFI.CreateFixedObject(4, 0, false);
5499 return DAG.getFrameIndex(FI, VT);
5500}
5501
5502#define GET_REGISTER_MATCHER
5503#include "AArch64GenAsmMatcher.inc"
5504
5505// FIXME? Maybe this could be a TableGen attribute on some registers and
5506// this table could be generated automatically from RegInfo.
5507Register AArch64TargetLowering::
5508getRegisterByName(const char* RegName, EVT VT, const MachineFunction &MF) const {
5509 Register Reg = MatchRegisterName(RegName);
5510 if (AArch64::X1 <= Reg && Reg <= AArch64::X28) {
5511 const MCRegisterInfo *MRI = Subtarget->getRegisterInfo();
5512 unsigned DwarfRegNum = MRI->getDwarfRegNum(Reg, false);
5513 if (!Subtarget->isXRegisterReserved(DwarfRegNum))
5514 Reg = 0;
5515 }
5516 if (Reg)
5517 return Reg;
5518 report_fatal_error(Twine("Invalid register name \""
5519 + StringRef(RegName) + "\"."));
5520}
5521
5522SDValue AArch64TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
5523 SelectionDAG &DAG) const {
5524 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
5525
5526 EVT VT = Op.getValueType();
5527 SDLoc DL(Op);
5528
5529 SDValue FrameAddr =
5530 DAG.getCopyFromReg(DAG.getEntryNode(), DL, AArch64::FP, VT);
5531 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5532
5533 return DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset);
5534}
5535
5536SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op,
5537 SelectionDAG &DAG) const {
5538 MachineFunction &MF = DAG.getMachineFunction();
5539 MachineFrameInfo &MFI = MF.getFrameInfo();
5540 MFI.setReturnAddressIsTaken(true);
5541
5542 EVT VT = Op.getValueType();
5543 SDLoc DL(Op);
5544 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5545 if (Depth) {
5546 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5547 SDValue Offset = DAG.getConstant(8, DL, getPointerTy(DAG.getDataLayout()));
5548 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
5549 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
5550 MachinePointerInfo());
5551 }
5552
5553 // Return LR, which contains the return address. Mark it an implicit live-in.
5554 unsigned Reg = MF.addLiveIn(AArch64::LR, &AArch64::GPR64RegClass);
5555 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, VT);
5556}
5557
5558/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
5559/// i64 values and take a 2 x i64 value to shift plus a shift amount.
5560SDValue AArch64TargetLowering::LowerShiftRightParts(SDValue Op,
5561 SelectionDAG &DAG) const {
5562 assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5562, __PRETTY_FUNCTION__))
;
5563 EVT VT = Op.getValueType();
5564 unsigned VTBits = VT.getSizeInBits();
5565 SDLoc dl(Op);
5566 SDValue ShOpLo = Op.getOperand(0);
5567 SDValue ShOpHi = Op.getOperand(1);
5568 SDValue ShAmt = Op.getOperand(2);
5569 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
5570
5571 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS)((Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::
SRL_PARTS) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5571, __PRETTY_FUNCTION__))
;
5572
5573 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5574 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5575 SDValue HiBitsForLo = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
5576
5577 // Unfortunately, if ShAmt == 0, we just calculated "(SHL ShOpHi, 64)" which
5578 // is "undef". We wanted 0, so CSEL it directly.
5579 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5580 ISD::SETEQ, dl, DAG);
5581 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5582 HiBitsForLo =
5583 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5584 HiBitsForLo, CCVal, Cmp);
5585
5586 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5587 DAG.getConstant(VTBits, dl, MVT::i64));
5588
5589 SDValue LoBitsForLo = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
5590 SDValue LoForNormalShift =
5591 DAG.getNode(ISD::OR, dl, VT, LoBitsForLo, HiBitsForLo);
5592
5593 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5594 dl, DAG);
5595 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5596 SDValue LoForBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
5597 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5598 LoForNormalShift, CCVal, Cmp);
5599
5600 // AArch64 shifts larger than the register width are wrapped rather than
5601 // clamped, so we can't just emit "hi >> x".
5602 SDValue HiForNormalShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
5603 SDValue HiForBigShift =
5604 Opc == ISD::SRA
5605 ? DAG.getNode(Opc, dl, VT, ShOpHi,
5606 DAG.getConstant(VTBits - 1, dl, MVT::i64))
5607 : DAG.getConstant(0, dl, VT);
5608 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5609 HiForNormalShift, CCVal, Cmp);
5610
5611 SDValue Ops[2] = { Lo, Hi };
5612 return DAG.getMergeValues(Ops, dl);
5613}
5614
5615/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
5616/// i64 values and take a 2 x i64 value to shift plus a shift amount.
5617SDValue AArch64TargetLowering::LowerShiftLeftParts(SDValue Op,
5618 SelectionDAG &DAG) const {
5619 assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5619, __PRETTY_FUNCTION__))
;
5620 EVT VT = Op.getValueType();
5621 unsigned VTBits = VT.getSizeInBits();
5622 SDLoc dl(Op);
5623 SDValue ShOpLo = Op.getOperand(0);
5624 SDValue ShOpHi = Op.getOperand(1);
5625 SDValue ShAmt = Op.getOperand(2);
5626
5627 assert(Op.getOpcode() == ISD::SHL_PARTS)((Op.getOpcode() == ISD::SHL_PARTS) ? static_cast<void>
(0) : __assert_fail ("Op.getOpcode() == ISD::SHL_PARTS", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/AArch64/AArch64ISelLowering.cpp"
, 5627, __PRETTY_FUNCTION__))
;
5628 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64,
5629 DAG.getConstant(VTBits, dl, MVT::i64), ShAmt);
5630 SDValue LoBitsForHi = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
5631
5632 // Unfortunately, if ShAmt == 0, we just calculated "(SRL ShOpLo, 64)" which
5633 // is "undef". We wanted 0, so CSEL it directly.
5634 SDValue Cmp = emitComparison(ShAmt, DAG.getConstant(0, dl, MVT::i64),
5635 ISD::SETEQ, dl, DAG);
5636 SDValue CCVal = DAG.getConstant(AArch64CC::EQ, dl, MVT::i32);
5637 LoBitsForHi =
5638 DAG.getNode(AArch64ISD::CSEL, dl, VT, DAG.getConstant(0, dl, MVT::i64),
5639 LoBitsForHi, CCVal, Cmp);
5640
5641 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i64, ShAmt,
5642 DAG.getConstant(VTBits, dl, MVT::i64));
5643 SDValue HiBitsForHi = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
5644 SDValue HiForNormalShift =
5645 DAG.getNode(ISD::OR, dl, VT, LoBitsForHi, HiBitsForHi);
5646
5647 SDValue HiForBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
5648
5649 Cmp = emitComparison(ExtraShAmt, DAG.getConstant(0, dl, MVT::i64), ISD::SETGE,
5650 dl, DAG);
5651 CCVal = DAG.getConstant(AArch64CC::GE, dl, MVT::i32);
5652 SDValue Hi = DAG.getNode(AArch64ISD::CSEL, dl, VT, HiForBigShift,
5653 HiForNormalShift, CCVal, Cmp);
5654
5655 // AArch64 shifts of larger than register sizes are wrapped rather than
5656 // clamped, so we can't just emit "lo << a" if a is too big.
5657 SDValue LoForBigShift = DAG.getConstant(0, dl, VT);
5658 SDValue LoForNormalShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
5659 SDValue Lo = DAG.getNode(AArch64ISD::CSEL, dl, VT, LoForBigShift,
5660 LoForNormalShift, CCVal, Cmp);
5661
5662 SDValue Ops[2] = { Lo, Hi };
5663 return DAG.getMergeValues(Ops, dl);
5664}
5665
5666bool AArch64TargetLowering::isOffsetFoldingLegal(
5667 const GlobalAddressSDNode *GA) const {
5668 // Offsets are folded in the DAG combine rather than here so that we can
5669 // intelligently choose an offset based on the uses.
5670 return false;
5671}
5672
5673bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
5674 bool OptForSize) const {
5675 bool IsLegal = false;
5676 // We can materialize #0.0 as fmov $Rd, XZR for 64-bit, 32-bit cases, and
5677 // 16-bit case when target has full fp16 support.
5678 // FIXME: We should be able to handle f128 as well with a clever lowering.
5679 const APInt ImmInt = Imm.bitcastToAPInt();
5680 if (VT == MVT::f64)
5681 IsLegal = AArch64_AM::getFP64Imm(ImmInt) != -1 || Imm.isPosZero();
5682 else if (VT == MVT::f32)
5683 IsLegal = AArch64_AM::getFP32Imm(ImmInt) != -1 || Imm.isPosZero();
5684 else if (VT == MVT::f16 && Subtarget->hasFullFP16())
5685 IsLegal = AArch64_AM::getFP16Imm(ImmInt) != -1 || Imm.isPosZero();
5686 // TODO: fmov h0, w0 is also legal, however on't have an isel pattern to
5687 // generate that fmov.
5688
5689 // If we can not materialize in immediate field for fmov, check if the
5690 // value can be encoded as the immediate operand of a logical instruction.
5691 // The immediate value will be created with either MOVZ, MOVN, or ORR.
5692 if (!IsLegal && (VT == MVT::f64 || VT == MVT::f32)) {
5693 // The cost is actually exactly the same for mov+fmov vs. adrp+ldr;
5694 // however the mov+fmov sequence is always better because of the reduced
5695 // cache pressure. The timings are still the same if you consider
5696 // movw+movk+fmov vs. adrp+ldr (it's one instruction longer, but the
5697 // movw+movk is fused). So we limit up to 2 instrdduction at most.
5698 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
5699 AArch64_IMM::expandMOVImm(ImmInt.getZExtValue(), VT.getSizeInBits(),
5700 Insn);
5701 unsigned Limit = (OptForSize ? 1 : (Subtarget->hasFuseLiterals() ? 5 : 2));
5702 IsLegal = Insn.size() <= Limit;
5703 }
5704
5705 LLVM_DEBUG(dbgs() << (IsLegal ? "Legal " : "Illegal ") << VT.getEVTString()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << (IsLegal ? "Legal " : "Illegal "
) << VT.getEVTString() << " imm value: "; Imm.dump
();; } } while (false)
5706 << " imm value: "; Imm.dump();)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-lower")) { dbgs() << (IsLegal ? "Legal " : "Illegal "
) << VT.getEVTString() << " imm value: "; Imm.dump
();; } } while (false)
;
5707 return IsLegal;
5708}
5709
5710//===----------------------------------------------------------------------===//
5711// AArch64 Optimization Hooks
5712//===----------------------------------------------------------------------===//
5713
5714static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode,
5715 SDValue Operand, SelectionDAG &DAG,
5716 int &ExtraSteps) {
5717 EVT VT = Operand.getValueType();
5718 if (ST->hasNEON() &&
5719 (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 ||
5720 VT == MVT::f32 || VT == MVT::v1f32 ||
5721 VT == MVT::v2f32 || VT == MVT::v4f32)) {
5722 if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified)
5723 // For the reciprocal estimates, convergence is quadratic, so the number
5724 // of digits is doubled after each iteration. In ARMv8, the accuracy of
5725 // the initial estimate is 2^-8. Thus the number of extra steps to refine
5726 // the result for float (23 mantissa bits) is 2 and for double (52
5727 // mantissa bits) is 3.
5728 ExtraSteps = VT.getScalarType() == MVT::f64 ? 3 : 2;
5729
5730 return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand);
5731 }
5732
5733 return SDValue();
5734}
5735
5736SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand,
5737 SelectionDAG &DAG, int Enabled,
5738 int &ExtraSteps,
5739 bool &UseOneConst,
5740 bool Reciprocal) const {
5741 if (Enabled == ReciprocalEstimate::Enabled ||
5742 (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt()))
5743 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand,
5744 DAG, ExtraSteps)) {
5745 SDLoc DL(Operand);
5746 EVT VT = Operand.getValueType();
5747
5748 SDNodeFlags Flags;
5749 Flags.setAllowReassociation(true);
5750
5751 // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2)
5752 // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N)
5753 for (int i = ExtraSteps; i > 0; --i) {
5754 SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate,
5755 Flags);
5756 Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, Flags);
5757 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5758 }
5759 if (!Reciprocal) {
5760 EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
5761 VT);
5762 SDValue FPZero = DAG.getConstantFP(0.0, DL, VT);
5763 SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ);
5764
5765 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, Flags);
5766 // Correct the result if the operand is 0.0.
5767 Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL,
5768 VT, Eq, Operand, Estimate);
5769 }
5770
5771 ExtraSteps = 0;
5772 return Estimate;
5773 }
5774
5775 return SDValue();
5776}
5777
5778SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand,
5779 SelectionDAG &DAG, int Enabled,
5780 int &ExtraSteps) const {
5781 if (Enabled == ReciprocalEstimate::Enabled)
5782 if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand,
5783 DAG, ExtraSteps)) {
5784 SDLoc DL(Operand);
5785 EVT VT = Operand.getValueType();
5786
5787 SDNodeFlags Flags;
5788 Flags.setAllowReassociation(true);
5789
5790 // Newton reciprocal iteration: E * (2 - X * E)
5791 // AArch64 reciprocal iteration instruction: (2 - M * N)
5792 for (int i = ExtraSteps; i > 0; --i) {
5793 SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand,
5794 Estimate, Flags);
5795 Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, Flags);
5796 }
5797
5798 ExtraSteps = 0;
5799 return Estimate;
5800 }
5801
5802 return SDValue();
5803}
5804
5805//===----------------------------------------------------------------------===//
5806// AArch64 Inline Assembly Support
5807//===----------------------------------------------------------------------===//
5808
5809// Table of Constraints
5810// TODO: This is the current set of constraints supported by ARM for the
5811// compiler, not all of them may make sense.
5812//
5813// r - A general register
5814// w - An FP/SIMD register of some size in the range v0-v31
5815// x - An FP/SIMD register of some size in the range v0-v15
5816// I - Constant that can be used with an ADD instruction
5817// J - Constant that can be used with a SUB instruction
5818// K - Constant that can be used with a 32-bit logical instruction
5819// L - Constant that can be used with a 64-bit logical instruction
5820// M - Constant that can be used as a 32-bit MOV immediate
5821// N - Constant that can be used as a 64-bit MOV immediate
5822// Q - A memory reference with base register and no offset
5823// S - A symbolic address
5824// Y - Floating point constant zero
5825// Z - Integer constant zero
5826//
5827// Note that general register operands will be output using their 64-bit x
5828// register name, whatever the size of the variable, unless the asm operand
5829// is prefixed by the %w modifier. Floating-point and SIMD register operands
5830// will be output with the v prefix unless prefixed by the %b, %h, %s, %d or
5831// %q modifier.
5832const char *AArch64TargetLowering::LowerXConstraint(EVT ConstraintVT) const {
5833 // At this point, we have to lower this constraint to something else, so we
5834 // lower it to an "r" or "w". However, by doing this we will force the result
5835 // to be in register, while the X constraint is much more permissive.
5836 //
5837 // Although we are correct (we are free to emit anything, without
5838 // constraints), we might break use cases that would expect us to be more
5839 // efficient and emit something else.
5840 if (!Subtarget->hasFPARMv8())
5841 return "r";
5842
5843 if (ConstraintVT.isFloatingPoint())
5844 return "w";
5845
5846 if (ConstraintVT.isVector() &&
5847 (ConstraintVT.getSizeInBits() == 64 ||
5848 ConstraintVT.getSizeInBits() == 128))
5849 return "w";
5850
5851 return "r";
5852}
5853
5854enum PredicateConstraint {
5855 Upl,
5856 Upa,
5857 Invalid
5858};
5859
5860static PredicateConstraint parsePredicateConstraint(StringRef Constraint) {
5861 PredicateConstraint P = PredicateConstraint::Invalid;
5862 if (Constraint == "Upa")
5863 P = PredicateConstraint::Upa;
5864 if (Constraint == "Upl")
5865 P = PredicateConstraint::Upl;
5866 return P;
5867}
5868
5869/// getConstraintType - Given a constraint letter, return the type of
5870/// constraint it is for this target.
5871AArch64TargetLowering::ConstraintType
5872AArch64TargetLowering::getConstraintType(StringRef Constraint) const {
5873 if (Constraint.size() == 1) {
5874 switch (Constraint[0]) {
5875 default:
5876 break;
5877 case 'x':
5878 case 'w':
5879 case 'y':
5880 return C_RegisterClass;
5881 // An address with a single base register. Due to the way we
5882 // currently handle addresses it is the same as 'r'.
5883 case 'Q':
5884 return C_Memory;
5885 case 'I':
5886 case 'J':
5887 case 'K':
5888 case 'L':
5889 case 'M':
5890 case 'N':
5891 case 'Y':
5892 case 'Z':
5893 return C_Immediate;
5894 case 'z':
5895 case 'S': // A symbolic address
5896 return C_Other;
5897 }
5898 } else if (parsePredicateConstraint(Constraint) !=
5899 PredicateConstraint::Invalid)
5900 return C_RegisterClass;
5901 return TargetLowering::getConstraintType(Constraint);
5902}
5903
5904/// Examine constraint type and operand type and determine a weight value.
5905/// This object must already have been set up with the operand type
5906/// and the current alternative constraint selected.
5907TargetLowering::ConstraintWeight
5908AArch64TargetLowering::getSingleConstraintMatchWeight(
5909 AsmOperandInfo &info, const char *constraint) const {
5910 ConstraintWeight weight = CW_Invalid;
5911 Value *CallOperandVal = info.CallOperandVal;
5912 // If we don't have a value, we can't do a match,
5913 // but allow it at the lowest weight.
5914 if (!CallOperandVal)
5915 return CW_Default;
5916 Type *type = CallOperandVal->getType();
5917 // Look at the constraint type.
5918 switch (*constraint) {
5919 default:
5920 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
5921 break;
5922 case 'x':
5923 case 'w':
5924 case 'y':
5925 if (type->isFloatingPointTy() || type->isVectorTy())