Bug Summary

File:llvm/lib/Target/ARM/ARMISelLowering.cpp
Warning:line 2530, column 20
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name ARMISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -fhalf-no-semantic-interposition -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/build-llvm/lib/Target/ARM -resource-dir /usr/lib/llvm-13/lib/clang/13.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/build-llvm/lib/Target/ARM -I /build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM -I /build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/build-llvm/include -I /build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-13/lib/clang/13.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/build-llvm/lib/Target/ARM -fdebug-prefix-map=/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3=. -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-05-28-190823-39734-1 -x c++ /build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp
1//===- ARMISelLowering.cpp - ARM DAG Lowering Implementation --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that ARM uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "ARMISelLowering.h"
15#include "ARMBaseInstrInfo.h"
16#include "ARMBaseRegisterInfo.h"
17#include "ARMCallingConv.h"
18#include "ARMConstantPoolValue.h"
19#include "ARMMachineFunctionInfo.h"
20#include "ARMPerfectShuffle.h"
21#include "ARMRegisterInfo.h"
22#include "ARMSelectionDAGInfo.h"
23#include "ARMSubtarget.h"
24#include "ARMTargetTransformInfo.h"
25#include "MCTargetDesc/ARMAddressingModes.h"
26#include "MCTargetDesc/ARMBaseInfo.h"
27#include "Utils/ARMBaseInfo.h"
28#include "llvm/ADT/APFloat.h"
29#include "llvm/ADT/APInt.h"
30#include "llvm/ADT/ArrayRef.h"
31#include "llvm/ADT/BitVector.h"
32#include "llvm/ADT/DenseMap.h"
33#include "llvm/ADT/STLExtras.h"
34#include "llvm/ADT/SmallPtrSet.h"
35#include "llvm/ADT/SmallVector.h"
36#include "llvm/ADT/Statistic.h"
37#include "llvm/ADT/StringExtras.h"
38#include "llvm/ADT/StringRef.h"
39#include "llvm/ADT/StringSwitch.h"
40#include "llvm/ADT/Triple.h"
41#include "llvm/ADT/Twine.h"
42#include "llvm/Analysis/VectorUtils.h"
43#include "llvm/CodeGen/CallingConvLower.h"
44#include "llvm/CodeGen/ISDOpcodes.h"
45#include "llvm/CodeGen/IntrinsicLowering.h"
46#include "llvm/CodeGen/MachineBasicBlock.h"
47#include "llvm/CodeGen/MachineConstantPool.h"
48#include "llvm/CodeGen/MachineFrameInfo.h"
49#include "llvm/CodeGen/MachineFunction.h"
50#include "llvm/CodeGen/MachineInstr.h"
51#include "llvm/CodeGen/MachineInstrBuilder.h"
52#include "llvm/CodeGen/MachineJumpTableInfo.h"
53#include "llvm/CodeGen/MachineMemOperand.h"
54#include "llvm/CodeGen/MachineOperand.h"
55#include "llvm/CodeGen/MachineRegisterInfo.h"
56#include "llvm/CodeGen/RuntimeLibcalls.h"
57#include "llvm/CodeGen/SelectionDAG.h"
58#include "llvm/CodeGen/SelectionDAGNodes.h"
59#include "llvm/CodeGen/TargetInstrInfo.h"
60#include "llvm/CodeGen/TargetLowering.h"
61#include "llvm/CodeGen/TargetOpcodes.h"
62#include "llvm/CodeGen/TargetRegisterInfo.h"
63#include "llvm/CodeGen/TargetSubtargetInfo.h"
64#include "llvm/CodeGen/ValueTypes.h"
65#include "llvm/IR/Attributes.h"
66#include "llvm/IR/CallingConv.h"
67#include "llvm/IR/Constant.h"
68#include "llvm/IR/Constants.h"
69#include "llvm/IR/DataLayout.h"
70#include "llvm/IR/DebugLoc.h"
71#include "llvm/IR/DerivedTypes.h"
72#include "llvm/IR/Function.h"
73#include "llvm/IR/GlobalAlias.h"
74#include "llvm/IR/GlobalValue.h"
75#include "llvm/IR/GlobalVariable.h"
76#include "llvm/IR/IRBuilder.h"
77#include "llvm/IR/InlineAsm.h"
78#include "llvm/IR/Instruction.h"
79#include "llvm/IR/Instructions.h"
80#include "llvm/IR/IntrinsicInst.h"
81#include "llvm/IR/Intrinsics.h"
82#include "llvm/IR/IntrinsicsARM.h"
83#include "llvm/IR/Module.h"
84#include "llvm/IR/PatternMatch.h"
85#include "llvm/IR/Type.h"
86#include "llvm/IR/User.h"
87#include "llvm/IR/Value.h"
88#include "llvm/MC/MCInstrDesc.h"
89#include "llvm/MC/MCInstrItineraries.h"
90#include "llvm/MC/MCRegisterInfo.h"
91#include "llvm/MC/MCSchedule.h"
92#include "llvm/Support/AtomicOrdering.h"
93#include "llvm/Support/BranchProbability.h"
94#include "llvm/Support/Casting.h"
95#include "llvm/Support/CodeGen.h"
96#include "llvm/Support/CommandLine.h"
97#include "llvm/Support/Compiler.h"
98#include "llvm/Support/Debug.h"
99#include "llvm/Support/ErrorHandling.h"
100#include "llvm/Support/KnownBits.h"
101#include "llvm/Support/MachineValueType.h"
102#include "llvm/Support/MathExtras.h"
103#include "llvm/Support/raw_ostream.h"
104#include "llvm/Target/TargetMachine.h"
105#include "llvm/Target/TargetOptions.h"
106#include <algorithm>
107#include <cassert>
108#include <cstdint>
109#include <cstdlib>
110#include <iterator>
111#include <limits>
112#include <string>
113#include <tuple>
114#include <utility>
115#include <vector>
116
117using namespace llvm;
118using namespace llvm::PatternMatch;
119
120#define DEBUG_TYPE"arm-isel" "arm-isel"
121
122STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"arm-isel", "NumTailCalls"
, "Number of tail calls"}
;
123STATISTIC(NumMovwMovt, "Number of GAs materialized with movw + movt")static llvm::Statistic NumMovwMovt = {"arm-isel", "NumMovwMovt"
, "Number of GAs materialized with movw + movt"}
;
124STATISTIC(NumLoopByVals, "Number of loops generated for byval arguments")static llvm::Statistic NumLoopByVals = {"arm-isel", "NumLoopByVals"
, "Number of loops generated for byval arguments"}
;
125STATISTIC(NumConstpoolPromoted,static llvm::Statistic NumConstpoolPromoted = {"arm-isel", "NumConstpoolPromoted"
, "Number of constants with their storage promoted into constant pools"
}
126 "Number of constants with their storage promoted into constant pools")static llvm::Statistic NumConstpoolPromoted = {"arm-isel", "NumConstpoolPromoted"
, "Number of constants with their storage promoted into constant pools"
}
;
127
128static cl::opt<bool>
129ARMInterworking("arm-interworking", cl::Hidden,
130 cl::desc("Enable / disable ARM interworking (for debugging only)"),
131 cl::init(true));
132
133static cl::opt<bool> EnableConstpoolPromotion(
134 "arm-promote-constant", cl::Hidden,
135 cl::desc("Enable / disable promotion of unnamed_addr constants into "
136 "constant pools"),
137 cl::init(false)); // FIXME: set to true by default once PR32780 is fixed
138static cl::opt<unsigned> ConstpoolPromotionMaxSize(
139 "arm-promote-constant-max-size", cl::Hidden,
140 cl::desc("Maximum size of constant to promote into a constant pool"),
141 cl::init(64));
142static cl::opt<unsigned> ConstpoolPromotionMaxTotal(
143 "arm-promote-constant-max-total", cl::Hidden,
144 cl::desc("Maximum size of ALL constants to promote into a constant pool"),
145 cl::init(128));
146
147cl::opt<unsigned>
148MVEMaxSupportedInterleaveFactor("mve-max-interleave-factor", cl::Hidden,
149 cl::desc("Maximum interleave factor for MVE VLDn to generate."),
150 cl::init(2));
151
152// The APCS parameter registers.
153static const MCPhysReg GPRArgRegs[] = {
154 ARM::R0, ARM::R1, ARM::R2, ARM::R3
155};
156
157void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
158 MVT PromotedBitwiseVT) {
159 if (VT != PromotedLdStVT) {
160 setOperationAction(ISD::LOAD, VT, Promote);
161 AddPromotedToType (ISD::LOAD, VT, PromotedLdStVT);
162
163 setOperationAction(ISD::STORE, VT, Promote);
164 AddPromotedToType (ISD::STORE, VT, PromotedLdStVT);
165 }
166
167 MVT ElemTy = VT.getVectorElementType();
168 if (ElemTy != MVT::f64)
169 setOperationAction(ISD::SETCC, VT, Custom);
170 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
171 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
172 if (ElemTy == MVT::i32) {
173 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
174 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
175 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
176 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
177 } else {
178 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
179 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
180 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
181 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
182 }
183 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
184 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
185 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
186 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
187 setOperationAction(ISD::SELECT, VT, Expand);
188 setOperationAction(ISD::SELECT_CC, VT, Expand);
189 setOperationAction(ISD::VSELECT, VT, Expand);
190 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
191 if (VT.isInteger()) {
192 setOperationAction(ISD::SHL, VT, Custom);
193 setOperationAction(ISD::SRA, VT, Custom);
194 setOperationAction(ISD::SRL, VT, Custom);
195 }
196
197 // Promote all bit-wise operations.
198 if (VT.isInteger() && VT != PromotedBitwiseVT) {
199 setOperationAction(ISD::AND, VT, Promote);
200 AddPromotedToType (ISD::AND, VT, PromotedBitwiseVT);
201 setOperationAction(ISD::OR, VT, Promote);
202 AddPromotedToType (ISD::OR, VT, PromotedBitwiseVT);
203 setOperationAction(ISD::XOR, VT, Promote);
204 AddPromotedToType (ISD::XOR, VT, PromotedBitwiseVT);
205 }
206
207 // Neon does not support vector divide/remainder operations.
208 setOperationAction(ISD::SDIV, VT, Expand);
209 setOperationAction(ISD::UDIV, VT, Expand);
210 setOperationAction(ISD::FDIV, VT, Expand);
211 setOperationAction(ISD::SREM, VT, Expand);
212 setOperationAction(ISD::UREM, VT, Expand);
213 setOperationAction(ISD::FREM, VT, Expand);
214 setOperationAction(ISD::SDIVREM, VT, Expand);
215 setOperationAction(ISD::UDIVREM, VT, Expand);
216
217 if (!VT.isFloatingPoint() &&
218 VT != MVT::v2i64 && VT != MVT::v1i64)
219 for (auto Opcode : {ISD::ABS, ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
220 setOperationAction(Opcode, VT, Legal);
221 if (!VT.isFloatingPoint())
222 for (auto Opcode : {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT})
223 setOperationAction(Opcode, VT, Legal);
224}
225
226void ARMTargetLowering::addDRTypeForNEON(MVT VT) {
227 addRegisterClass(VT, &ARM::DPRRegClass);
228 addTypeForNEON(VT, MVT::f64, MVT::v2i32);
229}
230
231void ARMTargetLowering::addQRTypeForNEON(MVT VT) {
232 addRegisterClass(VT, &ARM::DPairRegClass);
233 addTypeForNEON(VT, MVT::v2f64, MVT::v4i32);
234}
235
236void ARMTargetLowering::setAllExpand(MVT VT) {
237 for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
238 setOperationAction(Opc, VT, Expand);
239
240 // We support these really simple operations even on types where all
241 // the actual arithmetic has to be broken down into simpler
242 // operations or turned into library calls.
243 setOperationAction(ISD::BITCAST, VT, Legal);
244 setOperationAction(ISD::LOAD, VT, Legal);
245 setOperationAction(ISD::STORE, VT, Legal);
246 setOperationAction(ISD::UNDEF, VT, Legal);
247}
248
249void ARMTargetLowering::addAllExtLoads(const MVT From, const MVT To,
250 LegalizeAction Action) {
251 setLoadExtAction(ISD::EXTLOAD, From, To, Action);
252 setLoadExtAction(ISD::ZEXTLOAD, From, To, Action);
253 setLoadExtAction(ISD::SEXTLOAD, From, To, Action);
254}
255
256void ARMTargetLowering::addMVEVectorTypes(bool HasMVEFP) {
257 const MVT IntTypes[] = { MVT::v16i8, MVT::v8i16, MVT::v4i32 };
258
259 for (auto VT : IntTypes) {
260 addRegisterClass(VT, &ARM::MQPRRegClass);
261 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
262 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
263 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
264 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
265 setOperationAction(ISD::SHL, VT, Custom);
266 setOperationAction(ISD::SRA, VT, Custom);
267 setOperationAction(ISD::SRL, VT, Custom);
268 setOperationAction(ISD::SMIN, VT, Legal);
269 setOperationAction(ISD::SMAX, VT, Legal);
270 setOperationAction(ISD::UMIN, VT, Legal);
271 setOperationAction(ISD::UMAX, VT, Legal);
272 setOperationAction(ISD::ABS, VT, Legal);
273 setOperationAction(ISD::SETCC, VT, Custom);
274 setOperationAction(ISD::MLOAD, VT, Custom);
275 setOperationAction(ISD::MSTORE, VT, Legal);
276 setOperationAction(ISD::CTLZ, VT, Legal);
277 setOperationAction(ISD::CTTZ, VT, Custom);
278 setOperationAction(ISD::BITREVERSE, VT, Legal);
279 setOperationAction(ISD::BSWAP, VT, Legal);
280 setOperationAction(ISD::SADDSAT, VT, Legal);
281 setOperationAction(ISD::UADDSAT, VT, Legal);
282 setOperationAction(ISD::SSUBSAT, VT, Legal);
283 setOperationAction(ISD::USUBSAT, VT, Legal);
284
285 // No native support for these.
286 setOperationAction(ISD::UDIV, VT, Expand);
287 setOperationAction(ISD::SDIV, VT, Expand);
288 setOperationAction(ISD::UREM, VT, Expand);
289 setOperationAction(ISD::SREM, VT, Expand);
290 setOperationAction(ISD::UDIVREM, VT, Expand);
291 setOperationAction(ISD::SDIVREM, VT, Expand);
292 setOperationAction(ISD::CTPOP, VT, Expand);
293 setOperationAction(ISD::SELECT, VT, Expand);
294 setOperationAction(ISD::SELECT_CC, VT, Expand);
295
296 // Vector reductions
297 setOperationAction(ISD::VECREDUCE_ADD, VT, Legal);
298 setOperationAction(ISD::VECREDUCE_SMAX, VT, Legal);
299 setOperationAction(ISD::VECREDUCE_UMAX, VT, Legal);
300 setOperationAction(ISD::VECREDUCE_SMIN, VT, Legal);
301 setOperationAction(ISD::VECREDUCE_UMIN, VT, Legal);
302 setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
303 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
304 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
305 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
306
307 if (!HasMVEFP) {
308 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
309 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
310 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
311 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
312 }
313
314 // Pre and Post inc are supported on loads and stores
315 for (unsigned im = (unsigned)ISD::PRE_INC;
316 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
317 setIndexedLoadAction(im, VT, Legal);
318 setIndexedStoreAction(im, VT, Legal);
319 setIndexedMaskedLoadAction(im, VT, Legal);
320 setIndexedMaskedStoreAction(im, VT, Legal);
321 }
322 }
323
324 const MVT FloatTypes[] = { MVT::v8f16, MVT::v4f32 };
325 for (auto VT : FloatTypes) {
326 addRegisterClass(VT, &ARM::MQPRRegClass);
327 if (!HasMVEFP)
328 setAllExpand(VT);
329
330 // These are legal or custom whether we have MVE.fp or not
331 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
332 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
333 setOperationAction(ISD::INSERT_VECTOR_ELT, VT.getVectorElementType(), Custom);
334 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
335 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
336 setOperationAction(ISD::BUILD_VECTOR, VT.getVectorElementType(), Custom);
337 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Legal);
338 setOperationAction(ISD::SETCC, VT, Custom);
339 setOperationAction(ISD::MLOAD, VT, Custom);
340 setOperationAction(ISD::MSTORE, VT, Legal);
341 setOperationAction(ISD::SELECT, VT, Expand);
342 setOperationAction(ISD::SELECT_CC, VT, Expand);
343
344 // Pre and Post inc are supported on loads and stores
345 for (unsigned im = (unsigned)ISD::PRE_INC;
346 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
347 setIndexedLoadAction(im, VT, Legal);
348 setIndexedStoreAction(im, VT, Legal);
349 setIndexedMaskedLoadAction(im, VT, Legal);
350 setIndexedMaskedStoreAction(im, VT, Legal);
351 }
352
353 if (HasMVEFP) {
354 setOperationAction(ISD::FMINNUM, VT, Legal);
355 setOperationAction(ISD::FMAXNUM, VT, Legal);
356 setOperationAction(ISD::FROUND, VT, Legal);
357 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
358 setOperationAction(ISD::VECREDUCE_FMUL, VT, Custom);
359 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
360 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
361
362 // No native support for these.
363 setOperationAction(ISD::FDIV, VT, Expand);
364 setOperationAction(ISD::FREM, VT, Expand);
365 setOperationAction(ISD::FSQRT, VT, Expand);
366 setOperationAction(ISD::FSIN, VT, Expand);
367 setOperationAction(ISD::FCOS, VT, Expand);
368 setOperationAction(ISD::FPOW, VT, Expand);
369 setOperationAction(ISD::FLOG, VT, Expand);
370 setOperationAction(ISD::FLOG2, VT, Expand);
371 setOperationAction(ISD::FLOG10, VT, Expand);
372 setOperationAction(ISD::FEXP, VT, Expand);
373 setOperationAction(ISD::FEXP2, VT, Expand);
374 setOperationAction(ISD::FNEARBYINT, VT, Expand);
375 }
376 }
377
378 // Custom Expand smaller than legal vector reductions to prevent false zero
379 // items being added.
380 setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f16, Custom);
381 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v4f16, Custom);
382 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v4f16, Custom);
383 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v4f16, Custom);
384 setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f16, Custom);
385 setOperationAction(ISD::VECREDUCE_FMUL, MVT::v2f16, Custom);
386 setOperationAction(ISD::VECREDUCE_FMIN, MVT::v2f16, Custom);
387 setOperationAction(ISD::VECREDUCE_FMAX, MVT::v2f16, Custom);
388
389 // We 'support' these types up to bitcast/load/store level, regardless of
390 // MVE integer-only / float support. Only doing FP data processing on the FP
391 // vector types is inhibited at integer-only level.
392 const MVT LongTypes[] = { MVT::v2i64, MVT::v2f64 };
393 for (auto VT : LongTypes) {
394 addRegisterClass(VT, &ARM::MQPRRegClass);
395 setAllExpand(VT);
396 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
397 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
398 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
399 }
400 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v2f64, Legal);
401
402 // We can do bitwise operations on v2i64 vectors
403 setOperationAction(ISD::AND, MVT::v2i64, Legal);
404 setOperationAction(ISD::OR, MVT::v2i64, Legal);
405 setOperationAction(ISD::XOR, MVT::v2i64, Legal);
406
407 // It is legal to extload from v4i8 to v4i16 or v4i32.
408 addAllExtLoads(MVT::v8i16, MVT::v8i8, Legal);
409 addAllExtLoads(MVT::v4i32, MVT::v4i16, Legal);
410 addAllExtLoads(MVT::v4i32, MVT::v4i8, Legal);
411
412 // It is legal to sign extend from v4i8/v4i16 to v4i32 or v8i8 to v8i16.
413 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i8, Legal);
414 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i16, Legal);
415 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v4i32, Legal);
416 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i8, Legal);
417 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::v8i16, Legal);
418
419 // Some truncating stores are legal too.
420 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
421 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
422 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
423
424 // Pre and Post inc on these are legal, given the correct extends
425 for (unsigned im = (unsigned)ISD::PRE_INC;
426 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
427 for (auto VT : {MVT::v8i8, MVT::v4i8, MVT::v4i16}) {
428 setIndexedLoadAction(im, VT, Legal);
429 setIndexedStoreAction(im, VT, Legal);
430 setIndexedMaskedLoadAction(im, VT, Legal);
431 setIndexedMaskedStoreAction(im, VT, Legal);
432 }
433 }
434
435 // Predicate types
436 const MVT pTypes[] = {MVT::v16i1, MVT::v8i1, MVT::v4i1};
437 for (auto VT : pTypes) {
438 addRegisterClass(VT, &ARM::VCCRRegClass);
439 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
440 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
441 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
442 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
443 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
444 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
445 setOperationAction(ISD::SETCC, VT, Custom);
446 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Expand);
447 setOperationAction(ISD::LOAD, VT, Custom);
448 setOperationAction(ISD::STORE, VT, Custom);
449 setOperationAction(ISD::TRUNCATE, VT, Custom);
450 setOperationAction(ISD::VSELECT, VT, Expand);
451 setOperationAction(ISD::SELECT, VT, Expand);
452 }
453}
454
455ARMTargetLowering::ARMTargetLowering(const TargetMachine &TM,
456 const ARMSubtarget &STI)
457 : TargetLowering(TM), Subtarget(&STI) {
458 RegInfo = Subtarget->getRegisterInfo();
459 Itins = Subtarget->getInstrItineraryData();
460
461 setBooleanContents(ZeroOrOneBooleanContent);
462 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
463
464 if (!Subtarget->isTargetDarwin() && !Subtarget->isTargetIOS() &&
465 !Subtarget->isTargetWatchOS()) {
466 bool IsHFTarget = TM.Options.FloatABIType == FloatABI::Hard;
467 for (int LCID = 0; LCID < RTLIB::UNKNOWN_LIBCALL; ++LCID)
468 setLibcallCallingConv(static_cast<RTLIB::Libcall>(LCID),
469 IsHFTarget ? CallingConv::ARM_AAPCS_VFP
470 : CallingConv::ARM_AAPCS);
471 }
472
473 if (Subtarget->isTargetMachO()) {
474 // Uses VFP for Thumb libfuncs if available.
475 if (Subtarget->isThumb() && Subtarget->hasVFP2Base() &&
476 Subtarget->hasARMOps() && !Subtarget->useSoftFloat()) {
477 static const struct {
478 const RTLIB::Libcall Op;
479 const char * const Name;
480 const ISD::CondCode Cond;
481 } LibraryCalls[] = {
482 // Single-precision floating-point arithmetic.
483 { RTLIB::ADD_F32, "__addsf3vfp", ISD::SETCC_INVALID },
484 { RTLIB::SUB_F32, "__subsf3vfp", ISD::SETCC_INVALID },
485 { RTLIB::MUL_F32, "__mulsf3vfp", ISD::SETCC_INVALID },
486 { RTLIB::DIV_F32, "__divsf3vfp", ISD::SETCC_INVALID },
487
488 // Double-precision floating-point arithmetic.
489 { RTLIB::ADD_F64, "__adddf3vfp", ISD::SETCC_INVALID },
490 { RTLIB::SUB_F64, "__subdf3vfp", ISD::SETCC_INVALID },
491 { RTLIB::MUL_F64, "__muldf3vfp", ISD::SETCC_INVALID },
492 { RTLIB::DIV_F64, "__divdf3vfp", ISD::SETCC_INVALID },
493
494 // Single-precision comparisons.
495 { RTLIB::OEQ_F32, "__eqsf2vfp", ISD::SETNE },
496 { RTLIB::UNE_F32, "__nesf2vfp", ISD::SETNE },
497 { RTLIB::OLT_F32, "__ltsf2vfp", ISD::SETNE },
498 { RTLIB::OLE_F32, "__lesf2vfp", ISD::SETNE },
499 { RTLIB::OGE_F32, "__gesf2vfp", ISD::SETNE },
500 { RTLIB::OGT_F32, "__gtsf2vfp", ISD::SETNE },
501 { RTLIB::UO_F32, "__unordsf2vfp", ISD::SETNE },
502
503 // Double-precision comparisons.
504 { RTLIB::OEQ_F64, "__eqdf2vfp", ISD::SETNE },
505 { RTLIB::UNE_F64, "__nedf2vfp", ISD::SETNE },
506 { RTLIB::OLT_F64, "__ltdf2vfp", ISD::SETNE },
507 { RTLIB::OLE_F64, "__ledf2vfp", ISD::SETNE },
508 { RTLIB::OGE_F64, "__gedf2vfp", ISD::SETNE },
509 { RTLIB::OGT_F64, "__gtdf2vfp", ISD::SETNE },
510 { RTLIB::UO_F64, "__unorddf2vfp", ISD::SETNE },
511
512 // Floating-point to integer conversions.
513 // i64 conversions are done via library routines even when generating VFP
514 // instructions, so use the same ones.
515 { RTLIB::FPTOSINT_F64_I32, "__fixdfsivfp", ISD::SETCC_INVALID },
516 { RTLIB::FPTOUINT_F64_I32, "__fixunsdfsivfp", ISD::SETCC_INVALID },
517 { RTLIB::FPTOSINT_F32_I32, "__fixsfsivfp", ISD::SETCC_INVALID },
518 { RTLIB::FPTOUINT_F32_I32, "__fixunssfsivfp", ISD::SETCC_INVALID },
519
520 // Conversions between floating types.
521 { RTLIB::FPROUND_F64_F32, "__truncdfsf2vfp", ISD::SETCC_INVALID },
522 { RTLIB::FPEXT_F32_F64, "__extendsfdf2vfp", ISD::SETCC_INVALID },
523
524 // Integer to floating-point conversions.
525 // i64 conversions are done via library routines even when generating VFP
526 // instructions, so use the same ones.
527 // FIXME: There appears to be some naming inconsistency in ARM libgcc:
528 // e.g., __floatunsidf vs. __floatunssidfvfp.
529 { RTLIB::SINTTOFP_I32_F64, "__floatsidfvfp", ISD::SETCC_INVALID },
530 { RTLIB::UINTTOFP_I32_F64, "__floatunssidfvfp", ISD::SETCC_INVALID },
531 { RTLIB::SINTTOFP_I32_F32, "__floatsisfvfp", ISD::SETCC_INVALID },
532 { RTLIB::UINTTOFP_I32_F32, "__floatunssisfvfp", ISD::SETCC_INVALID },
533 };
534
535 for (const auto &LC : LibraryCalls) {
536 setLibcallName(LC.Op, LC.Name);
537 if (LC.Cond != ISD::SETCC_INVALID)
538 setCmpLibcallCC(LC.Op, LC.Cond);
539 }
540 }
541 }
542
543 // These libcalls are not available in 32-bit.
544 setLibcallName(RTLIB::SHL_I128, nullptr);
545 setLibcallName(RTLIB::SRL_I128, nullptr);
546 setLibcallName(RTLIB::SRA_I128, nullptr);
547
548 // RTLIB
549 if (Subtarget->isAAPCS_ABI() &&
550 (Subtarget->isTargetAEABI() || Subtarget->isTargetGNUAEABI() ||
551 Subtarget->isTargetMuslAEABI() || Subtarget->isTargetAndroid())) {
552 static const struct {
553 const RTLIB::Libcall Op;
554 const char * const Name;
555 const CallingConv::ID CC;
556 const ISD::CondCode Cond;
557 } LibraryCalls[] = {
558 // Double-precision floating-point arithmetic helper functions
559 // RTABI chapter 4.1.2, Table 2
560 { RTLIB::ADD_F64, "__aeabi_dadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
561 { RTLIB::DIV_F64, "__aeabi_ddiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
562 { RTLIB::MUL_F64, "__aeabi_dmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
563 { RTLIB::SUB_F64, "__aeabi_dsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
564
565 // Double-precision floating-point comparison helper functions
566 // RTABI chapter 4.1.2, Table 3
567 { RTLIB::OEQ_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
568 { RTLIB::UNE_F64, "__aeabi_dcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
569 { RTLIB::OLT_F64, "__aeabi_dcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
570 { RTLIB::OLE_F64, "__aeabi_dcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
571 { RTLIB::OGE_F64, "__aeabi_dcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
572 { RTLIB::OGT_F64, "__aeabi_dcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
573 { RTLIB::UO_F64, "__aeabi_dcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
574
575 // Single-precision floating-point arithmetic helper functions
576 // RTABI chapter 4.1.2, Table 4
577 { RTLIB::ADD_F32, "__aeabi_fadd", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
578 { RTLIB::DIV_F32, "__aeabi_fdiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
579 { RTLIB::MUL_F32, "__aeabi_fmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
580 { RTLIB::SUB_F32, "__aeabi_fsub", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
581
582 // Single-precision floating-point comparison helper functions
583 // RTABI chapter 4.1.2, Table 5
584 { RTLIB::OEQ_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETNE },
585 { RTLIB::UNE_F32, "__aeabi_fcmpeq", CallingConv::ARM_AAPCS, ISD::SETEQ },
586 { RTLIB::OLT_F32, "__aeabi_fcmplt", CallingConv::ARM_AAPCS, ISD::SETNE },
587 { RTLIB::OLE_F32, "__aeabi_fcmple", CallingConv::ARM_AAPCS, ISD::SETNE },
588 { RTLIB::OGE_F32, "__aeabi_fcmpge", CallingConv::ARM_AAPCS, ISD::SETNE },
589 { RTLIB::OGT_F32, "__aeabi_fcmpgt", CallingConv::ARM_AAPCS, ISD::SETNE },
590 { RTLIB::UO_F32, "__aeabi_fcmpun", CallingConv::ARM_AAPCS, ISD::SETNE },
591
592 // Floating-point to integer conversions.
593 // RTABI chapter 4.1.2, Table 6
594 { RTLIB::FPTOSINT_F64_I32, "__aeabi_d2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
595 { RTLIB::FPTOUINT_F64_I32, "__aeabi_d2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
596 { RTLIB::FPTOSINT_F64_I64, "__aeabi_d2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
597 { RTLIB::FPTOUINT_F64_I64, "__aeabi_d2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
598 { RTLIB::FPTOSINT_F32_I32, "__aeabi_f2iz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
599 { RTLIB::FPTOUINT_F32_I32, "__aeabi_f2uiz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
600 { RTLIB::FPTOSINT_F32_I64, "__aeabi_f2lz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
601 { RTLIB::FPTOUINT_F32_I64, "__aeabi_f2ulz", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
602
603 // Conversions between floating types.
604 // RTABI chapter 4.1.2, Table 7
605 { RTLIB::FPROUND_F64_F32, "__aeabi_d2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
606 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
607 { RTLIB::FPEXT_F32_F64, "__aeabi_f2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
608
609 // Integer to floating-point conversions.
610 // RTABI chapter 4.1.2, Table 8
611 { RTLIB::SINTTOFP_I32_F64, "__aeabi_i2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
612 { RTLIB::UINTTOFP_I32_F64, "__aeabi_ui2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
613 { RTLIB::SINTTOFP_I64_F64, "__aeabi_l2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
614 { RTLIB::UINTTOFP_I64_F64, "__aeabi_ul2d", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
615 { RTLIB::SINTTOFP_I32_F32, "__aeabi_i2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
616 { RTLIB::UINTTOFP_I32_F32, "__aeabi_ui2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
617 { RTLIB::SINTTOFP_I64_F32, "__aeabi_l2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
618 { RTLIB::UINTTOFP_I64_F32, "__aeabi_ul2f", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
619
620 // Long long helper functions
621 // RTABI chapter 4.2, Table 9
622 { RTLIB::MUL_I64, "__aeabi_lmul", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
623 { RTLIB::SHL_I64, "__aeabi_llsl", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
624 { RTLIB::SRL_I64, "__aeabi_llsr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
625 { RTLIB::SRA_I64, "__aeabi_lasr", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
626
627 // Integer division functions
628 // RTABI chapter 4.3.1
629 { RTLIB::SDIV_I8, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
630 { RTLIB::SDIV_I16, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
631 { RTLIB::SDIV_I32, "__aeabi_idiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
632 { RTLIB::SDIV_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
633 { RTLIB::UDIV_I8, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
634 { RTLIB::UDIV_I16, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
635 { RTLIB::UDIV_I32, "__aeabi_uidiv", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
636 { RTLIB::UDIV_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
637 };
638
639 for (const auto &LC : LibraryCalls) {
640 setLibcallName(LC.Op, LC.Name);
641 setLibcallCallingConv(LC.Op, LC.CC);
642 if (LC.Cond != ISD::SETCC_INVALID)
643 setCmpLibcallCC(LC.Op, LC.Cond);
644 }
645
646 // EABI dependent RTLIB
647 if (TM.Options.EABIVersion == EABI::EABI4 ||
648 TM.Options.EABIVersion == EABI::EABI5) {
649 static const struct {
650 const RTLIB::Libcall Op;
651 const char *const Name;
652 const CallingConv::ID CC;
653 const ISD::CondCode Cond;
654 } MemOpsLibraryCalls[] = {
655 // Memory operations
656 // RTABI chapter 4.3.4
657 { RTLIB::MEMCPY, "__aeabi_memcpy", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
658 { RTLIB::MEMMOVE, "__aeabi_memmove", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
659 { RTLIB::MEMSET, "__aeabi_memset", CallingConv::ARM_AAPCS, ISD::SETCC_INVALID },
660 };
661
662 for (const auto &LC : MemOpsLibraryCalls) {
663 setLibcallName(LC.Op, LC.Name);
664 setLibcallCallingConv(LC.Op, LC.CC);
665 if (LC.Cond != ISD::SETCC_INVALID)
666 setCmpLibcallCC(LC.Op, LC.Cond);
667 }
668 }
669 }
670
671 if (Subtarget->isTargetWindows()) {
672 static const struct {
673 const RTLIB::Libcall Op;
674 const char * const Name;
675 const CallingConv::ID CC;
676 } LibraryCalls[] = {
677 { RTLIB::FPTOSINT_F32_I64, "__stoi64", CallingConv::ARM_AAPCS_VFP },
678 { RTLIB::FPTOSINT_F64_I64, "__dtoi64", CallingConv::ARM_AAPCS_VFP },
679 { RTLIB::FPTOUINT_F32_I64, "__stou64", CallingConv::ARM_AAPCS_VFP },
680 { RTLIB::FPTOUINT_F64_I64, "__dtou64", CallingConv::ARM_AAPCS_VFP },
681 { RTLIB::SINTTOFP_I64_F32, "__i64tos", CallingConv::ARM_AAPCS_VFP },
682 { RTLIB::SINTTOFP_I64_F64, "__i64tod", CallingConv::ARM_AAPCS_VFP },
683 { RTLIB::UINTTOFP_I64_F32, "__u64tos", CallingConv::ARM_AAPCS_VFP },
684 { RTLIB::UINTTOFP_I64_F64, "__u64tod", CallingConv::ARM_AAPCS_VFP },
685 };
686
687 for (const auto &LC : LibraryCalls) {
688 setLibcallName(LC.Op, LC.Name);
689 setLibcallCallingConv(LC.Op, LC.CC);
690 }
691 }
692
693 // Use divmod compiler-rt calls for iOS 5.0 and later.
694 if (Subtarget->isTargetMachO() &&
695 !(Subtarget->isTargetIOS() &&
696 Subtarget->getTargetTriple().isOSVersionLT(5, 0))) {
697 setLibcallName(RTLIB::SDIVREM_I32, "__divmodsi4");
698 setLibcallName(RTLIB::UDIVREM_I32, "__udivmodsi4");
699 }
700
701 // The half <-> float conversion functions are always soft-float on
702 // non-watchos platforms, but are needed for some targets which use a
703 // hard-float calling convention by default.
704 if (!Subtarget->isTargetWatchABI()) {
705 if (Subtarget->isAAPCS_ABI()) {
706 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_AAPCS);
707 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_AAPCS);
708 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_AAPCS);
709 } else {
710 setLibcallCallingConv(RTLIB::FPROUND_F32_F16, CallingConv::ARM_APCS);
711 setLibcallCallingConv(RTLIB::FPROUND_F64_F16, CallingConv::ARM_APCS);
712 setLibcallCallingConv(RTLIB::FPEXT_F16_F32, CallingConv::ARM_APCS);
713 }
714 }
715
716 // In EABI, these functions have an __aeabi_ prefix, but in GNUEABI they have
717 // a __gnu_ prefix (which is the default).
718 if (Subtarget->isTargetAEABI()) {
719 static const struct {
720 const RTLIB::Libcall Op;
721 const char * const Name;
722 const CallingConv::ID CC;
723 } LibraryCalls[] = {
724 { RTLIB::FPROUND_F32_F16, "__aeabi_f2h", CallingConv::ARM_AAPCS },
725 { RTLIB::FPROUND_F64_F16, "__aeabi_d2h", CallingConv::ARM_AAPCS },
726 { RTLIB::FPEXT_F16_F32, "__aeabi_h2f", CallingConv::ARM_AAPCS },
727 };
728
729 for (const auto &LC : LibraryCalls) {
730 setLibcallName(LC.Op, LC.Name);
731 setLibcallCallingConv(LC.Op, LC.CC);
732 }
733 }
734
735 if (Subtarget->isThumb1Only())
736 addRegisterClass(MVT::i32, &ARM::tGPRRegClass);
737 else
738 addRegisterClass(MVT::i32, &ARM::GPRRegClass);
739
740 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only() &&
741 Subtarget->hasFPRegs()) {
742 addRegisterClass(MVT::f32, &ARM::SPRRegClass);
743 addRegisterClass(MVT::f64, &ARM::DPRRegClass);
744 if (!Subtarget->hasVFP2Base())
745 setAllExpand(MVT::f32);
746 if (!Subtarget->hasFP64())
747 setAllExpand(MVT::f64);
748 }
749
750 if (Subtarget->hasFullFP16()) {
751 addRegisterClass(MVT::f16, &ARM::HPRRegClass);
752 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
753 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
754
755 setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
756 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
757 }
758
759 if (Subtarget->hasBF16()) {
760 addRegisterClass(MVT::bf16, &ARM::HPRRegClass);
761 setAllExpand(MVT::bf16);
762 if (!Subtarget->hasFullFP16())
763 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
764 }
765
766 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
767 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
768 setTruncStoreAction(VT, InnerVT, Expand);
769 addAllExtLoads(VT, InnerVT, Expand);
770 }
771
772 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
773 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
774
775 setOperationAction(ISD::BSWAP, VT, Expand);
776 }
777
778 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
779 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
780
781 setOperationAction(ISD::READ_REGISTER, MVT::i64, Custom);
782 setOperationAction(ISD::WRITE_REGISTER, MVT::i64, Custom);
783
784 if (Subtarget->hasMVEIntegerOps())
785 addMVEVectorTypes(Subtarget->hasMVEFloatOps());
786
787 // Combine low-overhead loop intrinsics so that we can lower i1 types.
788 if (Subtarget->hasLOB()) {
789 setTargetDAGCombine(ISD::BRCOND);
790 setTargetDAGCombine(ISD::BR_CC);
791 }
792
793 if (Subtarget->hasNEON()) {
794 addDRTypeForNEON(MVT::v2f32);
795 addDRTypeForNEON(MVT::v8i8);
796 addDRTypeForNEON(MVT::v4i16);
797 addDRTypeForNEON(MVT::v2i32);
798 addDRTypeForNEON(MVT::v1i64);
799
800 addQRTypeForNEON(MVT::v4f32);
801 addQRTypeForNEON(MVT::v2f64);
802 addQRTypeForNEON(MVT::v16i8);
803 addQRTypeForNEON(MVT::v8i16);
804 addQRTypeForNEON(MVT::v4i32);
805 addQRTypeForNEON(MVT::v2i64);
806
807 if (Subtarget->hasFullFP16()) {
808 addQRTypeForNEON(MVT::v8f16);
809 addDRTypeForNEON(MVT::v4f16);
810 }
811
812 if (Subtarget->hasBF16()) {
813 addQRTypeForNEON(MVT::v8bf16);
814 addDRTypeForNEON(MVT::v4bf16);
815 }
816 }
817
818 if (Subtarget->hasMVEIntegerOps() || Subtarget->hasNEON()) {
819 // v2f64 is legal so that QR subregs can be extracted as f64 elements, but
820 // none of Neon, MVE or VFP supports any arithmetic operations on it.
821 setOperationAction(ISD::FADD, MVT::v2f64, Expand);
822 setOperationAction(ISD::FSUB, MVT::v2f64, Expand);
823 setOperationAction(ISD::FMUL, MVT::v2f64, Expand);
824 // FIXME: Code duplication: FDIV and FREM are expanded always, see
825 // ARMTargetLowering::addTypeForNEON method for details.
826 setOperationAction(ISD::FDIV, MVT::v2f64, Expand);
827 setOperationAction(ISD::FREM, MVT::v2f64, Expand);
828 // FIXME: Create unittest.
829 // In another words, find a way when "copysign" appears in DAG with vector
830 // operands.
831 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Expand);
832 // FIXME: Code duplication: SETCC has custom operation action, see
833 // ARMTargetLowering::addTypeForNEON method for details.
834 setOperationAction(ISD::SETCC, MVT::v2f64, Expand);
835 // FIXME: Create unittest for FNEG and for FABS.
836 setOperationAction(ISD::FNEG, MVT::v2f64, Expand);
837 setOperationAction(ISD::FABS, MVT::v2f64, Expand);
838 setOperationAction(ISD::FSQRT, MVT::v2f64, Expand);
839 setOperationAction(ISD::FSIN, MVT::v2f64, Expand);
840 setOperationAction(ISD::FCOS, MVT::v2f64, Expand);
841 setOperationAction(ISD::FPOW, MVT::v2f64, Expand);
842 setOperationAction(ISD::FLOG, MVT::v2f64, Expand);
843 setOperationAction(ISD::FLOG2, MVT::v2f64, Expand);
844 setOperationAction(ISD::FLOG10, MVT::v2f64, Expand);
845 setOperationAction(ISD::FEXP, MVT::v2f64, Expand);
846 setOperationAction(ISD::FEXP2, MVT::v2f64, Expand);
847 // FIXME: Create unittest for FCEIL, FTRUNC, FRINT, FNEARBYINT, FFLOOR.
848 setOperationAction(ISD::FCEIL, MVT::v2f64, Expand);
849 setOperationAction(ISD::FTRUNC, MVT::v2f64, Expand);
850 setOperationAction(ISD::FRINT, MVT::v2f64, Expand);
851 setOperationAction(ISD::FNEARBYINT, MVT::v2f64, Expand);
852 setOperationAction(ISD::FFLOOR, MVT::v2f64, Expand);
853 setOperationAction(ISD::FMA, MVT::v2f64, Expand);
854 }
855
856 if (Subtarget->hasNEON()) {
857 // The same with v4f32. But keep in mind that vadd, vsub, vmul are natively
858 // supported for v4f32.
859 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
860 setOperationAction(ISD::FSIN, MVT::v4f32, Expand);
861 setOperationAction(ISD::FCOS, MVT::v4f32, Expand);
862 setOperationAction(ISD::FPOW, MVT::v4f32, Expand);
863 setOperationAction(ISD::FLOG, MVT::v4f32, Expand);
864 setOperationAction(ISD::FLOG2, MVT::v4f32, Expand);
865 setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
866 setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
867 setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
868 setOperationAction(ISD::FCEIL, MVT::v4f32, Expand);
869 setOperationAction(ISD::FTRUNC, MVT::v4f32, Expand);
870 setOperationAction(ISD::FRINT, MVT::v4f32, Expand);
871 setOperationAction(ISD::FNEARBYINT, MVT::v4f32, Expand);
872 setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
873
874 // Mark v2f32 intrinsics.
875 setOperationAction(ISD::FSQRT, MVT::v2f32, Expand);
876 setOperationAction(ISD::FSIN, MVT::v2f32, Expand);
877 setOperationAction(ISD::FCOS, MVT::v2f32, Expand);
878 setOperationAction(ISD::FPOW, MVT::v2f32, Expand);
879 setOperationAction(ISD::FLOG, MVT::v2f32, Expand);
880 setOperationAction(ISD::FLOG2, MVT::v2f32, Expand);
881 setOperationAction(ISD::FLOG10, MVT::v2f32, Expand);
882 setOperationAction(ISD::FEXP, MVT::v2f32, Expand);
883 setOperationAction(ISD::FEXP2, MVT::v2f32, Expand);
884 setOperationAction(ISD::FCEIL, MVT::v2f32, Expand);
885 setOperationAction(ISD::FTRUNC, MVT::v2f32, Expand);
886 setOperationAction(ISD::FRINT, MVT::v2f32, Expand);
887 setOperationAction(ISD::FNEARBYINT, MVT::v2f32, Expand);
888 setOperationAction(ISD::FFLOOR, MVT::v2f32, Expand);
889
890 // Neon does not support some operations on v1i64 and v2i64 types.
891 setOperationAction(ISD::MUL, MVT::v1i64, Expand);
892 // Custom handling for some quad-vector types to detect VMULL.
893 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
894 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
895 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
896 // Custom handling for some vector types to avoid expensive expansions
897 setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
898 setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
899 setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
900 setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
901 // Neon does not have single instruction SINT_TO_FP and UINT_TO_FP with
902 // a destination type that is wider than the source, and nor does
903 // it have a FP_TO_[SU]INT instruction with a narrower destination than
904 // source.
905 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
906 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
907 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
908 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
909 setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
910 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
911 setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
912 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
913
914 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Expand);
915 setOperationAction(ISD::FP_EXTEND, MVT::v2f64, Expand);
916
917 // NEON does not have single instruction CTPOP for vectors with element
918 // types wider than 8-bits. However, custom lowering can leverage the
919 // v8i8/v16i8 vcnt instruction.
920 setOperationAction(ISD::CTPOP, MVT::v2i32, Custom);
921 setOperationAction(ISD::CTPOP, MVT::v4i32, Custom);
922 setOperationAction(ISD::CTPOP, MVT::v4i16, Custom);
923 setOperationAction(ISD::CTPOP, MVT::v8i16, Custom);
924 setOperationAction(ISD::CTPOP, MVT::v1i64, Custom);
925 setOperationAction(ISD::CTPOP, MVT::v2i64, Custom);
926
927 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
928 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
929
930 // NEON does not have single instruction CTTZ for vectors.
931 setOperationAction(ISD::CTTZ, MVT::v8i8, Custom);
932 setOperationAction(ISD::CTTZ, MVT::v4i16, Custom);
933 setOperationAction(ISD::CTTZ, MVT::v2i32, Custom);
934 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
935
936 setOperationAction(ISD::CTTZ, MVT::v16i8, Custom);
937 setOperationAction(ISD::CTTZ, MVT::v8i16, Custom);
938 setOperationAction(ISD::CTTZ, MVT::v4i32, Custom);
939 setOperationAction(ISD::CTTZ, MVT::v2i64, Custom);
940
941 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i8, Custom);
942 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i16, Custom);
943 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i32, Custom);
944 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v1i64, Custom);
945
946 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v16i8, Custom);
947 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v8i16, Custom);
948 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v4i32, Custom);
949 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::v2i64, Custom);
950
951 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
952 setOperationAction(ISD::MULHS, VT, Expand);
953 setOperationAction(ISD::MULHU, VT, Expand);
954 }
955
956 // NEON only has FMA instructions as of VFP4.
957 if (!Subtarget->hasVFP4Base()) {
958 setOperationAction(ISD::FMA, MVT::v2f32, Expand);
959 setOperationAction(ISD::FMA, MVT::v4f32, Expand);
960 }
961
962 setTargetDAGCombine(ISD::SHL);
963 setTargetDAGCombine(ISD::SRL);
964 setTargetDAGCombine(ISD::SRA);
965 setTargetDAGCombine(ISD::FP_TO_SINT);
966 setTargetDAGCombine(ISD::FP_TO_UINT);
967 setTargetDAGCombine(ISD::FDIV);
968 setTargetDAGCombine(ISD::LOAD);
969
970 // It is legal to extload from v4i8 to v4i16 or v4i32.
971 for (MVT Ty : {MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v4i16, MVT::v2i16,
972 MVT::v2i32}) {
973 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
974 setLoadExtAction(ISD::EXTLOAD, VT, Ty, Legal);
975 setLoadExtAction(ISD::ZEXTLOAD, VT, Ty, Legal);
976 setLoadExtAction(ISD::SEXTLOAD, VT, Ty, Legal);
977 }
978 }
979 }
980
981 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
982 setTargetDAGCombine(ISD::BUILD_VECTOR);
983 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
984 setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
985 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
986 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
987 setTargetDAGCombine(ISD::STORE);
988 setTargetDAGCombine(ISD::SIGN_EXTEND);
989 setTargetDAGCombine(ISD::ZERO_EXTEND);
990 setTargetDAGCombine(ISD::ANY_EXTEND);
991 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
992 setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
993 setTargetDAGCombine(ISD::INTRINSIC_VOID);
994 setTargetDAGCombine(ISD::VECREDUCE_ADD);
995 setTargetDAGCombine(ISD::ADD);
996 setTargetDAGCombine(ISD::BITCAST);
997 }
998 if (Subtarget->hasMVEIntegerOps()) {
999 setTargetDAGCombine(ISD::SMIN);
1000 setTargetDAGCombine(ISD::UMIN);
1001 setTargetDAGCombine(ISD::SMAX);
1002 setTargetDAGCombine(ISD::UMAX);
1003 setTargetDAGCombine(ISD::FP_EXTEND);
1004 setTargetDAGCombine(ISD::SELECT);
1005 setTargetDAGCombine(ISD::SELECT_CC);
1006 }
1007
1008 if (!Subtarget->hasFP64()) {
1009 // When targeting a floating-point unit with only single-precision
1010 // operations, f64 is legal for the few double-precision instructions which
1011 // are present However, no double-precision operations other than moves,
1012 // loads and stores are provided by the hardware.
1013 setOperationAction(ISD::FADD, MVT::f64, Expand);
1014 setOperationAction(ISD::FSUB, MVT::f64, Expand);
1015 setOperationAction(ISD::FMUL, MVT::f64, Expand);
1016 setOperationAction(ISD::FMA, MVT::f64, Expand);
1017 setOperationAction(ISD::FDIV, MVT::f64, Expand);
1018 setOperationAction(ISD::FREM, MVT::f64, Expand);
1019 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
1020 setOperationAction(ISD::FGETSIGN, MVT::f64, Expand);
1021 setOperationAction(ISD::FNEG, MVT::f64, Expand);
1022 setOperationAction(ISD::FABS, MVT::f64, Expand);
1023 setOperationAction(ISD::FSQRT, MVT::f64, Expand);
1024 setOperationAction(ISD::FSIN, MVT::f64, Expand);
1025 setOperationAction(ISD::FCOS, MVT::f64, Expand);
1026 setOperationAction(ISD::FPOW, MVT::f64, Expand);
1027 setOperationAction(ISD::FLOG, MVT::f64, Expand);
1028 setOperationAction(ISD::FLOG2, MVT::f64, Expand);
1029 setOperationAction(ISD::FLOG10, MVT::f64, Expand);
1030 setOperationAction(ISD::FEXP, MVT::f64, Expand);
1031 setOperationAction(ISD::FEXP2, MVT::f64, Expand);
1032 setOperationAction(ISD::FCEIL, MVT::f64, Expand);
1033 setOperationAction(ISD::FTRUNC, MVT::f64, Expand);
1034 setOperationAction(ISD::FRINT, MVT::f64, Expand);
1035 setOperationAction(ISD::FNEARBYINT, MVT::f64, Expand);
1036 setOperationAction(ISD::FFLOOR, MVT::f64, Expand);
1037 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
1038 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
1039 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
1040 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
1041 setOperationAction(ISD::FP_TO_SINT, MVT::f64, Custom);
1042 setOperationAction(ISD::FP_TO_UINT, MVT::f64, Custom);
1043 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
1044 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
1045 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
1046 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::f64, Custom);
1047 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::f64, Custom);
1048 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
1049 }
1050
1051 if (!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) {
1052 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
1053 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
1054 if (Subtarget->hasFullFP16()) {
1055 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
1056 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
1057 }
1058 }
1059
1060 if (!Subtarget->hasFP16()) {
1061 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
1062 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
1063 }
1064
1065 computeRegisterProperties(Subtarget->getRegisterInfo());
1066
1067 // ARM does not have floating-point extending loads.
1068 for (MVT VT : MVT::fp_valuetypes()) {
1069 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1070 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
1071 }
1072
1073 // ... or truncating stores
1074 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
1075 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
1076 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
1077
1078 // ARM does not have i1 sign extending load.
1079 for (MVT VT : MVT::integer_valuetypes())
1080 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
1081
1082 // ARM supports all 4 flavors of integer indexed load / store.
1083 if (!Subtarget->isThumb1Only()) {
1084 for (unsigned im = (unsigned)ISD::PRE_INC;
1085 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1086 setIndexedLoadAction(im, MVT::i1, Legal);
1087 setIndexedLoadAction(im, MVT::i8, Legal);
1088 setIndexedLoadAction(im, MVT::i16, Legal);
1089 setIndexedLoadAction(im, MVT::i32, Legal);
1090 setIndexedStoreAction(im, MVT::i1, Legal);
1091 setIndexedStoreAction(im, MVT::i8, Legal);
1092 setIndexedStoreAction(im, MVT::i16, Legal);
1093 setIndexedStoreAction(im, MVT::i32, Legal);
1094 }
1095 } else {
1096 // Thumb-1 has limited post-inc load/store support - LDM r0!, {r1}.
1097 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1098 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1099 }
1100
1101 setOperationAction(ISD::SADDO, MVT::i32, Custom);
1102 setOperationAction(ISD::UADDO, MVT::i32, Custom);
1103 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
1104 setOperationAction(ISD::USUBO, MVT::i32, Custom);
1105
1106 setOperationAction(ISD::ADDCARRY, MVT::i32, Custom);
1107 setOperationAction(ISD::SUBCARRY, MVT::i32, Custom);
1108 if (Subtarget->hasDSP()) {
1109 setOperationAction(ISD::SADDSAT, MVT::i8, Custom);
1110 setOperationAction(ISD::SSUBSAT, MVT::i8, Custom);
1111 setOperationAction(ISD::SADDSAT, MVT::i16, Custom);
1112 setOperationAction(ISD::SSUBSAT, MVT::i16, Custom);
1113 }
1114 if (Subtarget->hasBaseDSP()) {
1115 setOperationAction(ISD::SADDSAT, MVT::i32, Legal);
1116 setOperationAction(ISD::SSUBSAT, MVT::i32, Legal);
1117 }
1118
1119 // i64 operation support.
1120 setOperationAction(ISD::MUL, MVT::i64, Expand);
1121 setOperationAction(ISD::MULHU, MVT::i32, Expand);
1122 if (Subtarget->isThumb1Only()) {
1123 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
1124 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
1125 }
1126 if (Subtarget->isThumb1Only() || !Subtarget->hasV6Ops()
1127 || (Subtarget->isThumb2() && !Subtarget->hasDSP()))
1128 setOperationAction(ISD::MULHS, MVT::i32, Expand);
1129
1130 setOperationAction(ISD::SHL_PARTS, MVT::i32, Custom);
1131 setOperationAction(ISD::SRA_PARTS, MVT::i32, Custom);
1132 setOperationAction(ISD::SRL_PARTS, MVT::i32, Custom);
1133 setOperationAction(ISD::SRL, MVT::i64, Custom);
1134 setOperationAction(ISD::SRA, MVT::i64, Custom);
1135 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1136 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
1137 setOperationAction(ISD::LOAD, MVT::i64, Custom);
1138 setOperationAction(ISD::STORE, MVT::i64, Custom);
1139
1140 // MVE lowers 64 bit shifts to lsll and lsrl
1141 // assuming that ISD::SRL and SRA of i64 are already marked custom
1142 if (Subtarget->hasMVEIntegerOps())
1143 setOperationAction(ISD::SHL, MVT::i64, Custom);
1144
1145 // Expand to __aeabi_l{lsl,lsr,asr} calls for Thumb1.
1146 if (Subtarget->isThumb1Only()) {
1147 setOperationAction(ISD::SHL_PARTS, MVT::i32, Expand);
1148 setOperationAction(ISD::SRA_PARTS, MVT::i32, Expand);
1149 setOperationAction(ISD::SRL_PARTS, MVT::i32, Expand);
1150 }
1151
1152 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops())
1153 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
1154
1155 // ARM does not have ROTL.
1156 setOperationAction(ISD::ROTL, MVT::i32, Expand);
1157 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1158 setOperationAction(ISD::ROTL, VT, Expand);
1159 setOperationAction(ISD::ROTR, VT, Expand);
1160 }
1161 setOperationAction(ISD::CTTZ, MVT::i32, Custom);
1162 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
1163 if (!Subtarget->hasV5TOps() || Subtarget->isThumb1Only()) {
1164 setOperationAction(ISD::CTLZ, MVT::i32, Expand);
1165 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, LibCall);
1166 }
1167
1168 // @llvm.readcyclecounter requires the Performance Monitors extension.
1169 // Default to the 0 expansion on unsupported platforms.
1170 // FIXME: Technically there are older ARM CPUs that have
1171 // implementation-specific ways of obtaining this information.
1172 if (Subtarget->hasPerfMon())
1173 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Custom);
1174
1175 // Only ARMv6 has BSWAP.
1176 if (!Subtarget->hasV6Ops())
1177 setOperationAction(ISD::BSWAP, MVT::i32, Expand);
1178
1179 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
1180 : Subtarget->hasDivideInARMMode();
1181 if (!hasDivide) {
1182 // These are expanded into libcalls if the cpu doesn't have HW divider.
1183 setOperationAction(ISD::SDIV, MVT::i32, LibCall);
1184 setOperationAction(ISD::UDIV, MVT::i32, LibCall);
1185 }
1186
1187 if (Subtarget->isTargetWindows() && !Subtarget->hasDivideInThumbMode()) {
1188 setOperationAction(ISD::SDIV, MVT::i32, Custom);
1189 setOperationAction(ISD::UDIV, MVT::i32, Custom);
1190
1191 setOperationAction(ISD::SDIV, MVT::i64, Custom);
1192 setOperationAction(ISD::UDIV, MVT::i64, Custom);
1193 }
1194
1195 setOperationAction(ISD::SREM, MVT::i32, Expand);
1196 setOperationAction(ISD::UREM, MVT::i32, Expand);
1197
1198 // Register based DivRem for AEABI (RTABI 4.2)
1199 if (Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||
1200 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||
1201 Subtarget->isTargetWindows()) {
1202 setOperationAction(ISD::SREM, MVT::i64, Custom);
1203 setOperationAction(ISD::UREM, MVT::i64, Custom);
1204 HasStandaloneRem = false;
1205
1206 if (Subtarget->isTargetWindows()) {
1207 const struct {
1208 const RTLIB::Libcall Op;
1209 const char * const Name;
1210 const CallingConv::ID CC;
1211 } LibraryCalls[] = {
1212 { RTLIB::SDIVREM_I8, "__rt_sdiv", CallingConv::ARM_AAPCS },
1213 { RTLIB::SDIVREM_I16, "__rt_sdiv", CallingConv::ARM_AAPCS },
1214 { RTLIB::SDIVREM_I32, "__rt_sdiv", CallingConv::ARM_AAPCS },
1215 { RTLIB::SDIVREM_I64, "__rt_sdiv64", CallingConv::ARM_AAPCS },
1216
1217 { RTLIB::UDIVREM_I8, "__rt_udiv", CallingConv::ARM_AAPCS },
1218 { RTLIB::UDIVREM_I16, "__rt_udiv", CallingConv::ARM_AAPCS },
1219 { RTLIB::UDIVREM_I32, "__rt_udiv", CallingConv::ARM_AAPCS },
1220 { RTLIB::UDIVREM_I64, "__rt_udiv64", CallingConv::ARM_AAPCS },
1221 };
1222
1223 for (const auto &LC : LibraryCalls) {
1224 setLibcallName(LC.Op, LC.Name);
1225 setLibcallCallingConv(LC.Op, LC.CC);
1226 }
1227 } else {
1228 const struct {
1229 const RTLIB::Libcall Op;
1230 const char * const Name;
1231 const CallingConv::ID CC;
1232 } LibraryCalls[] = {
1233 { RTLIB::SDIVREM_I8, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1234 { RTLIB::SDIVREM_I16, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1235 { RTLIB::SDIVREM_I32, "__aeabi_idivmod", CallingConv::ARM_AAPCS },
1236 { RTLIB::SDIVREM_I64, "__aeabi_ldivmod", CallingConv::ARM_AAPCS },
1237
1238 { RTLIB::UDIVREM_I8, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1239 { RTLIB::UDIVREM_I16, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1240 { RTLIB::UDIVREM_I32, "__aeabi_uidivmod", CallingConv::ARM_AAPCS },
1241 { RTLIB::UDIVREM_I64, "__aeabi_uldivmod", CallingConv::ARM_AAPCS },
1242 };
1243
1244 for (const auto &LC : LibraryCalls) {
1245 setLibcallName(LC.Op, LC.Name);
1246 setLibcallCallingConv(LC.Op, LC.CC);
1247 }
1248 }
1249
1250 setOperationAction(ISD::SDIVREM, MVT::i32, Custom);
1251 setOperationAction(ISD::UDIVREM, MVT::i32, Custom);
1252 setOperationAction(ISD::SDIVREM, MVT::i64, Custom);
1253 setOperationAction(ISD::UDIVREM, MVT::i64, Custom);
1254 } else {
1255 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
1256 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
1257 }
1258
1259 if (Subtarget->getTargetTriple().isOSMSVCRT()) {
1260 // MSVCRT doesn't have powi; fall back to pow
1261 setLibcallName(RTLIB::POWI_F32, nullptr);
1262 setLibcallName(RTLIB::POWI_F64, nullptr);
1263 }
1264
1265 setOperationAction(ISD::GlobalAddress, MVT::i32, Custom);
1266 setOperationAction(ISD::ConstantPool, MVT::i32, Custom);
1267 setOperationAction(ISD::GlobalTLSAddress, MVT::i32, Custom);
1268 setOperationAction(ISD::BlockAddress, MVT::i32, Custom);
1269
1270 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1271 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
1272
1273 // Use the default implementation.
1274 setOperationAction(ISD::VASTART, MVT::Other, Custom);
1275 setOperationAction(ISD::VAARG, MVT::Other, Expand);
1276 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
1277 setOperationAction(ISD::VAEND, MVT::Other, Expand);
1278 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
1279 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
1280
1281 if (Subtarget->isTargetWindows())
1282 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Custom);
1283 else
1284 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32, Expand);
1285
1286 // ARMv6 Thumb1 (except for CPUs that support dmb / dsb) and earlier use
1287 // the default expansion.
1288 InsertFencesForAtomic = false;
1289 if (Subtarget->hasAnyDataBarrier() &&
1290 (!Subtarget->isThumb() || Subtarget->hasV8MBaselineOps())) {
1291 // ATOMIC_FENCE needs custom lowering; the others should have been expanded
1292 // to ldrex/strex loops already.
1293 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
1294 if (!Subtarget->isThumb() || !Subtarget->isMClass())
1295 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, Custom);
1296
1297 // On v8, we have particularly efficient implementations of atomic fences
1298 // if they can be combined with nearby atomic loads and stores.
1299 if (!Subtarget->hasAcquireRelease() ||
1300 getTargetMachine().getOptLevel() == 0) {
1301 // Automatically insert fences (dmb ish) around ATOMIC_SWAP etc.
1302 InsertFencesForAtomic = true;
1303 }
1304 } else {
1305 // If there's anything we can use as a barrier, go through custom lowering
1306 // for ATOMIC_FENCE.
1307 // If target has DMB in thumb, Fences can be inserted.
1308 if (Subtarget->hasDataBarrier())
1309 InsertFencesForAtomic = true;
1310
1311 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other,
1312 Subtarget->hasAnyDataBarrier() ? Custom : Expand);
1313
1314 // Set them all for expansion, which will force libcalls.
1315 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, Expand);
1316 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, Expand);
1317 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, Expand);
1318 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
1319 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Expand);
1320 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, Expand);
1321 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, Expand);
1322 setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i32, Expand);
1323 setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i32, Expand);
1324 setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i32, Expand);
1325 setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i32, Expand);
1326 setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i32, Expand);
1327 // Mark ATOMIC_LOAD and ATOMIC_STORE custom so we can handle the
1328 // Unordered/Monotonic case.
1329 if (!InsertFencesForAtomic) {
1330 setOperationAction(ISD::ATOMIC_LOAD, MVT::i32, Custom);
1331 setOperationAction(ISD::ATOMIC_STORE, MVT::i32, Custom);
1332 }
1333 }
1334
1335 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
1336
1337 // Requires SXTB/SXTH, available on v6 and up in both ARM and Thumb modes.
1338 if (!Subtarget->hasV6Ops()) {
1339 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
1340 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
1341 }
1342 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
1343
1344 if (!Subtarget->useSoftFloat() && Subtarget->hasFPRegs() &&
1345 !Subtarget->isThumb1Only()) {
1346 // Turn f64->i64 into VMOVRRD, i64 -> f64 to VMOVDRR
1347 // iff target supports vfp2.
1348 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1349 setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
1350 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
1351 }
1352
1353 // We want to custom lower some of our intrinsics.
1354 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1355 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
1356 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
1357 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
1358 if (Subtarget->useSjLjEH())
1359 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
1360
1361 setOperationAction(ISD::SETCC, MVT::i32, Expand);
1362 setOperationAction(ISD::SETCC, MVT::f32, Expand);
1363 setOperationAction(ISD::SETCC, MVT::f64, Expand);
1364 setOperationAction(ISD::SELECT, MVT::i32, Custom);
1365 setOperationAction(ISD::SELECT, MVT::f32, Custom);
1366 setOperationAction(ISD::SELECT, MVT::f64, Custom);
1367 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
1368 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
1369 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
1370 if (Subtarget->hasFullFP16()) {
1371 setOperationAction(ISD::SETCC, MVT::f16, Expand);
1372 setOperationAction(ISD::SELECT, MVT::f16, Custom);
1373 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
1374 }
1375
1376 setOperationAction(ISD::SETCCCARRY, MVT::i32, Custom);
1377
1378 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
1379 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
1380 if (Subtarget->hasFullFP16())
1381 setOperationAction(ISD::BR_CC, MVT::f16, Custom);
1382 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
1383 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
1384 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
1385
1386 // We don't support sin/cos/fmod/copysign/pow
1387 setOperationAction(ISD::FSIN, MVT::f64, Expand);
1388 setOperationAction(ISD::FSIN, MVT::f32, Expand);
1389 setOperationAction(ISD::FCOS, MVT::f32, Expand);
1390 setOperationAction(ISD::FCOS, MVT::f64, Expand);
1391 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
1392 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
1393 setOperationAction(ISD::FREM, MVT::f64, Expand);
1394 setOperationAction(ISD::FREM, MVT::f32, Expand);
1395 if (!Subtarget->useSoftFloat() && Subtarget->hasVFP2Base() &&
1396 !Subtarget->isThumb1Only()) {
1397 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
1398 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
1399 }
1400 setOperationAction(ISD::FPOW, MVT::f64, Expand);
1401 setOperationAction(ISD::FPOW, MVT::f32, Expand);
1402
1403 if (!Subtarget->hasVFP4Base()) {
1404 setOperationAction(ISD::FMA, MVT::f64, Expand);
1405 setOperationAction(ISD::FMA, MVT::f32, Expand);
1406 }
1407
1408 // Various VFP goodness
1409 if (!Subtarget->useSoftFloat() && !Subtarget->isThumb1Only()) {
1410 // FP-ARMv8 adds f64 <-> f16 conversion. Before that it should be expanded.
1411 if (!Subtarget->hasFPARMv8Base() || !Subtarget->hasFP64()) {
1412 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
1413 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
1414 }
1415
1416 // fp16 is a special v7 extension that adds f16 <-> f32 conversions.
1417 if (!Subtarget->hasFP16()) {
1418 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
1419 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
1420 }
1421
1422 // Strict floating-point comparisons need custom lowering.
1423 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
1424 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
1425 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
1426 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
1427 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
1428 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
1429 }
1430
1431 // Use __sincos_stret if available.
1432 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
1433 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
1434 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1435 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1436 }
1437
1438 // FP-ARMv8 implements a lot of rounding-like FP operations.
1439 if (Subtarget->hasFPARMv8Base()) {
1440 setOperationAction(ISD::FFLOOR, MVT::f32, Legal);
1441 setOperationAction(ISD::FCEIL, MVT::f32, Legal);
1442 setOperationAction(ISD::FROUND, MVT::f32, Legal);
1443 setOperationAction(ISD::FTRUNC, MVT::f32, Legal);
1444 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
1445 setOperationAction(ISD::FRINT, MVT::f32, Legal);
1446 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
1447 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
1448 if (Subtarget->hasNEON()) {
1449 setOperationAction(ISD::FMINNUM, MVT::v2f32, Legal);
1450 setOperationAction(ISD::FMAXNUM, MVT::v2f32, Legal);
1451 setOperationAction(ISD::FMINNUM, MVT::v4f32, Legal);
1452 setOperationAction(ISD::FMAXNUM, MVT::v4f32, Legal);
1453 }
1454
1455 if (Subtarget->hasFP64()) {
1456 setOperationAction(ISD::FFLOOR, MVT::f64, Legal);
1457 setOperationAction(ISD::FCEIL, MVT::f64, Legal);
1458 setOperationAction(ISD::FROUND, MVT::f64, Legal);
1459 setOperationAction(ISD::FTRUNC, MVT::f64, Legal);
1460 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
1461 setOperationAction(ISD::FRINT, MVT::f64, Legal);
1462 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
1463 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
1464 }
1465 }
1466
1467 // FP16 often need to be promoted to call lib functions
1468 if (Subtarget->hasFullFP16()) {
1469 setOperationAction(ISD::FREM, MVT::f16, Promote);
1470 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
1471 setOperationAction(ISD::FSIN, MVT::f16, Promote);
1472 setOperationAction(ISD::FCOS, MVT::f16, Promote);
1473 setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
1474 setOperationAction(ISD::FPOWI, MVT::f16, Promote);
1475 setOperationAction(ISD::FPOW, MVT::f16, Promote);
1476 setOperationAction(ISD::FEXP, MVT::f16, Promote);
1477 setOperationAction(ISD::FEXP2, MVT::f16, Promote);
1478 setOperationAction(ISD::FLOG, MVT::f16, Promote);
1479 setOperationAction(ISD::FLOG10, MVT::f16, Promote);
1480 setOperationAction(ISD::FLOG2, MVT::f16, Promote);
1481
1482 setOperationAction(ISD::FROUND, MVT::f16, Legal);
1483 }
1484
1485 if (Subtarget->hasNEON()) {
1486 // vmin and vmax aren't available in a scalar form, so we can use
1487 // a NEON instruction with an undef lane instead. This has a performance
1488 // penalty on some cores, so we don't do this unless we have been
1489 // asked to by the core tuning model.
1490 if (Subtarget->useNEONForSinglePrecisionFP()) {
1491 setOperationAction(ISD::FMINIMUM, MVT::f32, Legal);
1492 setOperationAction(ISD::FMAXIMUM, MVT::f32, Legal);
1493 setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
1494 setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
1495 }
1496 setOperationAction(ISD::FMINIMUM, MVT::v2f32, Legal);
1497 setOperationAction(ISD::FMAXIMUM, MVT::v2f32, Legal);
1498 setOperationAction(ISD::FMINIMUM, MVT::v4f32, Legal);
1499 setOperationAction(ISD::FMAXIMUM, MVT::v4f32, Legal);
1500
1501 if (Subtarget->hasFullFP16()) {
1502 setOperationAction(ISD::FMINNUM, MVT::v4f16, Legal);
1503 setOperationAction(ISD::FMAXNUM, MVT::v4f16, Legal);
1504 setOperationAction(ISD::FMINNUM, MVT::v8f16, Legal);
1505 setOperationAction(ISD::FMAXNUM, MVT::v8f16, Legal);
1506
1507 setOperationAction(ISD::FMINIMUM, MVT::v4f16, Legal);
1508 setOperationAction(ISD::FMAXIMUM, MVT::v4f16, Legal);
1509 setOperationAction(ISD::FMINIMUM, MVT::v8f16, Legal);
1510 setOperationAction(ISD::FMAXIMUM, MVT::v8f16, Legal);
1511 }
1512 }
1513
1514 // We have target-specific dag combine patterns for the following nodes:
1515 // ARMISD::VMOVRRD - No need to call setTargetDAGCombine
1516 setTargetDAGCombine(ISD::ADD);
1517 setTargetDAGCombine(ISD::SUB);
1518 setTargetDAGCombine(ISD::MUL);
1519 setTargetDAGCombine(ISD::AND);
1520 setTargetDAGCombine(ISD::OR);
1521 setTargetDAGCombine(ISD::XOR);
1522
1523 if (Subtarget->hasMVEIntegerOps())
1524 setTargetDAGCombine(ISD::VSELECT);
1525
1526 if (Subtarget->hasV6Ops())
1527 setTargetDAGCombine(ISD::SRL);
1528 if (Subtarget->isThumb1Only())
1529 setTargetDAGCombine(ISD::SHL);
1530
1531 setStackPointerRegisterToSaveRestore(ARM::SP);
1532
1533 if (Subtarget->useSoftFloat() || Subtarget->isThumb1Only() ||
1534 !Subtarget->hasVFP2Base() || Subtarget->hasMinSize())
1535 setSchedulingPreference(Sched::RegPressure);
1536 else
1537 setSchedulingPreference(Sched::Hybrid);
1538
1539 //// temporary - rewrite interface to use type
1540 MaxStoresPerMemset = 8;
1541 MaxStoresPerMemsetOptSize = 4;
1542 MaxStoresPerMemcpy = 4; // For @llvm.memcpy -> sequence of stores
1543 MaxStoresPerMemcpyOptSize = 2;
1544 MaxStoresPerMemmove = 4; // For @llvm.memmove -> sequence of stores
1545 MaxStoresPerMemmoveOptSize = 2;
1546
1547 // On ARM arguments smaller than 4 bytes are extended, so all arguments
1548 // are at least 4 bytes aligned.
1549 setMinStackArgumentAlignment(Align(4));
1550
1551 // Prefer likely predicted branches to selects on out-of-order cores.
1552 PredictableSelectIsExpensive = Subtarget->getSchedModel().isOutOfOrder();
1553
1554 setPrefLoopAlignment(Align(1ULL << Subtarget->getPrefLoopLogAlignment()));
1555
1556 setMinFunctionAlignment(Subtarget->isThumb() ? Align(2) : Align(4));
1557
1558 if (Subtarget->isThumb() || Subtarget->isThumb2())
1559 setTargetDAGCombine(ISD::ABS);
1560}
1561
1562bool ARMTargetLowering::useSoftFloat() const {
1563 return Subtarget->useSoftFloat();
1564}
1565
1566// FIXME: It might make sense to define the representative register class as the
1567// nearest super-register that has a non-null superset. For example, DPR_VFP2 is
1568// a super-register of SPR, and DPR is a superset if DPR_VFP2. Consequently,
1569// SPR's representative would be DPR_VFP2. This should work well if register
1570// pressure tracking were modified such that a register use would increment the
1571// pressure of the register class's representative and all of it's super
1572// classes' representatives transitively. We have not implemented this because
1573// of the difficulty prior to coalescing of modeling operand register classes
1574// due to the common occurrence of cross class copies and subregister insertions
1575// and extractions.
1576std::pair<const TargetRegisterClass *, uint8_t>
1577ARMTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
1578 MVT VT) const {
1579 const TargetRegisterClass *RRC = nullptr;
1580 uint8_t Cost = 1;
1581 switch (VT.SimpleTy) {
1582 default:
1583 return TargetLowering::findRepresentativeClass(TRI, VT);
1584 // Use DPR as representative register class for all floating point
1585 // and vector types. Since there are 32 SPR registers and 32 DPR registers so
1586 // the cost is 1 for both f32 and f64.
1587 case MVT::f32: case MVT::f64: case MVT::v8i8: case MVT::v4i16:
1588 case MVT::v2i32: case MVT::v1i64: case MVT::v2f32:
1589 RRC = &ARM::DPRRegClass;
1590 // When NEON is used for SP, only half of the register file is available
1591 // because operations that define both SP and DP results will be constrained
1592 // to the VFP2 class (D0-D15). We currently model this constraint prior to
1593 // coalescing by double-counting the SP regs. See the FIXME above.
1594 if (Subtarget->useNEONForSinglePrecisionFP())
1595 Cost = 2;
1596 break;
1597 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
1598 case MVT::v4f32: case MVT::v2f64:
1599 RRC = &ARM::DPRRegClass;
1600 Cost = 2;
1601 break;
1602 case MVT::v4i64:
1603 RRC = &ARM::DPRRegClass;
1604 Cost = 4;
1605 break;
1606 case MVT::v8i64:
1607 RRC = &ARM::DPRRegClass;
1608 Cost = 8;
1609 break;
1610 }
1611 return std::make_pair(RRC, Cost);
1612}
1613
1614const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
1615#define MAKE_CASE(V) \
1616 case V: \
1617 return #V;
1618 switch ((ARMISD::NodeType)Opcode) {
1619 case ARMISD::FIRST_NUMBER:
1620 break;
1621 MAKE_CASE(ARMISD::Wrapper)
1622 MAKE_CASE(ARMISD::WrapperPIC)
1623 MAKE_CASE(ARMISD::WrapperJT)
1624 MAKE_CASE(ARMISD::COPY_STRUCT_BYVAL)
1625 MAKE_CASE(ARMISD::CALL)
1626 MAKE_CASE(ARMISD::CALL_PRED)
1627 MAKE_CASE(ARMISD::CALL_NOLINK)
1628 MAKE_CASE(ARMISD::tSECALL)
1629 MAKE_CASE(ARMISD::BRCOND)
1630 MAKE_CASE(ARMISD::BR_JT)
1631 MAKE_CASE(ARMISD::BR2_JT)
1632 MAKE_CASE(ARMISD::RET_FLAG)
1633 MAKE_CASE(ARMISD::SERET_FLAG)
1634 MAKE_CASE(ARMISD::INTRET_FLAG)
1635 MAKE_CASE(ARMISD::PIC_ADD)
1636 MAKE_CASE(ARMISD::CMP)
1637 MAKE_CASE(ARMISD::CMN)
1638 MAKE_CASE(ARMISD::CMPZ)
1639 MAKE_CASE(ARMISD::CMPFP)
1640 MAKE_CASE(ARMISD::CMPFPE)
1641 MAKE_CASE(ARMISD::CMPFPw0)
1642 MAKE_CASE(ARMISD::CMPFPEw0)
1643 MAKE_CASE(ARMISD::BCC_i64)
1644 MAKE_CASE(ARMISD::FMSTAT)
1645 MAKE_CASE(ARMISD::CMOV)
1646 MAKE_CASE(ARMISD::SUBS)
1647 MAKE_CASE(ARMISD::SSAT)
1648 MAKE_CASE(ARMISD::USAT)
1649 MAKE_CASE(ARMISD::ASRL)
1650 MAKE_CASE(ARMISD::LSRL)
1651 MAKE_CASE(ARMISD::LSLL)
1652 MAKE_CASE(ARMISD::SRL_FLAG)
1653 MAKE_CASE(ARMISD::SRA_FLAG)
1654 MAKE_CASE(ARMISD::RRX)
1655 MAKE_CASE(ARMISD::ADDC)
1656 MAKE_CASE(ARMISD::ADDE)
1657 MAKE_CASE(ARMISD::SUBC)
1658 MAKE_CASE(ARMISD::SUBE)
1659 MAKE_CASE(ARMISD::LSLS)
1660 MAKE_CASE(ARMISD::VMOVRRD)
1661 MAKE_CASE(ARMISD::VMOVDRR)
1662 MAKE_CASE(ARMISD::VMOVhr)
1663 MAKE_CASE(ARMISD::VMOVrh)
1664 MAKE_CASE(ARMISD::VMOVSR)
1665 MAKE_CASE(ARMISD::EH_SJLJ_SETJMP)
1666 MAKE_CASE(ARMISD::EH_SJLJ_LONGJMP)
1667 MAKE_CASE(ARMISD::EH_SJLJ_SETUP_DISPATCH)
1668 MAKE_CASE(ARMISD::TC_RETURN)
1669 MAKE_CASE(ARMISD::THREAD_POINTER)
1670 MAKE_CASE(ARMISD::DYN_ALLOC)
1671 MAKE_CASE(ARMISD::MEMBARRIER_MCR)
1672 MAKE_CASE(ARMISD::PRELOAD)
1673 MAKE_CASE(ARMISD::LDRD)
1674 MAKE_CASE(ARMISD::STRD)
1675 MAKE_CASE(ARMISD::WIN__CHKSTK)
1676 MAKE_CASE(ARMISD::WIN__DBZCHK)
1677 MAKE_CASE(ARMISD::PREDICATE_CAST)
1678 MAKE_CASE(ARMISD::VECTOR_REG_CAST)
1679 MAKE_CASE(ARMISD::VCMP)
1680 MAKE_CASE(ARMISD::VCMPZ)
1681 MAKE_CASE(ARMISD::VTST)
1682 MAKE_CASE(ARMISD::VSHLs)
1683 MAKE_CASE(ARMISD::VSHLu)
1684 MAKE_CASE(ARMISD::VSHLIMM)
1685 MAKE_CASE(ARMISD::VSHRsIMM)
1686 MAKE_CASE(ARMISD::VSHRuIMM)
1687 MAKE_CASE(ARMISD::VRSHRsIMM)
1688 MAKE_CASE(ARMISD::VRSHRuIMM)
1689 MAKE_CASE(ARMISD::VRSHRNIMM)
1690 MAKE_CASE(ARMISD::VQSHLsIMM)
1691 MAKE_CASE(ARMISD::VQSHLuIMM)
1692 MAKE_CASE(ARMISD::VQSHLsuIMM)
1693 MAKE_CASE(ARMISD::VQSHRNsIMM)
1694 MAKE_CASE(ARMISD::VQSHRNuIMM)
1695 MAKE_CASE(ARMISD::VQSHRNsuIMM)
1696 MAKE_CASE(ARMISD::VQRSHRNsIMM)
1697 MAKE_CASE(ARMISD::VQRSHRNuIMM)
1698 MAKE_CASE(ARMISD::VQRSHRNsuIMM)
1699 MAKE_CASE(ARMISD::VSLIIMM)
1700 MAKE_CASE(ARMISD::VSRIIMM)
1701 MAKE_CASE(ARMISD::VGETLANEu)
1702 MAKE_CASE(ARMISD::VGETLANEs)
1703 MAKE_CASE(ARMISD::VMOVIMM)
1704 MAKE_CASE(ARMISD::VMVNIMM)
1705 MAKE_CASE(ARMISD::VMOVFPIMM)
1706 MAKE_CASE(ARMISD::VDUP)
1707 MAKE_CASE(ARMISD::VDUPLANE)
1708 MAKE_CASE(ARMISD::VEXT)
1709 MAKE_CASE(ARMISD::VREV64)
1710 MAKE_CASE(ARMISD::VREV32)
1711 MAKE_CASE(ARMISD::VREV16)
1712 MAKE_CASE(ARMISD::VZIP)
1713 MAKE_CASE(ARMISD::VUZP)
1714 MAKE_CASE(ARMISD::VTRN)
1715 MAKE_CASE(ARMISD::VTBL1)
1716 MAKE_CASE(ARMISD::VTBL2)
1717 MAKE_CASE(ARMISD::VMOVN)
1718 MAKE_CASE(ARMISD::VQMOVNs)
1719 MAKE_CASE(ARMISD::VQMOVNu)
1720 MAKE_CASE(ARMISD::VCVTN)
1721 MAKE_CASE(ARMISD::VCVTL)
1722 MAKE_CASE(ARMISD::VIDUP)
1723 MAKE_CASE(ARMISD::VMULLs)
1724 MAKE_CASE(ARMISD::VMULLu)
1725 MAKE_CASE(ARMISD::VQDMULH)
1726 MAKE_CASE(ARMISD::VADDVs)
1727 MAKE_CASE(ARMISD::VADDVu)
1728 MAKE_CASE(ARMISD::VADDVps)
1729 MAKE_CASE(ARMISD::VADDVpu)
1730 MAKE_CASE(ARMISD::VADDLVs)
1731 MAKE_CASE(ARMISD::VADDLVu)
1732 MAKE_CASE(ARMISD::VADDLVAs)
1733 MAKE_CASE(ARMISD::VADDLVAu)
1734 MAKE_CASE(ARMISD::VADDLVps)
1735 MAKE_CASE(ARMISD::VADDLVpu)
1736 MAKE_CASE(ARMISD::VADDLVAps)
1737 MAKE_CASE(ARMISD::VADDLVApu)
1738 MAKE_CASE(ARMISD::VMLAVs)
1739 MAKE_CASE(ARMISD::VMLAVu)
1740 MAKE_CASE(ARMISD::VMLAVps)
1741 MAKE_CASE(ARMISD::VMLAVpu)
1742 MAKE_CASE(ARMISD::VMLALVs)
1743 MAKE_CASE(ARMISD::VMLALVu)
1744 MAKE_CASE(ARMISD::VMLALVps)
1745 MAKE_CASE(ARMISD::VMLALVpu)
1746 MAKE_CASE(ARMISD::VMLALVAs)
1747 MAKE_CASE(ARMISD::VMLALVAu)
1748 MAKE_CASE(ARMISD::VMLALVAps)
1749 MAKE_CASE(ARMISD::VMLALVApu)
1750 MAKE_CASE(ARMISD::VMINVu)
1751 MAKE_CASE(ARMISD::VMINVs)
1752 MAKE_CASE(ARMISD::VMAXVu)
1753 MAKE_CASE(ARMISD::VMAXVs)
1754 MAKE_CASE(ARMISD::UMAAL)
1755 MAKE_CASE(ARMISD::UMLAL)
1756 MAKE_CASE(ARMISD::SMLAL)
1757 MAKE_CASE(ARMISD::SMLALBB)
1758 MAKE_CASE(ARMISD::SMLALBT)
1759 MAKE_CASE(ARMISD::SMLALTB)
1760 MAKE_CASE(ARMISD::SMLALTT)
1761 MAKE_CASE(ARMISD::SMULWB)
1762 MAKE_CASE(ARMISD::SMULWT)
1763 MAKE_CASE(ARMISD::SMLALD)
1764 MAKE_CASE(ARMISD::SMLALDX)
1765 MAKE_CASE(ARMISD::SMLSLD)
1766 MAKE_CASE(ARMISD::SMLSLDX)
1767 MAKE_CASE(ARMISD::SMMLAR)
1768 MAKE_CASE(ARMISD::SMMLSR)
1769 MAKE_CASE(ARMISD::QADD16b)
1770 MAKE_CASE(ARMISD::QSUB16b)
1771 MAKE_CASE(ARMISD::QADD8b)
1772 MAKE_CASE(ARMISD::QSUB8b)
1773 MAKE_CASE(ARMISD::BUILD_VECTOR)
1774 MAKE_CASE(ARMISD::BFI)
1775 MAKE_CASE(ARMISD::VORRIMM)
1776 MAKE_CASE(ARMISD::VBICIMM)
1777 MAKE_CASE(ARMISD::VBSP)
1778 MAKE_CASE(ARMISD::MEMCPY)
1779 MAKE_CASE(ARMISD::VLD1DUP)
1780 MAKE_CASE(ARMISD::VLD2DUP)
1781 MAKE_CASE(ARMISD::VLD3DUP)
1782 MAKE_CASE(ARMISD::VLD4DUP)
1783 MAKE_CASE(ARMISD::VLD1_UPD)
1784 MAKE_CASE(ARMISD::VLD2_UPD)
1785 MAKE_CASE(ARMISD::VLD3_UPD)
1786 MAKE_CASE(ARMISD::VLD4_UPD)
1787 MAKE_CASE(ARMISD::VLD1x2_UPD)
1788 MAKE_CASE(ARMISD::VLD1x3_UPD)
1789 MAKE_CASE(ARMISD::VLD1x4_UPD)
1790 MAKE_CASE(ARMISD::VLD2LN_UPD)
1791 MAKE_CASE(ARMISD::VLD3LN_UPD)
1792 MAKE_CASE(ARMISD::VLD4LN_UPD)
1793 MAKE_CASE(ARMISD::VLD1DUP_UPD)
1794 MAKE_CASE(ARMISD::VLD2DUP_UPD)
1795 MAKE_CASE(ARMISD::VLD3DUP_UPD)
1796 MAKE_CASE(ARMISD::VLD4DUP_UPD)
1797 MAKE_CASE(ARMISD::VST1_UPD)
1798 MAKE_CASE(ARMISD::VST2_UPD)
1799 MAKE_CASE(ARMISD::VST3_UPD)
1800 MAKE_CASE(ARMISD::VST4_UPD)
1801 MAKE_CASE(ARMISD::VST1x2_UPD)
1802 MAKE_CASE(ARMISD::VST1x3_UPD)
1803 MAKE_CASE(ARMISD::VST1x4_UPD)
1804 MAKE_CASE(ARMISD::VST2LN_UPD)
1805 MAKE_CASE(ARMISD::VST3LN_UPD)
1806 MAKE_CASE(ARMISD::VST4LN_UPD)
1807 MAKE_CASE(ARMISD::WLS)
1808 MAKE_CASE(ARMISD::WLSSETUP)
1809 MAKE_CASE(ARMISD::LE)
1810 MAKE_CASE(ARMISD::LOOP_DEC)
1811 MAKE_CASE(ARMISD::CSINV)
1812 MAKE_CASE(ARMISD::CSNEG)
1813 MAKE_CASE(ARMISD::CSINC)
1814 MAKE_CASE(ARMISD::MEMCPYLOOP)
1815 MAKE_CASE(ARMISD::MEMSETLOOP)
1816#undef MAKE_CASE
1817 }
1818 return nullptr;
1819}
1820
1821EVT ARMTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
1822 EVT VT) const {
1823 if (!VT.isVector())
1824 return getPointerTy(DL);
1825
1826 // MVE has a predicate register.
1827 if (Subtarget->hasMVEIntegerOps() &&
1828 (VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8))
1829 return MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
1830 return VT.changeVectorElementTypeToInteger();
1831}
1832
1833/// getRegClassFor - Return the register class that should be used for the
1834/// specified value type.
1835const TargetRegisterClass *
1836ARMTargetLowering::getRegClassFor(MVT VT, bool isDivergent) const {
1837 (void)isDivergent;
1838 // Map v4i64 to QQ registers but do not make the type legal. Similarly map
1839 // v8i64 to QQQQ registers. v4i64 and v8i64 are only used for REG_SEQUENCE to
1840 // load / store 4 to 8 consecutive NEON D registers, or 2 to 4 consecutive
1841 // MVE Q registers.
1842 if (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) {
1843 if (VT == MVT::v4i64)
1844 return &ARM::QQPRRegClass;
1845 if (VT == MVT::v8i64)
1846 return &ARM::QQQQPRRegClass;
1847 }
1848 return TargetLowering::getRegClassFor(VT);
1849}
1850
1851// memcpy, and other memory intrinsics, typically tries to use LDM/STM if the
1852// source/dest is aligned and the copy size is large enough. We therefore want
1853// to align such objects passed to memory intrinsics.
1854bool ARMTargetLowering::shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize,
1855 unsigned &PrefAlign) const {
1856 if (!isa<MemIntrinsic>(CI))
1857 return false;
1858 MinSize = 8;
1859 // On ARM11 onwards (excluding M class) 8-byte aligned LDM is typically 1
1860 // cycle faster than 4-byte aligned LDM.
1861 PrefAlign = (Subtarget->hasV6Ops() && !Subtarget->isMClass() ? 8 : 4);
1862 return true;
1863}
1864
1865// Create a fast isel object.
1866FastISel *
1867ARMTargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1868 const TargetLibraryInfo *libInfo) const {
1869 return ARM::createFastISel(funcInfo, libInfo);
1870}
1871
1872Sched::Preference ARMTargetLowering::getSchedulingPreference(SDNode *N) const {
1873 unsigned NumVals = N->getNumValues();
1874 if (!NumVals)
1875 return Sched::RegPressure;
1876
1877 for (unsigned i = 0; i != NumVals; ++i) {
1878 EVT VT = N->getValueType(i);
1879 if (VT == MVT::Glue || VT == MVT::Other)
1880 continue;
1881 if (VT.isFloatingPoint() || VT.isVector())
1882 return Sched::ILP;
1883 }
1884
1885 if (!N->isMachineOpcode())
1886 return Sched::RegPressure;
1887
1888 // Load are scheduled for latency even if there instruction itinerary
1889 // is not available.
1890 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
1891 const MCInstrDesc &MCID = TII->get(N->getMachineOpcode());
1892
1893 if (MCID.getNumDefs() == 0)
1894 return Sched::RegPressure;
1895 if (!Itins->isEmpty() &&
1896 Itins->getOperandCycle(MCID.getSchedClass(), 0) > 2)
1897 return Sched::ILP;
1898
1899 return Sched::RegPressure;
1900}
1901
1902//===----------------------------------------------------------------------===//
1903// Lowering Code
1904//===----------------------------------------------------------------------===//
1905
1906static bool isSRL16(const SDValue &Op) {
1907 if (Op.getOpcode() != ISD::SRL)
1908 return false;
1909 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1910 return Const->getZExtValue() == 16;
1911 return false;
1912}
1913
1914static bool isSRA16(const SDValue &Op) {
1915 if (Op.getOpcode() != ISD::SRA)
1916 return false;
1917 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1918 return Const->getZExtValue() == 16;
1919 return false;
1920}
1921
1922static bool isSHL16(const SDValue &Op) {
1923 if (Op.getOpcode() != ISD::SHL)
1924 return false;
1925 if (auto Const = dyn_cast<ConstantSDNode>(Op.getOperand(1)))
1926 return Const->getZExtValue() == 16;
1927 return false;
1928}
1929
1930// Check for a signed 16-bit value. We special case SRA because it makes it
1931// more simple when also looking for SRAs that aren't sign extending a
1932// smaller value. Without the check, we'd need to take extra care with
1933// checking order for some operations.
1934static bool isS16(const SDValue &Op, SelectionDAG &DAG) {
1935 if (isSRA16(Op))
1936 return isSHL16(Op.getOperand(0));
1937 return DAG.ComputeNumSignBits(Op) == 17;
1938}
1939
1940/// IntCCToARMCC - Convert a DAG integer condition code to an ARM CC
1941static ARMCC::CondCodes IntCCToARMCC(ISD::CondCode CC) {
1942 switch (CC) {
1943 default: llvm_unreachable("Unknown condition code!")::llvm::llvm_unreachable_internal("Unknown condition code!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 1943)
;
1944 case ISD::SETNE: return ARMCC::NE;
1945 case ISD::SETEQ: return ARMCC::EQ;
1946 case ISD::SETGT: return ARMCC::GT;
1947 case ISD::SETGE: return ARMCC::GE;
1948 case ISD::SETLT: return ARMCC::LT;
1949 case ISD::SETLE: return ARMCC::LE;
1950 case ISD::SETUGT: return ARMCC::HI;
1951 case ISD::SETUGE: return ARMCC::HS;
1952 case ISD::SETULT: return ARMCC::LO;
1953 case ISD::SETULE: return ARMCC::LS;
1954 }
1955}
1956
1957/// FPCCToARMCC - Convert a DAG fp condition code to an ARM CC.
1958static void FPCCToARMCC(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
1959 ARMCC::CondCodes &CondCode2) {
1960 CondCode2 = ARMCC::AL;
1961 switch (CC) {
1962 default: llvm_unreachable("Unknown FP condition!")::llvm::llvm_unreachable_internal("Unknown FP condition!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 1962)
;
1963 case ISD::SETEQ:
1964 case ISD::SETOEQ: CondCode = ARMCC::EQ; break;
1965 case ISD::SETGT:
1966 case ISD::SETOGT: CondCode = ARMCC::GT; break;
1967 case ISD::SETGE:
1968 case ISD::SETOGE: CondCode = ARMCC::GE; break;
1969 case ISD::SETOLT: CondCode = ARMCC::MI; break;
1970 case ISD::SETOLE: CondCode = ARMCC::LS; break;
1971 case ISD::SETONE: CondCode = ARMCC::MI; CondCode2 = ARMCC::GT; break;
1972 case ISD::SETO: CondCode = ARMCC::VC; break;
1973 case ISD::SETUO: CondCode = ARMCC::VS; break;
1974 case ISD::SETUEQ: CondCode = ARMCC::EQ; CondCode2 = ARMCC::VS; break;
1975 case ISD::SETUGT: CondCode = ARMCC::HI; break;
1976 case ISD::SETUGE: CondCode = ARMCC::PL; break;
1977 case ISD::SETLT:
1978 case ISD::SETULT: CondCode = ARMCC::LT; break;
1979 case ISD::SETLE:
1980 case ISD::SETULE: CondCode = ARMCC::LE; break;
1981 case ISD::SETNE:
1982 case ISD::SETUNE: CondCode = ARMCC::NE; break;
1983 }
1984}
1985
1986//===----------------------------------------------------------------------===//
1987// Calling Convention Implementation
1988//===----------------------------------------------------------------------===//
1989
1990/// getEffectiveCallingConv - Get the effective calling convention, taking into
1991/// account presence of floating point hardware and calling convention
1992/// limitations, such as support for variadic functions.
1993CallingConv::ID
1994ARMTargetLowering::getEffectiveCallingConv(CallingConv::ID CC,
1995 bool isVarArg) const {
1996 switch (CC) {
1997 default:
1998 report_fatal_error("Unsupported calling convention");
1999 case CallingConv::ARM_AAPCS:
2000 case CallingConv::ARM_APCS:
2001 case CallingConv::GHC:
2002 case CallingConv::CFGuard_Check:
2003 return CC;
2004 case CallingConv::PreserveMost:
2005 return CallingConv::PreserveMost;
2006 case CallingConv::ARM_AAPCS_VFP:
2007 case CallingConv::Swift:
2008 case CallingConv::SwiftTail:
2009 return isVarArg ? CallingConv::ARM_AAPCS : CallingConv::ARM_AAPCS_VFP;
2010 case CallingConv::C:
2011 if (!Subtarget->isAAPCS_ABI())
2012 return CallingConv::ARM_APCS;
2013 else if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() &&
2014 getTargetMachine().Options.FloatABIType == FloatABI::Hard &&
2015 !isVarArg)
2016 return CallingConv::ARM_AAPCS_VFP;
2017 else
2018 return CallingConv::ARM_AAPCS;
2019 case CallingConv::Fast:
2020 case CallingConv::CXX_FAST_TLS:
2021 if (!Subtarget->isAAPCS_ABI()) {
2022 if (Subtarget->hasVFP2Base() && !Subtarget->isThumb1Only() && !isVarArg)
2023 return CallingConv::Fast;
2024 return CallingConv::ARM_APCS;
2025 } else if (Subtarget->hasVFP2Base() &&
2026 !Subtarget->isThumb1Only() && !isVarArg)
2027 return CallingConv::ARM_AAPCS_VFP;
2028 else
2029 return CallingConv::ARM_AAPCS;
2030 }
2031}
2032
2033CCAssignFn *ARMTargetLowering::CCAssignFnForCall(CallingConv::ID CC,
2034 bool isVarArg) const {
2035 return CCAssignFnForNode(CC, false, isVarArg);
2036}
2037
2038CCAssignFn *ARMTargetLowering::CCAssignFnForReturn(CallingConv::ID CC,
2039 bool isVarArg) const {
2040 return CCAssignFnForNode(CC, true, isVarArg);
2041}
2042
2043/// CCAssignFnForNode - Selects the correct CCAssignFn for the given
2044/// CallingConvention.
2045CCAssignFn *ARMTargetLowering::CCAssignFnForNode(CallingConv::ID CC,
2046 bool Return,
2047 bool isVarArg) const {
2048 switch (getEffectiveCallingConv(CC, isVarArg)) {
2049 default:
2050 report_fatal_error("Unsupported calling convention");
2051 case CallingConv::ARM_APCS:
2052 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS);
2053 case CallingConv::ARM_AAPCS:
2054 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
2055 case CallingConv::ARM_AAPCS_VFP:
2056 return (Return ? RetCC_ARM_AAPCS_VFP : CC_ARM_AAPCS_VFP);
2057 case CallingConv::Fast:
2058 return (Return ? RetFastCC_ARM_APCS : FastCC_ARM_APCS);
2059 case CallingConv::GHC:
2060 return (Return ? RetCC_ARM_APCS : CC_ARM_APCS_GHC);
2061 case CallingConv::PreserveMost:
2062 return (Return ? RetCC_ARM_AAPCS : CC_ARM_AAPCS);
2063 case CallingConv::CFGuard_Check:
2064 return (Return ? RetCC_ARM_AAPCS : CC_ARM_Win32_CFGuard_Check);
2065 }
2066}
2067
2068SDValue ARMTargetLowering::MoveToHPR(const SDLoc &dl, SelectionDAG &DAG,
2069 MVT LocVT, MVT ValVT, SDValue Val) const {
2070 Val = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocVT.getSizeInBits()),
2071 Val);
2072 if (Subtarget->hasFullFP16()) {
2073 Val = DAG.getNode(ARMISD::VMOVhr, dl, ValVT, Val);
2074 } else {
2075 Val = DAG.getNode(ISD::TRUNCATE, dl,
2076 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
2077 Val = DAG.getNode(ISD::BITCAST, dl, ValVT, Val);
2078 }
2079 return Val;
2080}
2081
2082SDValue ARMTargetLowering::MoveFromHPR(const SDLoc &dl, SelectionDAG &DAG,
2083 MVT LocVT, MVT ValVT,
2084 SDValue Val) const {
2085 if (Subtarget->hasFullFP16()) {
2086 Val = DAG.getNode(ARMISD::VMOVrh, dl,
2087 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
2088 } else {
2089 Val = DAG.getNode(ISD::BITCAST, dl,
2090 MVT::getIntegerVT(ValVT.getSizeInBits()), Val);
2091 Val = DAG.getNode(ISD::ZERO_EXTEND, dl,
2092 MVT::getIntegerVT(LocVT.getSizeInBits()), Val);
2093 }
2094 return DAG.getNode(ISD::BITCAST, dl, LocVT, Val);
2095}
2096
2097/// LowerCallResult - Lower the result values of a call into the
2098/// appropriate copies out of appropriate physical registers.
2099SDValue ARMTargetLowering::LowerCallResult(
2100 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2101 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
2102 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
2103 SDValue ThisVal) const {
2104 // Assign locations to each value returned by this call.
2105 SmallVector<CCValAssign, 16> RVLocs;
2106 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2107 *DAG.getContext());
2108 CCInfo.AnalyzeCallResult(Ins, CCAssignFnForReturn(CallConv, isVarArg));
2109
2110 // Copy all of the result registers out of their specified physreg.
2111 for (unsigned i = 0; i != RVLocs.size(); ++i) {
2112 CCValAssign VA = RVLocs[i];
2113
2114 // Pass 'this' value directly from the argument to return value, to avoid
2115 // reg unit interference
2116 if (i == 0 && isThisReturn) {
2117 assert(!VA.needsCustom() && VA.getLocVT() == MVT::i32 &&(static_cast <bool> (!VA.needsCustom() && VA.getLocVT
() == MVT::i32 && "unexpected return calling convention register assignment"
) ? void (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i32 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2118, __extension__ __PRETTY_FUNCTION__))
2118 "unexpected return calling convention register assignment")(static_cast <bool> (!VA.needsCustom() && VA.getLocVT
() == MVT::i32 && "unexpected return calling convention register assignment"
) ? void (0) : __assert_fail ("!VA.needsCustom() && VA.getLocVT() == MVT::i32 && \"unexpected return calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2118, __extension__ __PRETTY_FUNCTION__))
;
2119 InVals.push_back(ThisVal);
2120 continue;
2121 }
2122
2123 SDValue Val;
2124 if (VA.needsCustom() &&
2125 (VA.getLocVT() == MVT::f64 || VA.getLocVT() == MVT::v2f64)) {
2126 // Handle f64 or half of a v2f64.
2127 SDValue Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2128 InFlag);
2129 Chain = Lo.getValue(1);
2130 InFlag = Lo.getValue(2);
2131 VA = RVLocs[++i]; // skip ahead to next loc
2132 SDValue Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32,
2133 InFlag);
2134 Chain = Hi.getValue(1);
2135 InFlag = Hi.getValue(2);
2136 if (!Subtarget->isLittle())
2137 std::swap (Lo, Hi);
2138 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2139
2140 if (VA.getLocVT() == MVT::v2f64) {
2141 SDValue Vec = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
2142 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2143 DAG.getConstant(0, dl, MVT::i32));
2144
2145 VA = RVLocs[++i]; // skip ahead to next loc
2146 Lo = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
2147 Chain = Lo.getValue(1);
2148 InFlag = Lo.getValue(2);
2149 VA = RVLocs[++i]; // skip ahead to next loc
2150 Hi = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), MVT::i32, InFlag);
2151 Chain = Hi.getValue(1);
2152 InFlag = Hi.getValue(2);
2153 if (!Subtarget->isLittle())
2154 std::swap (Lo, Hi);
2155 Val = DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
2156 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Vec, Val,
2157 DAG.getConstant(1, dl, MVT::i32));
2158 }
2159 } else {
2160 Val = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), VA.getLocVT(),
2161 InFlag);
2162 Chain = Val.getValue(1);
2163 InFlag = Val.getValue(2);
2164 }
2165
2166 switch (VA.getLocInfo()) {
2167 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2167)
;
2168 case CCValAssign::Full: break;
2169 case CCValAssign::BCvt:
2170 Val = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), Val);
2171 break;
2172 }
2173
2174 // f16 arguments have their size extended to 4 bytes and passed as if they
2175 // had been copied to the LSBs of a 32-bit register.
2176 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2177 if (VA.needsCustom() &&
2178 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
2179 Val = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Val);
2180
2181 InVals.push_back(Val);
2182 }
2183
2184 return Chain;
2185}
2186
2187/// LowerMemOpCallTo - Store the argument to the stack.
2188SDValue ARMTargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
2189 SDValue Arg, const SDLoc &dl,
2190 SelectionDAG &DAG,
2191 const CCValAssign &VA,
2192 ISD::ArgFlagsTy Flags) const {
2193 unsigned LocMemOffset = VA.getLocMemOffset();
2194 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2195 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
2196 StackPtr, PtrOff);
2197 return DAG.getStore(
2198 Chain, dl, Arg, PtrOff,
2199 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset));
2200}
2201
2202void ARMTargetLowering::PassF64ArgInRegs(const SDLoc &dl, SelectionDAG &DAG,
2203 SDValue Chain, SDValue &Arg,
2204 RegsToPassVector &RegsToPass,
2205 CCValAssign &VA, CCValAssign &NextVA,
2206 SDValue &StackPtr,
2207 SmallVectorImpl<SDValue> &MemOpChains,
2208 ISD::ArgFlagsTy Flags) const {
2209 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
2210 DAG.getVTList(MVT::i32, MVT::i32), Arg);
2211 unsigned id = Subtarget->isLittle() ? 0 : 1;
2212 RegsToPass.push_back(std::make_pair(VA.getLocReg(), fmrrd.getValue(id)));
2213
2214 if (NextVA.isRegLoc())
2215 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), fmrrd.getValue(1-id)));
2216 else {
2217 assert(NextVA.isMemLoc())(static_cast <bool> (NextVA.isMemLoc()) ? void (0) : __assert_fail
("NextVA.isMemLoc()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2217, __extension__ __PRETTY_FUNCTION__))
;
2218 if (!StackPtr.getNode())
2219 StackPtr = DAG.getCopyFromReg(Chain, dl, ARM::SP,
2220 getPointerTy(DAG.getDataLayout()));
2221
2222 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, fmrrd.getValue(1-id),
2223 dl, DAG, NextVA,
2224 Flags));
2225 }
2226}
2227
2228/// LowerCall - Lowering a call into a callseq_start <-
2229/// ARMISD:CALL <- callseq_end chain. Also add input and output parameter
2230/// nodes.
2231SDValue
2232ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
2233 SmallVectorImpl<SDValue> &InVals) const {
2234 SelectionDAG &DAG = CLI.DAG;
2235 SDLoc &dl = CLI.DL;
2236 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2237 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2238 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2239 SDValue Chain = CLI.Chain;
2240 SDValue Callee = CLI.Callee;
2241 bool &isTailCall = CLI.IsTailCall;
2242 CallingConv::ID CallConv = CLI.CallConv;
2243 bool doesNotRet = CLI.DoesNotReturn;
2244 bool isVarArg = CLI.IsVarArg;
2245
2246 MachineFunction &MF = DAG.getMachineFunction();
2247 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2248 MachineFunction::CallSiteInfo CSInfo;
2249 bool isStructRet = (Outs.empty()) ? false : Outs[0].Flags.isSRet();
1
'?' condition is false
2250 bool isThisReturn = false;
2251 bool isCmseNSCall = false;
2252 bool PreferIndirect = false;
2253
2254 // Determine whether this is a non-secure function call.
2255 if (CLI.CB && CLI.CB->getAttributes().hasFnAttribute("cmse_nonsecure_call"))
2
Assuming field 'CB' is null
3
Taking false branch
2256 isCmseNSCall = true;
2257
2258 // Disable tail calls if they're not supported.
2259 if (!Subtarget->supportsTailCall())
4
Assuming the condition is false
5
Taking false branch
2260 isTailCall = false;
2261
2262 // For both the non-secure calls and the returns from a CMSE entry function,
2263 // the function needs to do some extra work afte r the call, or before the
2264 // return, respectively, thus it cannot end with atail call
2265 if (isCmseNSCall
5.1
'isCmseNSCall' is false
|| AFI->isCmseNSEntryFunction())
6
Assuming the condition is false
7
Taking false branch
2266 isTailCall = false;
2267
2268 if (isa<GlobalAddressSDNode>(Callee)) {
8
Assuming 'Callee' is not a 'GlobalAddressSDNode'
9
Taking false branch
2269 // If we're optimizing for minimum size and the function is called three or
2270 // more times in this block, we can improve codesize by calling indirectly
2271 // as BLXr has a 16-bit encoding.
2272 auto *GV = cast<GlobalAddressSDNode>(Callee)->getGlobal();
2273 if (CLI.CB) {
2274 auto *BB = CLI.CB->getParent();
2275 PreferIndirect = Subtarget->isThumb() && Subtarget->hasMinSize() &&
2276 count_if(GV->users(), [&BB](const User *U) {
2277 return isa<Instruction>(U) &&
2278 cast<Instruction>(U)->getParent() == BB;
2279 }) > 2;
2280 }
2281 }
2282 if (isTailCall) {
10
Assuming 'isTailCall' is false
11
Taking false branch
2283 // Check if it's really possible to do a tail call.
2284 isTailCall = IsEligibleForTailCallOptimization(
2285 Callee, CallConv, isVarArg, isStructRet,
2286 MF.getFunction().hasStructRetAttr(), Outs, OutVals, Ins, DAG,
2287 PreferIndirect);
2288 if (!isTailCall && CLI.CB && CLI.CB->isMustTailCall())
2289 report_fatal_error("failed to perform tail call elimination on a call "
2290 "site marked musttail");
2291 // We don't support GuaranteedTailCallOpt for ARM, only automatically
2292 // detected sibcalls.
2293 if (isTailCall)
2294 ++NumTailCalls;
2295 }
2296
2297 // Analyze operands of the call, assigning locations to each operand.
2298 SmallVector<CCValAssign, 16> ArgLocs;
2299 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
2300 *DAG.getContext());
2301 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CallConv, isVarArg));
2302
2303 // Get a count of how many bytes are to be pushed on the stack.
2304 unsigned NumBytes = CCInfo.getNextStackOffset();
2305
2306 if (isTailCall) {
12
Assuming 'isTailCall' is false
13
Taking false branch
2307 // For tail calls, memory operands are available in our caller's stack.
2308 NumBytes = 0;
2309 } else {
2310 // Adjust the stack pointer for the new arguments...
2311 // These operations are automatically eliminated by the prolog/epilog pass
2312 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, dl);
2313 }
2314
2315 SDValue StackPtr =
2316 DAG.getCopyFromReg(Chain, dl, ARM::SP, getPointerTy(DAG.getDataLayout()));
2317
2318 RegsToPassVector RegsToPass;
2319 SmallVector<SDValue, 8> MemOpChains;
2320
2321 // Walk the register/memloc assignments, inserting copies/loads. In the case
2322 // of tail call optimization, arguments are handled later.
2323 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
15
Loop condition is false. Execution continues on line 2465
2324 i != e;
14
Assuming 'i' is equal to 'e'
2325 ++i, ++realArgIdx) {
2326 CCValAssign &VA = ArgLocs[i];
2327 SDValue Arg = OutVals[realArgIdx];
2328 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2329 bool isByVal = Flags.isByVal();
2330
2331 // Promote the value if needed.
2332 switch (VA.getLocInfo()) {
2333 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2333)
;
2334 case CCValAssign::Full: break;
2335 case CCValAssign::SExt:
2336 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
2337 break;
2338 case CCValAssign::ZExt:
2339 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
2340 break;
2341 case CCValAssign::AExt:
2342 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
2343 break;
2344 case CCValAssign::BCvt:
2345 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2346 break;
2347 }
2348
2349 // f16 arguments have their size extended to 4 bytes and passed as if they
2350 // had been copied to the LSBs of a 32-bit register.
2351 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
2352 if (VA.needsCustom() &&
2353 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16)) {
2354 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
2355 } else {
2356 // f16 arguments could have been extended prior to argument lowering.
2357 // Mask them arguments if this is a CMSE nonsecure call.
2358 auto ArgVT = Outs[realArgIdx].ArgVT;
2359 if (isCmseNSCall && (ArgVT == MVT::f16)) {
2360 auto LocBits = VA.getLocVT().getSizeInBits();
2361 auto MaskValue = APInt::getLowBitsSet(LocBits, ArgVT.getSizeInBits());
2362 SDValue Mask =
2363 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
2364 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
2365 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
2366 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
2367 }
2368 }
2369
2370 // f64 and v2f64 might be passed in i32 pairs and must be split into pieces
2371 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
2372 SDValue Op0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2373 DAG.getConstant(0, dl, MVT::i32));
2374 SDValue Op1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
2375 DAG.getConstant(1, dl, MVT::i32));
2376
2377 PassF64ArgInRegs(dl, DAG, Chain, Op0, RegsToPass, VA, ArgLocs[++i],
2378 StackPtr, MemOpChains, Flags);
2379
2380 VA = ArgLocs[++i]; // skip ahead to next loc
2381 if (VA.isRegLoc()) {
2382 PassF64ArgInRegs(dl, DAG, Chain, Op1, RegsToPass, VA, ArgLocs[++i],
2383 StackPtr, MemOpChains, Flags);
2384 } else {
2385 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2385, __extension__ __PRETTY_FUNCTION__))
;
2386
2387 MemOpChains.push_back(
2388 LowerMemOpCallTo(Chain, StackPtr, Op1, dl, DAG, VA, Flags));
2389 }
2390 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
2391 PassF64ArgInRegs(dl, DAG, Chain, Arg, RegsToPass, VA, ArgLocs[++i],
2392 StackPtr, MemOpChains, Flags);
2393 } else if (VA.isRegLoc()) {
2394 if (realArgIdx == 0 && Flags.isReturned() && !Flags.isSwiftSelf() &&
2395 Outs[0].VT == MVT::i32) {
2396 assert(VA.getLocVT() == MVT::i32 &&(static_cast <bool> (VA.getLocVT() == MVT::i32 &&
"unexpected calling convention register assignment") ? void (
0) : __assert_fail ("VA.getLocVT() == MVT::i32 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2397, __extension__ __PRETTY_FUNCTION__))
2397 "unexpected calling convention register assignment")(static_cast <bool> (VA.getLocVT() == MVT::i32 &&
"unexpected calling convention register assignment") ? void (
0) : __assert_fail ("VA.getLocVT() == MVT::i32 && \"unexpected calling convention register assignment\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2397, __extension__ __PRETTY_FUNCTION__))
;
2398 assert(!Ins.empty() && Ins[0].VT == MVT::i32 &&(static_cast <bool> (!Ins.empty() && Ins[0].VT ==
MVT::i32 && "unexpected use of 'returned'") ? void (
0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i32 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2399, __extension__ __PRETTY_FUNCTION__))
2399 "unexpected use of 'returned'")(static_cast <bool> (!Ins.empty() && Ins[0].VT ==
MVT::i32 && "unexpected use of 'returned'") ? void (
0) : __assert_fail ("!Ins.empty() && Ins[0].VT == MVT::i32 && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2399, __extension__ __PRETTY_FUNCTION__))
;
2400 isThisReturn = true;
2401 }
2402 const TargetOptions &Options = DAG.getTarget().Options;
2403 if (Options.EmitCallSiteInfo)
2404 CSInfo.emplace_back(VA.getLocReg(), i);
2405 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
2406 } else if (isByVal) {
2407 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2407, __extension__ __PRETTY_FUNCTION__))
;
2408 unsigned offset = 0;
2409
2410 // True if this byval aggregate will be split between registers
2411 // and memory.
2412 unsigned ByValArgsCount = CCInfo.getInRegsParamsCount();
2413 unsigned CurByValIdx = CCInfo.getInRegsParamsProcessed();
2414
2415 if (CurByValIdx < ByValArgsCount) {
2416
2417 unsigned RegBegin, RegEnd;
2418 CCInfo.getInRegsParamInfo(CurByValIdx, RegBegin, RegEnd);
2419
2420 EVT PtrVT =
2421 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2422 unsigned int i, j;
2423 for (i = 0, j = RegBegin; j < RegEnd; i++, j++) {
2424 SDValue Const = DAG.getConstant(4*i, dl, MVT::i32);
2425 SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
2426 SDValue Load =
2427 DAG.getLoad(PtrVT, dl, Chain, AddArg, MachinePointerInfo(),
2428 DAG.InferPtrAlign(AddArg));
2429 MemOpChains.push_back(Load.getValue(1));
2430 RegsToPass.push_back(std::make_pair(j, Load));
2431 }
2432
2433 // If parameter size outsides register area, "offset" value
2434 // helps us to calculate stack slot for remained part properly.
2435 offset = RegEnd - RegBegin;
2436
2437 CCInfo.nextInRegsParam();
2438 }
2439
2440 if (Flags.getByValSize() > 4*offset) {
2441 auto PtrVT = getPointerTy(DAG.getDataLayout());
2442 unsigned LocMemOffset = VA.getLocMemOffset();
2443 SDValue StkPtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
2444 SDValue Dst = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, StkPtrOff);
2445 SDValue SrcOffset = DAG.getIntPtrConstant(4*offset, dl);
2446 SDValue Src = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, SrcOffset);
2447 SDValue SizeNode = DAG.getConstant(Flags.getByValSize() - 4*offset, dl,
2448 MVT::i32);
2449 SDValue AlignNode =
2450 DAG.getConstant(Flags.getNonZeroByValAlign().value(), dl, MVT::i32);
2451
2452 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
2453 SDValue Ops[] = { Chain, Dst, Src, SizeNode, AlignNode};
2454 MemOpChains.push_back(DAG.getNode(ARMISD::COPY_STRUCT_BYVAL, dl, VTs,
2455 Ops));
2456 }
2457 } else if (!isTailCall) {
2458 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2458, __extension__ __PRETTY_FUNCTION__))
;
2459
2460 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
2461 dl, DAG, VA, Flags));
2462 }
2463 }
2464
2465 if (!MemOpChains.empty())
16
Taking false branch
2466 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
2467
2468 // Build a sequence of copy-to-reg nodes chained together with token chain
2469 // and flag operands which copy the outgoing args into the appropriate regs.
2470 SDValue InFlag;
2471 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
17
Assuming 'i' is equal to 'e'
18
Loop condition is false. Execution continues on line 2480
2472 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
2473 RegsToPass[i].second, InFlag);
2474 InFlag = Chain.getValue(1);
2475 }
2476
2477 // If the callee is a GlobalAddress/ExternalSymbol node (quite common, every
2478 // direct call is) turn it into a TargetGlobalAddress/TargetExternalSymbol
2479 // node so that legalize doesn't hack it.
2480 bool isDirect = false;
2481
2482 const TargetMachine &TM = getTargetMachine();
2483 const Module *Mod = MF.getFunction().getParent();
2484 const GlobalValue *GV = nullptr;
19
'GV' initialized to a null pointer value
2485 if (GlobalAddressSDNode *G
19.1
'G' is null
= dyn_cast<GlobalAddressSDNode>(Callee))
20
Taking false branch
2486 GV = G->getGlobal();
2487 bool isStub =
2488 !TM.shouldAssumeDSOLocal(*Mod, GV) && Subtarget->isTargetMachO();
21
Assuming the condition is false
2489
2490 bool isARMFunc = !Subtarget->isThumb() || (isStub && !Subtarget->isMClass());
22
Assuming the condition is true
2491 bool isLocalARMFunc = false;
2492 auto PtrVt = getPointerTy(DAG.getDataLayout());
2493
2494 if (Subtarget->genLongCalls()) {
23
Assuming the condition is false
24
Taking false branch
2495 assert((!isPositionIndependent() || Subtarget->isTargetWindows()) &&(static_cast <bool> ((!isPositionIndependent() || Subtarget
->isTargetWindows()) && "long-calls codegen is not position independent!"
) ? void (0) : __assert_fail ("(!isPositionIndependent() || Subtarget->isTargetWindows()) && \"long-calls codegen is not position independent!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2496, __extension__ __PRETTY_FUNCTION__))
2496 "long-calls codegen is not position independent!")(static_cast <bool> ((!isPositionIndependent() || Subtarget
->isTargetWindows()) && "long-calls codegen is not position independent!"
) ? void (0) : __assert_fail ("(!isPositionIndependent() || Subtarget->isTargetWindows()) && \"long-calls codegen is not position independent!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2496, __extension__ __PRETTY_FUNCTION__))
;
2497 // Handle a global address or an external symbol. If it's not one of
2498 // those, the target's already in a register, so we don't need to do
2499 // anything extra.
2500 if (isa<GlobalAddressSDNode>(Callee)) {
2501 // Create a constant pool entry for the callee address
2502 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2503 ARMConstantPoolValue *CPV =
2504 ARMConstantPoolConstant::Create(GV, ARMPCLabelIndex, ARMCP::CPValue, 0);
2505
2506 // Get the address of the callee into a register
2507 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2508 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2509 Callee = DAG.getLoad(
2510 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2511 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2512 } else if (ExternalSymbolSDNode *S=dyn_cast<ExternalSymbolSDNode>(Callee)) {
2513 const char *Sym = S->getSymbol();
2514
2515 // Create a constant pool entry for the callee address
2516 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2517 ARMConstantPoolValue *CPV =
2518 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2519 ARMPCLabelIndex, 0);
2520 // Get the address of the callee into a register
2521 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2522 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2523 Callee = DAG.getLoad(
2524 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2525 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2526 }
2527 } else if (isa<GlobalAddressSDNode>(Callee)) {
25
Assuming 'Callee' is a 'GlobalAddressSDNode'
26
Taking true branch
2528 if (!PreferIndirect
26.1
'PreferIndirect' is false
) {
27
Taking true branch
2529 isDirect = true;
2530 bool isDef = GV->isStrongDefinitionForLinker();
28
Called C++ object pointer is null
2531
2532 // ARM call to a local ARM function is predicable.
2533 isLocalARMFunc = !Subtarget->isThumb() && (isDef || !ARMInterworking);
2534 // tBX takes a register source operand.
2535 if (isStub && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2536 assert(Subtarget->isTargetMachO() && "WrapperPIC use on non-MachO?")(static_cast <bool> (Subtarget->isTargetMachO() &&
"WrapperPIC use on non-MachO?") ? void (0) : __assert_fail (
"Subtarget->isTargetMachO() && \"WrapperPIC use on non-MachO?\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2536, __extension__ __PRETTY_FUNCTION__))
;
2537 Callee = DAG.getNode(
2538 ARMISD::WrapperPIC, dl, PtrVt,
2539 DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY));
2540 Callee = DAG.getLoad(
2541 PtrVt, dl, DAG.getEntryNode(), Callee,
2542 MachinePointerInfo::getGOT(DAG.getMachineFunction()), MaybeAlign(),
2543 MachineMemOperand::MODereferenceable |
2544 MachineMemOperand::MOInvariant);
2545 } else if (Subtarget->isTargetCOFF()) {
2546 assert(Subtarget->isTargetWindows() &&(static_cast <bool> (Subtarget->isTargetWindows() &&
"Windows is the only supported COFF target") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2547, __extension__ __PRETTY_FUNCTION__))
2547 "Windows is the only supported COFF target")(static_cast <bool> (Subtarget->isTargetWindows() &&
"Windows is the only supported COFF target") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"Windows is the only supported COFF target\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2547, __extension__ __PRETTY_FUNCTION__))
;
2548 unsigned TargetFlags = ARMII::MO_NO_FLAG;
2549 if (GV->hasDLLImportStorageClass())
2550 TargetFlags = ARMII::MO_DLLIMPORT;
2551 else if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
2552 TargetFlags = ARMII::MO_COFFSTUB;
2553 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, /*offset=*/0,
2554 TargetFlags);
2555 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
2556 Callee =
2557 DAG.getLoad(PtrVt, dl, DAG.getEntryNode(),
2558 DAG.getNode(ARMISD::Wrapper, dl, PtrVt, Callee),
2559 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
2560 } else {
2561 Callee = DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, 0);
2562 }
2563 }
2564 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2565 isDirect = true;
2566 // tBX takes a register source operand.
2567 const char *Sym = S->getSymbol();
2568 if (isARMFunc && Subtarget->isThumb1Only() && !Subtarget->hasV5TOps()) {
2569 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
2570 ARMConstantPoolValue *CPV =
2571 ARMConstantPoolSymbol::Create(*DAG.getContext(), Sym,
2572 ARMPCLabelIndex, 4);
2573 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVt, Align(4));
2574 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
2575 Callee = DAG.getLoad(
2576 PtrVt, dl, DAG.getEntryNode(), CPAddr,
2577 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
2578 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
2579 Callee = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVt, Callee, PICLabel);
2580 } else {
2581 Callee = DAG.getTargetExternalSymbol(Sym, PtrVt, 0);
2582 }
2583 }
2584
2585 if (isCmseNSCall) {
2586 assert(!isARMFunc && !isDirect &&(static_cast <bool> (!isARMFunc && !isDirect &&
"Cannot handle call to ARM function or direct call") ? void (
0) : __assert_fail ("!isARMFunc && !isDirect && \"Cannot handle call to ARM function or direct call\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2587, __extension__ __PRETTY_FUNCTION__))
2587 "Cannot handle call to ARM function or direct call")(static_cast <bool> (!isARMFunc && !isDirect &&
"Cannot handle call to ARM function or direct call") ? void (
0) : __assert_fail ("!isARMFunc && !isDirect && \"Cannot handle call to ARM function or direct call\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2587, __extension__ __PRETTY_FUNCTION__))
;
2588 if (NumBytes > 0) {
2589 DiagnosticInfoUnsupported Diag(DAG.getMachineFunction().getFunction(),
2590 "call to non-secure function would "
2591 "require passing arguments on stack",
2592 dl.getDebugLoc());
2593 DAG.getContext()->diagnose(Diag);
2594 }
2595 if (isStructRet) {
2596 DiagnosticInfoUnsupported Diag(
2597 DAG.getMachineFunction().getFunction(),
2598 "call to non-secure function would return value through pointer",
2599 dl.getDebugLoc());
2600 DAG.getContext()->diagnose(Diag);
2601 }
2602 }
2603
2604 // FIXME: handle tail calls differently.
2605 unsigned CallOpc;
2606 if (Subtarget->isThumb()) {
2607 if (isCmseNSCall)
2608 CallOpc = ARMISD::tSECALL;
2609 else if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
2610 CallOpc = ARMISD::CALL_NOLINK;
2611 else
2612 CallOpc = ARMISD::CALL;
2613 } else {
2614 if (!isDirect && !Subtarget->hasV5TOps())
2615 CallOpc = ARMISD::CALL_NOLINK;
2616 else if (doesNotRet && isDirect && Subtarget->hasRetAddrStack() &&
2617 // Emit regular call when code size is the priority
2618 !Subtarget->hasMinSize())
2619 // "mov lr, pc; b _foo" to avoid confusing the RSP
2620 CallOpc = ARMISD::CALL_NOLINK;
2621 else
2622 CallOpc = isLocalARMFunc ? ARMISD::CALL_PRED : ARMISD::CALL;
2623 }
2624
2625 std::vector<SDValue> Ops;
2626 Ops.push_back(Chain);
2627 Ops.push_back(Callee);
2628
2629 // Add argument registers to the end of the list so that they are known live
2630 // into the call.
2631 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
2632 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
2633 RegsToPass[i].second.getValueType()));
2634
2635 // Add a register mask operand representing the call-preserved registers.
2636 if (!isTailCall) {
2637 const uint32_t *Mask;
2638 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
2639 if (isThisReturn) {
2640 // For 'this' returns, use the R0-preserving mask if applicable
2641 Mask = ARI->getThisReturnPreservedMask(MF, CallConv);
2642 if (!Mask) {
2643 // Set isThisReturn to false if the calling convention is not one that
2644 // allows 'returned' to be modeled in this way, so LowerCallResult does
2645 // not try to pass 'this' straight through
2646 isThisReturn = false;
2647 Mask = ARI->getCallPreservedMask(MF, CallConv);
2648 }
2649 } else
2650 Mask = ARI->getCallPreservedMask(MF, CallConv);
2651
2652 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2652, __extension__ __PRETTY_FUNCTION__))
;
2653 Ops.push_back(DAG.getRegisterMask(Mask));
2654 }
2655
2656 if (InFlag.getNode())
2657 Ops.push_back(InFlag);
2658
2659 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2660 if (isTailCall) {
2661 MF.getFrameInfo().setHasTailCall();
2662 SDValue Ret = DAG.getNode(ARMISD::TC_RETURN, dl, NodeTys, Ops);
2663 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
2664 return Ret;
2665 }
2666
2667 // Returns a chain and a flag for retval copy to use.
2668 Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops);
2669 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
2670 InFlag = Chain.getValue(1);
2671 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
2672
2673 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(NumBytes, dl, true),
2674 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
2675 if (!Ins.empty())
2676 InFlag = Chain.getValue(1);
2677
2678 // Handle result values, copying them out of physregs into vregs that we
2679 // return.
2680 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
2681 InVals, isThisReturn,
2682 isThisReturn ? OutVals[0] : SDValue());
2683}
2684
2685/// HandleByVal - Every parameter *after* a byval parameter is passed
2686/// on the stack. Remember the next parameter register to allocate,
2687/// and then confiscate the rest of the parameter registers to insure
2688/// this.
2689void ARMTargetLowering::HandleByVal(CCState *State, unsigned &Size,
2690 Align Alignment) const {
2691 // Byval (as with any stack) slots are always at least 4 byte aligned.
2692 Alignment = std::max(Alignment, Align(4));
2693
2694 unsigned Reg = State->AllocateReg(GPRArgRegs);
2695 if (!Reg)
2696 return;
2697
2698 unsigned AlignInRegs = Alignment.value() / 4;
2699 unsigned Waste = (ARM::R4 - Reg) % AlignInRegs;
2700 for (unsigned i = 0; i < Waste; ++i)
2701 Reg = State->AllocateReg(GPRArgRegs);
2702
2703 if (!Reg)
2704 return;
2705
2706 unsigned Excess = 4 * (ARM::R4 - Reg);
2707
2708 // Special case when NSAA != SP and parameter size greater than size of
2709 // all remained GPR regs. In that case we can't split parameter, we must
2710 // send it to stack. We also must set NCRN to R4, so waste all
2711 // remained registers.
2712 const unsigned NSAAOffset = State->getNextStackOffset();
2713 if (NSAAOffset != 0 && Size > Excess) {
2714 while (State->AllocateReg(GPRArgRegs))
2715 ;
2716 return;
2717 }
2718
2719 // First register for byval parameter is the first register that wasn't
2720 // allocated before this method call, so it would be "reg".
2721 // If parameter is small enough to be saved in range [reg, r4), then
2722 // the end (first after last) register would be reg + param-size-in-regs,
2723 // else parameter would be splitted between registers and stack,
2724 // end register would be r4 in this case.
2725 unsigned ByValRegBegin = Reg;
2726 unsigned ByValRegEnd = std::min<unsigned>(Reg + Size / 4, ARM::R4);
2727 State->addInRegsParamInfo(ByValRegBegin, ByValRegEnd);
2728 // Note, first register is allocated in the beginning of function already,
2729 // allocate remained amount of registers we need.
2730 for (unsigned i = Reg + 1; i != ByValRegEnd; ++i)
2731 State->AllocateReg(GPRArgRegs);
2732 // A byval parameter that is split between registers and memory needs its
2733 // size truncated here.
2734 // In the case where the entire structure fits in registers, we set the
2735 // size in memory to zero.
2736 Size = std::max<int>(Size - Excess, 0);
2737}
2738
2739/// MatchingStackOffset - Return true if the given stack call argument is
2740/// already available in the same position (relatively) of the caller's
2741/// incoming argument stack.
2742static
2743bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
2744 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
2745 const TargetInstrInfo *TII) {
2746 unsigned Bytes = Arg.getValueSizeInBits() / 8;
2747 int FI = std::numeric_limits<int>::max();
2748 if (Arg.getOpcode() == ISD::CopyFromReg) {
2749 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
2750 if (!Register::isVirtualRegister(VR))
2751 return false;
2752 MachineInstr *Def = MRI->getVRegDef(VR);
2753 if (!Def)
2754 return false;
2755 if (!Flags.isByVal()) {
2756 if (!TII->isLoadFromStackSlot(*Def, FI))
2757 return false;
2758 } else {
2759 return false;
2760 }
2761 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
2762 if (Flags.isByVal())
2763 // ByVal argument is passed in as a pointer but it's now being
2764 // dereferenced. e.g.
2765 // define @foo(%struct.X* %A) {
2766 // tail call @bar(%struct.X* byval %A)
2767 // }
2768 return false;
2769 SDValue Ptr = Ld->getBasePtr();
2770 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
2771 if (!FINode)
2772 return false;
2773 FI = FINode->getIndex();
2774 } else
2775 return false;
2776
2777 assert(FI != std::numeric_limits<int>::max())(static_cast <bool> (FI != std::numeric_limits<int>
::max()) ? void (0) : __assert_fail ("FI != std::numeric_limits<int>::max()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2777, __extension__ __PRETTY_FUNCTION__))
;
2778 if (!MFI.isFixedObjectIndex(FI))
2779 return false;
2780 return Offset == MFI.getObjectOffset(FI) && Bytes == MFI.getObjectSize(FI);
2781}
2782
2783/// IsEligibleForTailCallOptimization - Check whether the call is eligible
2784/// for tail call optimization. Targets which want to do tail call
2785/// optimization should implement this function.
2786bool ARMTargetLowering::IsEligibleForTailCallOptimization(
2787 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
2788 bool isCalleeStructRet, bool isCallerStructRet,
2789 const SmallVectorImpl<ISD::OutputArg> &Outs,
2790 const SmallVectorImpl<SDValue> &OutVals,
2791 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG,
2792 const bool isIndirect) const {
2793 MachineFunction &MF = DAG.getMachineFunction();
2794 const Function &CallerF = MF.getFunction();
2795 CallingConv::ID CallerCC = CallerF.getCallingConv();
2796
2797 assert(Subtarget->supportsTailCall())(static_cast <bool> (Subtarget->supportsTailCall()) ?
void (0) : __assert_fail ("Subtarget->supportsTailCall()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2797, __extension__ __PRETTY_FUNCTION__))
;
2798
2799 // Indirect tail calls cannot be optimized for Thumb1 if the args
2800 // to the call take up r0-r3. The reason is that there are no legal registers
2801 // left to hold the pointer to the function to be called.
2802 if (Subtarget->isThumb1Only() && Outs.size() >= 4 &&
2803 (!isa<GlobalAddressSDNode>(Callee.getNode()) || isIndirect))
2804 return false;
2805
2806 // Look for obvious safe cases to perform tail call optimization that do not
2807 // require ABI changes. This is what gcc calls sibcall.
2808
2809 // Exception-handling functions need a special set of instructions to indicate
2810 // a return to the hardware. Tail-calling another function would probably
2811 // break this.
2812 if (CallerF.hasFnAttribute("interrupt"))
2813 return false;
2814
2815 // Also avoid sibcall optimization if either caller or callee uses struct
2816 // return semantics.
2817 if (isCalleeStructRet || isCallerStructRet)
2818 return false;
2819
2820 // Externally-defined functions with weak linkage should not be
2821 // tail-called on ARM when the OS does not support dynamic
2822 // pre-emption of symbols, as the AAELF spec requires normal calls
2823 // to undefined weak functions to be replaced with a NOP or jump to the
2824 // next instruction. The behaviour of branch instructions in this
2825 // situation (as used for tail calls) is implementation-defined, so we
2826 // cannot rely on the linker replacing the tail call with a return.
2827 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2828 const GlobalValue *GV = G->getGlobal();
2829 const Triple &TT = getTargetMachine().getTargetTriple();
2830 if (GV->hasExternalWeakLinkage() &&
2831 (!TT.isOSWindows() || TT.isOSBinFormatELF() || TT.isOSBinFormatMachO()))
2832 return false;
2833 }
2834
2835 // Check that the call results are passed in the same way.
2836 LLVMContext &C = *DAG.getContext();
2837 if (!CCState::resultsCompatible(
2838 getEffectiveCallingConv(CalleeCC, isVarArg),
2839 getEffectiveCallingConv(CallerCC, CallerF.isVarArg()), MF, C, Ins,
2840 CCAssignFnForReturn(CalleeCC, isVarArg),
2841 CCAssignFnForReturn(CallerCC, CallerF.isVarArg())))
2842 return false;
2843 // The callee has to preserve all registers the caller needs to preserve.
2844 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
2845 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2846 if (CalleeCC != CallerCC) {
2847 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2848 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
2849 return false;
2850 }
2851
2852 // If Caller's vararg or byval argument has been split between registers and
2853 // stack, do not perform tail call, since part of the argument is in caller's
2854 // local frame.
2855 const ARMFunctionInfo *AFI_Caller = MF.getInfo<ARMFunctionInfo>();
2856 if (AFI_Caller->getArgRegsSaveSize())
2857 return false;
2858
2859 // If the callee takes no arguments then go on to check the results of the
2860 // call.
2861 if (!Outs.empty()) {
2862 // Check if stack adjustment is needed. For now, do not do this if any
2863 // argument is passed on the stack.
2864 SmallVector<CCValAssign, 16> ArgLocs;
2865 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
2866 CCInfo.AnalyzeCallOperands(Outs, CCAssignFnForCall(CalleeCC, isVarArg));
2867 if (CCInfo.getNextStackOffset()) {
2868 // Check if the arguments are already laid out in the right way as
2869 // the caller's fixed stack objects.
2870 MachineFrameInfo &MFI = MF.getFrameInfo();
2871 const MachineRegisterInfo *MRI = &MF.getRegInfo();
2872 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2873 for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size();
2874 i != e;
2875 ++i, ++realArgIdx) {
2876 CCValAssign &VA = ArgLocs[i];
2877 EVT RegVT = VA.getLocVT();
2878 SDValue Arg = OutVals[realArgIdx];
2879 ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags;
2880 if (VA.getLocInfo() == CCValAssign::Indirect)
2881 return false;
2882 if (VA.needsCustom() && (RegVT == MVT::f64 || RegVT == MVT::v2f64)) {
2883 // f64 and vector types are split into multiple registers or
2884 // register/stack-slot combinations. The types will not match
2885 // the registers; give up on memory f64 refs until we figure
2886 // out what to do about this.
2887 if (!VA.isRegLoc())
2888 return false;
2889 if (!ArgLocs[++i].isRegLoc())
2890 return false;
2891 if (RegVT == MVT::v2f64) {
2892 if (!ArgLocs[++i].isRegLoc())
2893 return false;
2894 if (!ArgLocs[++i].isRegLoc())
2895 return false;
2896 }
2897 } else if (!VA.isRegLoc()) {
2898 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
2899 MFI, MRI, TII))
2900 return false;
2901 }
2902 }
2903 }
2904
2905 const MachineRegisterInfo &MRI = MF.getRegInfo();
2906 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
2907 return false;
2908 }
2909
2910 return true;
2911}
2912
2913bool
2914ARMTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
2915 MachineFunction &MF, bool isVarArg,
2916 const SmallVectorImpl<ISD::OutputArg> &Outs,
2917 LLVMContext &Context) const {
2918 SmallVector<CCValAssign, 16> RVLocs;
2919 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2920 return CCInfo.CheckReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2921}
2922
2923static SDValue LowerInterruptReturn(SmallVectorImpl<SDValue> &RetOps,
2924 const SDLoc &DL, SelectionDAG &DAG) {
2925 const MachineFunction &MF = DAG.getMachineFunction();
2926 const Function &F = MF.getFunction();
2927
2928 StringRef IntKind = F.getFnAttribute("interrupt").getValueAsString();
2929
2930 // See ARM ARM v7 B1.8.3. On exception entry LR is set to a possibly offset
2931 // version of the "preferred return address". These offsets affect the return
2932 // instruction if this is a return from PL1 without hypervisor extensions.
2933 // IRQ/FIQ: +4 "subs pc, lr, #4"
2934 // SWI: 0 "subs pc, lr, #0"
2935 // ABORT: +4 "subs pc, lr, #4"
2936 // UNDEF: +4/+2 "subs pc, lr, #0"
2937 // UNDEF varies depending on where the exception came from ARM or Thumb
2938 // mode. Alongside GCC, we throw our hands up in disgust and pretend it's 0.
2939
2940 int64_t LROffset;
2941 if (IntKind == "" || IntKind == "IRQ" || IntKind == "FIQ" ||
2942 IntKind == "ABORT")
2943 LROffset = 4;
2944 else if (IntKind == "SWI" || IntKind == "UNDEF")
2945 LROffset = 0;
2946 else
2947 report_fatal_error("Unsupported interrupt attribute. If present, value "
2948 "must be one of: IRQ, FIQ, SWI, ABORT or UNDEF");
2949
2950 RetOps.insert(RetOps.begin() + 1,
2951 DAG.getConstant(LROffset, DL, MVT::i32, false));
2952
2953 return DAG.getNode(ARMISD::INTRET_FLAG, DL, MVT::Other, RetOps);
2954}
2955
2956SDValue
2957ARMTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2958 bool isVarArg,
2959 const SmallVectorImpl<ISD::OutputArg> &Outs,
2960 const SmallVectorImpl<SDValue> &OutVals,
2961 const SDLoc &dl, SelectionDAG &DAG) const {
2962 // CCValAssign - represent the assignment of the return value to a location.
2963 SmallVector<CCValAssign, 16> RVLocs;
2964
2965 // CCState - Info about the registers and stack slots.
2966 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2967 *DAG.getContext());
2968
2969 // Analyze outgoing return values.
2970 CCInfo.AnalyzeReturn(Outs, CCAssignFnForReturn(CallConv, isVarArg));
2971
2972 SDValue Flag;
2973 SmallVector<SDValue, 4> RetOps;
2974 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2975 bool isLittleEndian = Subtarget->isLittle();
2976
2977 MachineFunction &MF = DAG.getMachineFunction();
2978 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
2979 AFI->setReturnRegsCount(RVLocs.size());
2980
2981 // Report error if cmse entry function returns structure through first ptr arg.
2982 if (AFI->isCmseNSEntryFunction() && MF.getFunction().hasStructRetAttr()) {
2983 // Note: using an empty SDLoc(), as the first line of the function is a
2984 // better place to report than the last line.
2985 DiagnosticInfoUnsupported Diag(
2986 DAG.getMachineFunction().getFunction(),
2987 "secure entry function would return value through pointer",
2988 SDLoc().getDebugLoc());
2989 DAG.getContext()->diagnose(Diag);
2990 }
2991
2992 // Copy the result values into the output registers.
2993 for (unsigned i = 0, realRVLocIdx = 0;
2994 i != RVLocs.size();
2995 ++i, ++realRVLocIdx) {
2996 CCValAssign &VA = RVLocs[i];
2997 assert(VA.isRegLoc() && "Can only return in registers!")(static_cast <bool> (VA.isRegLoc() && "Can only return in registers!"
) ? void (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 2997, __extension__ __PRETTY_FUNCTION__))
;
2998
2999 SDValue Arg = OutVals[realRVLocIdx];
3000 bool ReturnF16 = false;
3001
3002 if (Subtarget->hasFullFP16() && Subtarget->isTargetHardFloat()) {
3003 // Half-precision return values can be returned like this:
3004 //
3005 // t11 f16 = fadd ...
3006 // t12: i16 = bitcast t11
3007 // t13: i32 = zero_extend t12
3008 // t14: f32 = bitcast t13 <~~~~~~~ Arg
3009 //
3010 // to avoid code generation for bitcasts, we simply set Arg to the node
3011 // that produces the f16 value, t11 in this case.
3012 //
3013 if (Arg.getValueType() == MVT::f32 && Arg.getOpcode() == ISD::BITCAST) {
3014 SDValue ZE = Arg.getOperand(0);
3015 if (ZE.getOpcode() == ISD::ZERO_EXTEND && ZE.getValueType() == MVT::i32) {
3016 SDValue BC = ZE.getOperand(0);
3017 if (BC.getOpcode() == ISD::BITCAST && BC.getValueType() == MVT::i16) {
3018 Arg = BC.getOperand(0);
3019 ReturnF16 = true;
3020 }
3021 }
3022 }
3023 }
3024
3025 switch (VA.getLocInfo()) {
3026 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3026)
;
3027 case CCValAssign::Full: break;
3028 case CCValAssign::BCvt:
3029 if (!ReturnF16)
3030 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3031 break;
3032 }
3033
3034 // Mask f16 arguments if this is a CMSE nonsecure entry.
3035 auto RetVT = Outs[realRVLocIdx].ArgVT;
3036 if (AFI->isCmseNSEntryFunction() && (RetVT == MVT::f16)) {
3037 if (VA.needsCustom() && VA.getValVT() == MVT::f16) {
3038 Arg = MoveFromHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), Arg);
3039 } else {
3040 auto LocBits = VA.getLocVT().getSizeInBits();
3041 auto MaskValue = APInt::getLowBitsSet(LocBits, RetVT.getSizeInBits());
3042 SDValue Mask =
3043 DAG.getConstant(MaskValue, dl, MVT::getIntegerVT(LocBits));
3044 Arg = DAG.getNode(ISD::BITCAST, dl, MVT::getIntegerVT(LocBits), Arg);
3045 Arg = DAG.getNode(ISD::AND, dl, MVT::getIntegerVT(LocBits), Arg, Mask);
3046 Arg = DAG.getNode(ISD::BITCAST, dl, VA.getLocVT(), Arg);
3047 }
3048 }
3049
3050 if (VA.needsCustom() &&
3051 (VA.getLocVT() == MVT::v2f64 || VA.getLocVT() == MVT::f64)) {
3052 if (VA.getLocVT() == MVT::v2f64) {
3053 // Extract the first half and return it in two registers.
3054 SDValue Half = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3055 DAG.getConstant(0, dl, MVT::i32));
3056 SDValue HalfGPRs = DAG.getNode(ARMISD::VMOVRRD, dl,
3057 DAG.getVTList(MVT::i32, MVT::i32), Half);
3058
3059 Chain =
3060 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3061 HalfGPRs.getValue(isLittleEndian ? 0 : 1), Flag);
3062 Flag = Chain.getValue(1);
3063 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3064 VA = RVLocs[++i]; // skip ahead to next loc
3065 Chain =
3066 DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3067 HalfGPRs.getValue(isLittleEndian ? 1 : 0), Flag);
3068 Flag = Chain.getValue(1);
3069 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3070 VA = RVLocs[++i]; // skip ahead to next loc
3071
3072 // Extract the 2nd half and fall through to handle it as an f64 value.
3073 Arg = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Arg,
3074 DAG.getConstant(1, dl, MVT::i32));
3075 }
3076 // Legalize ret f64 -> ret 2 x i32. We always have fmrrd if f64 is
3077 // available.
3078 SDValue fmrrd = DAG.getNode(ARMISD::VMOVRRD, dl,
3079 DAG.getVTList(MVT::i32, MVT::i32), Arg);
3080 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3081 fmrrd.getValue(isLittleEndian ? 0 : 1), Flag);
3082 Flag = Chain.getValue(1);
3083 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
3084 VA = RVLocs[++i]; // skip ahead to next loc
3085 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
3086 fmrrd.getValue(isLittleEndian ? 1 : 0), Flag);
3087 } else
3088 Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
3089
3090 // Guarantee that all emitted copies are
3091 // stuck together, avoiding something bad.
3092 Flag = Chain.getValue(1);
3093 RetOps.push_back(DAG.getRegister(
3094 VA.getLocReg(), ReturnF16 ? Arg.getValueType() : VA.getLocVT()));
3095 }
3096 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
3097 const MCPhysReg *I =
3098 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
3099 if (I) {
3100 for (; *I; ++I) {
3101 if (ARM::GPRRegClass.contains(*I))
3102 RetOps.push_back(DAG.getRegister(*I, MVT::i32));
3103 else if (ARM::DPRRegClass.contains(*I))
3104 RetOps.push_back(DAG.getRegister(*I, MVT::getFloatingPointVT(64)));
3105 else
3106 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3106)
;
3107 }
3108 }
3109
3110 // Update chain and glue.
3111 RetOps[0] = Chain;
3112 if (Flag.getNode())
3113 RetOps.push_back(Flag);
3114
3115 // CPUs which aren't M-class use a special sequence to return from
3116 // exceptions (roughly, any instruction setting pc and cpsr simultaneously,
3117 // though we use "subs pc, lr, #N").
3118 //
3119 // M-class CPUs actually use a normal return sequence with a special
3120 // (hardware-provided) value in LR, so the normal code path works.
3121 if (DAG.getMachineFunction().getFunction().hasFnAttribute("interrupt") &&
3122 !Subtarget->isMClass()) {
3123 if (Subtarget->isThumb1Only())
3124 report_fatal_error("interrupt attribute is not supported in Thumb1");
3125 return LowerInterruptReturn(RetOps, dl, DAG);
3126 }
3127
3128 ARMISD::NodeType RetNode = AFI->isCmseNSEntryFunction() ? ARMISD::SERET_FLAG :
3129 ARMISD::RET_FLAG;
3130 return DAG.getNode(RetNode, dl, MVT::Other, RetOps);
3131}
3132
3133bool ARMTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
3134 if (N->getNumValues() != 1)
3135 return false;
3136 if (!N->hasNUsesOfValue(1, 0))
3137 return false;
3138
3139 SDValue TCChain = Chain;
3140 SDNode *Copy = *N->use_begin();
3141 if (Copy->getOpcode() == ISD::CopyToReg) {
3142 // If the copy has a glue operand, we conservatively assume it isn't safe to
3143 // perform a tail call.
3144 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3145 return false;
3146 TCChain = Copy->getOperand(0);
3147 } else if (Copy->getOpcode() == ARMISD::VMOVRRD) {
3148 SDNode *VMov = Copy;
3149 // f64 returned in a pair of GPRs.
3150 SmallPtrSet<SDNode*, 2> Copies;
3151 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
3152 UI != UE; ++UI) {
3153 if (UI->getOpcode() != ISD::CopyToReg)
3154 return false;
3155 Copies.insert(*UI);
3156 }
3157 if (Copies.size() > 2)
3158 return false;
3159
3160 for (SDNode::use_iterator UI = VMov->use_begin(), UE = VMov->use_end();
3161 UI != UE; ++UI) {
3162 SDValue UseChain = UI->getOperand(0);
3163 if (Copies.count(UseChain.getNode()))
3164 // Second CopyToReg
3165 Copy = *UI;
3166 else {
3167 // We are at the top of this chain.
3168 // If the copy has a glue operand, we conservatively assume it
3169 // isn't safe to perform a tail call.
3170 if (UI->getOperand(UI->getNumOperands()-1).getValueType() == MVT::Glue)
3171 return false;
3172 // First CopyToReg
3173 TCChain = UseChain;
3174 }
3175 }
3176 } else if (Copy->getOpcode() == ISD::BITCAST) {
3177 // f32 returned in a single GPR.
3178 if (!Copy->hasOneUse())
3179 return false;
3180 Copy = *Copy->use_begin();
3181 if (Copy->getOpcode() != ISD::CopyToReg || !Copy->hasNUsesOfValue(1, 0))
3182 return false;
3183 // If the copy has a glue operand, we conservatively assume it isn't safe to
3184 // perform a tail call.
3185 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
3186 return false;
3187 TCChain = Copy->getOperand(0);
3188 } else {
3189 return false;
3190 }
3191
3192 bool HasRet = false;
3193 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
3194 UI != UE; ++UI) {
3195 if (UI->getOpcode() != ARMISD::RET_FLAG &&
3196 UI->getOpcode() != ARMISD::INTRET_FLAG)
3197 return false;
3198 HasRet = true;
3199 }
3200
3201 if (!HasRet)
3202 return false;
3203
3204 Chain = TCChain;
3205 return true;
3206}
3207
3208bool ARMTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3209 if (!Subtarget->supportsTailCall())
3210 return false;
3211
3212 if (!CI->isTailCall())
3213 return false;
3214
3215 return true;
3216}
3217
3218// Trying to write a 64 bit value so need to split into two 32 bit values first,
3219// and pass the lower and high parts through.
3220static SDValue LowerWRITE_REGISTER(SDValue Op, SelectionDAG &DAG) {
3221 SDLoc DL(Op);
3222 SDValue WriteValue = Op->getOperand(2);
3223
3224 // This function is only supposed to be called for i64 type argument.
3225 assert(WriteValue.getValueType() == MVT::i64(static_cast <bool> (WriteValue.getValueType() == MVT::
i64 && "LowerWRITE_REGISTER called for non-i64 type argument."
) ? void (0) : __assert_fail ("WriteValue.getValueType() == MVT::i64 && \"LowerWRITE_REGISTER called for non-i64 type argument.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3226, __extension__ __PRETTY_FUNCTION__))
3226 && "LowerWRITE_REGISTER called for non-i64 type argument.")(static_cast <bool> (WriteValue.getValueType() == MVT::
i64 && "LowerWRITE_REGISTER called for non-i64 type argument."
) ? void (0) : __assert_fail ("WriteValue.getValueType() == MVT::i64 && \"LowerWRITE_REGISTER called for non-i64 type argument.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3226, __extension__ __PRETTY_FUNCTION__))
;
3227
3228 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3229 DAG.getConstant(0, DL, MVT::i32));
3230 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, WriteValue,
3231 DAG.getConstant(1, DL, MVT::i32));
3232 SDValue Ops[] = { Op->getOperand(0), Op->getOperand(1), Lo, Hi };
3233 return DAG.getNode(ISD::WRITE_REGISTER, DL, MVT::Other, Ops);
3234}
3235
3236// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
3237// their target counterpart wrapped in the ARMISD::Wrapper node. Suppose N is
3238// one of the above mentioned nodes. It has to be wrapped because otherwise
3239// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
3240// be used to form addressing mode. These wrapped nodes will be selected
3241// into MOVi.
3242SDValue ARMTargetLowering::LowerConstantPool(SDValue Op,
3243 SelectionDAG &DAG) const {
3244 EVT PtrVT = Op.getValueType();
3245 // FIXME there is no actual debug info here
3246 SDLoc dl(Op);
3247 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
3248 SDValue Res;
3249
3250 // When generating execute-only code Constant Pools must be promoted to the
3251 // global data section. It's a bit ugly that we can't share them across basic
3252 // blocks, but this way we guarantee that execute-only behaves correct with
3253 // position-independent addressing modes.
3254 if (Subtarget->genExecuteOnly()) {
3255 auto AFI = DAG.getMachineFunction().getInfo<ARMFunctionInfo>();
3256 auto T = const_cast<Type*>(CP->getType());
3257 auto C = const_cast<Constant*>(CP->getConstVal());
3258 auto M = const_cast<Module*>(DAG.getMachineFunction().
3259 getFunction().getParent());
3260 auto GV = new GlobalVariable(
3261 *M, T, /*isConstant=*/true, GlobalVariable::InternalLinkage, C,
3262 Twine(DAG.getDataLayout().getPrivateGlobalPrefix()) + "CP" +
3263 Twine(DAG.getMachineFunction().getFunctionNumber()) + "_" +
3264 Twine(AFI->createPICLabelUId())
3265 );
3266 SDValue GA = DAG.getTargetGlobalAddress(dyn_cast<GlobalValue>(GV),
3267 dl, PtrVT);
3268 return LowerGlobalAddress(GA, DAG);
3269 }
3270
3271 if (CP->isMachineConstantPoolEntry())
3272 Res =
3273 DAG.getTargetConstantPool(CP->getMachineCPVal(), PtrVT, CP->getAlign());
3274 else
3275 Res = DAG.getTargetConstantPool(CP->getConstVal(), PtrVT, CP->getAlign());
3276 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Res);
3277}
3278
3279unsigned ARMTargetLowering::getJumpTableEncoding() const {
3280 return MachineJumpTableInfo::EK_Inline;
3281}
3282
3283SDValue ARMTargetLowering::LowerBlockAddress(SDValue Op,
3284 SelectionDAG &DAG) const {
3285 MachineFunction &MF = DAG.getMachineFunction();
3286 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3287 unsigned ARMPCLabelIndex = 0;
3288 SDLoc DL(Op);
3289 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3290 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
3291 SDValue CPAddr;
3292 bool IsPositionIndependent = isPositionIndependent() || Subtarget->isROPI();
3293 if (!IsPositionIndependent) {
3294 CPAddr = DAG.getTargetConstantPool(BA, PtrVT, Align(4));
3295 } else {
3296 unsigned PCAdj = Subtarget->isThumb() ? 4 : 8;
3297 ARMPCLabelIndex = AFI->createPICLabelUId();
3298 ARMConstantPoolValue *CPV =
3299 ARMConstantPoolConstant::Create(BA, ARMPCLabelIndex,
3300 ARMCP::CPBlockAddress, PCAdj);
3301 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3302 }
3303 CPAddr = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, CPAddr);
3304 SDValue Result = DAG.getLoad(
3305 PtrVT, DL, DAG.getEntryNode(), CPAddr,
3306 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3307 if (!IsPositionIndependent)
3308 return Result;
3309 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, DL, MVT::i32);
3310 return DAG.getNode(ARMISD::PIC_ADD, DL, PtrVT, Result, PICLabel);
3311}
3312
3313/// Convert a TLS address reference into the correct sequence of loads
3314/// and calls to compute the variable's address for Darwin, and return an
3315/// SDValue containing the final node.
3316
3317/// Darwin only has one TLS scheme which must be capable of dealing with the
3318/// fully general situation, in the worst case. This means:
3319/// + "extern __thread" declaration.
3320/// + Defined in a possibly unknown dynamic library.
3321///
3322/// The general system is that each __thread variable has a [3 x i32] descriptor
3323/// which contains information used by the runtime to calculate the address. The
3324/// only part of this the compiler needs to know about is the first word, which
3325/// contains a function pointer that must be called with the address of the
3326/// entire descriptor in "r0".
3327///
3328/// Since this descriptor may be in a different unit, in general access must
3329/// proceed along the usual ARM rules. A common sequence to produce is:
3330///
3331/// movw rT1, :lower16:_var$non_lazy_ptr
3332/// movt rT1, :upper16:_var$non_lazy_ptr
3333/// ldr r0, [rT1]
3334/// ldr rT2, [r0]
3335/// blx rT2
3336/// [...address now in r0...]
3337SDValue
3338ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op,
3339 SelectionDAG &DAG) const {
3340 assert(Subtarget->isTargetDarwin() &&(static_cast <bool> (Subtarget->isTargetDarwin() &&
"This function expects a Darwin target") ? void (0) : __assert_fail
("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3341, __extension__ __PRETTY_FUNCTION__))
3341 "This function expects a Darwin target")(static_cast <bool> (Subtarget->isTargetDarwin() &&
"This function expects a Darwin target") ? void (0) : __assert_fail
("Subtarget->isTargetDarwin() && \"This function expects a Darwin target\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3341, __extension__ __PRETTY_FUNCTION__))
;
3342 SDLoc DL(Op);
3343
3344 // First step is to get the address of the actua global symbol. This is where
3345 // the TLS descriptor lives.
3346 SDValue DescAddr = LowerGlobalAddressDarwin(Op, DAG);
3347
3348 // The first entry in the descriptor is a function pointer that we must call
3349 // to obtain the address of the variable.
3350 SDValue Chain = DAG.getEntryNode();
3351 SDValue FuncTLVGet = DAG.getLoad(
3352 MVT::i32, DL, Chain, DescAddr,
3353 MachinePointerInfo::getGOT(DAG.getMachineFunction()), Align(4),
3354 MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable |
3355 MachineMemOperand::MOInvariant);
3356 Chain = FuncTLVGet.getValue(1);
3357
3358 MachineFunction &F = DAG.getMachineFunction();
3359 MachineFrameInfo &MFI = F.getFrameInfo();
3360 MFI.setAdjustsStack(true);
3361
3362 // TLS calls preserve all registers except those that absolutely must be
3363 // trashed: R0 (it takes an argument), LR (it's a call) and CPSR (let's not be
3364 // silly).
3365 auto TRI =
3366 getTargetMachine().getSubtargetImpl(F.getFunction())->getRegisterInfo();
3367 auto ARI = static_cast<const ARMRegisterInfo *>(TRI);
3368 const uint32_t *Mask = ARI->getTLSCallPreservedMask(DAG.getMachineFunction());
3369
3370 // Finally, we can make the call. This is just a degenerate version of a
3371 // normal AArch64 call node: r0 takes the address of the descriptor, and
3372 // returns the address of the variable in this thread.
3373 Chain = DAG.getCopyToReg(Chain, DL, ARM::R0, DescAddr, SDValue());
3374 Chain =
3375 DAG.getNode(ARMISD::CALL, DL, DAG.getVTList(MVT::Other, MVT::Glue),
3376 Chain, FuncTLVGet, DAG.getRegister(ARM::R0, MVT::i32),
3377 DAG.getRegisterMask(Mask), Chain.getValue(1));
3378 return DAG.getCopyFromReg(Chain, DL, ARM::R0, MVT::i32, Chain.getValue(1));
3379}
3380
3381SDValue
3382ARMTargetLowering::LowerGlobalTLSAddressWindows(SDValue Op,
3383 SelectionDAG &DAG) const {
3384 assert(Subtarget->isTargetWindows() && "Windows specific TLS lowering")(static_cast <bool> (Subtarget->isTargetWindows() &&
"Windows specific TLS lowering") ? void (0) : __assert_fail (
"Subtarget->isTargetWindows() && \"Windows specific TLS lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3384, __extension__ __PRETTY_FUNCTION__))
;
3385
3386 SDValue Chain = DAG.getEntryNode();
3387 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3388 SDLoc DL(Op);
3389
3390 // Load the current TEB (thread environment block)
3391 SDValue Ops[] = {Chain,
3392 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
3393 DAG.getTargetConstant(15, DL, MVT::i32),
3394 DAG.getTargetConstant(0, DL, MVT::i32),
3395 DAG.getTargetConstant(13, DL, MVT::i32),
3396 DAG.getTargetConstant(0, DL, MVT::i32),
3397 DAG.getTargetConstant(2, DL, MVT::i32)};
3398 SDValue CurrentTEB = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
3399 DAG.getVTList(MVT::i32, MVT::Other), Ops);
3400
3401 SDValue TEB = CurrentTEB.getValue(0);
3402 Chain = CurrentTEB.getValue(1);
3403
3404 // Load the ThreadLocalStoragePointer from the TEB
3405 // A pointer to the TLS array is located at offset 0x2c from the TEB.
3406 SDValue TLSArray =
3407 DAG.getNode(ISD::ADD, DL, PtrVT, TEB, DAG.getIntPtrConstant(0x2c, DL));
3408 TLSArray = DAG.getLoad(PtrVT, DL, Chain, TLSArray, MachinePointerInfo());
3409
3410 // The pointer to the thread's TLS data area is at the TLS Index scaled by 4
3411 // offset into the TLSArray.
3412
3413 // Load the TLS index from the C runtime
3414 SDValue TLSIndex =
3415 DAG.getTargetExternalSymbol("_tls_index", PtrVT, ARMII::MO_NO_FLAG);
3416 TLSIndex = DAG.getNode(ARMISD::Wrapper, DL, PtrVT, TLSIndex);
3417 TLSIndex = DAG.getLoad(PtrVT, DL, Chain, TLSIndex, MachinePointerInfo());
3418
3419 SDValue Slot = DAG.getNode(ISD::SHL, DL, PtrVT, TLSIndex,
3420 DAG.getConstant(2, DL, MVT::i32));
3421 SDValue TLS = DAG.getLoad(PtrVT, DL, Chain,
3422 DAG.getNode(ISD::ADD, DL, PtrVT, TLSArray, Slot),
3423 MachinePointerInfo());
3424
3425 // Get the offset of the start of the .tls section (section base)
3426 const auto *GA = cast<GlobalAddressSDNode>(Op);
3427 auto *CPV = ARMConstantPoolConstant::Create(GA->getGlobal(), ARMCP::SECREL);
3428 SDValue Offset = DAG.getLoad(
3429 PtrVT, DL, Chain,
3430 DAG.getNode(ARMISD::Wrapper, DL, MVT::i32,
3431 DAG.getTargetConstantPool(CPV, PtrVT, Align(4))),
3432 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3433
3434 return DAG.getNode(ISD::ADD, DL, PtrVT, TLS, Offset);
3435}
3436
3437// Lower ISD::GlobalTLSAddress using the "general dynamic" model
3438SDValue
3439ARMTargetLowering::LowerToTLSGeneralDynamicModel(GlobalAddressSDNode *GA,
3440 SelectionDAG &DAG) const {
3441 SDLoc dl(GA);
3442 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3443 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3444 MachineFunction &MF = DAG.getMachineFunction();
3445 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3446 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3447 ARMConstantPoolValue *CPV =
3448 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3449 ARMCP::CPValue, PCAdj, ARMCP::TLSGD, true);
3450 SDValue Argument = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3451 Argument = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Argument);
3452 Argument = DAG.getLoad(
3453 PtrVT, dl, DAG.getEntryNode(), Argument,
3454 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3455 SDValue Chain = Argument.getValue(1);
3456
3457 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3458 Argument = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Argument, PICLabel);
3459
3460 // call __tls_get_addr.
3461 ArgListTy Args;
3462 ArgListEntry Entry;
3463 Entry.Node = Argument;
3464 Entry.Ty = (Type *) Type::getInt32Ty(*DAG.getContext());
3465 Args.push_back(Entry);
3466
3467 // FIXME: is there useful debug info available here?
3468 TargetLowering::CallLoweringInfo CLI(DAG);
3469 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3470 CallingConv::C, Type::getInt32Ty(*DAG.getContext()),
3471 DAG.getExternalSymbol("__tls_get_addr", PtrVT), std::move(Args));
3472
3473 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3474 return CallResult.first;
3475}
3476
3477// Lower ISD::GlobalTLSAddress using the "initial exec" or
3478// "local exec" model.
3479SDValue
3480ARMTargetLowering::LowerToTLSExecModels(GlobalAddressSDNode *GA,
3481 SelectionDAG &DAG,
3482 TLSModel::Model model) const {
3483 const GlobalValue *GV = GA->getGlobal();
3484 SDLoc dl(GA);
3485 SDValue Offset;
3486 SDValue Chain = DAG.getEntryNode();
3487 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3488 // Get the Thread Pointer
3489 SDValue ThreadPointer = DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3490
3491 if (model == TLSModel::InitialExec) {
3492 MachineFunction &MF = DAG.getMachineFunction();
3493 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3494 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3495 // Initial exec model.
3496 unsigned char PCAdj = Subtarget->isThumb() ? 4 : 8;
3497 ARMConstantPoolValue *CPV =
3498 ARMConstantPoolConstant::Create(GA->getGlobal(), ARMPCLabelIndex,
3499 ARMCP::CPValue, PCAdj, ARMCP::GOTTPOFF,
3500 true);
3501 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3502 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3503 Offset = DAG.getLoad(
3504 PtrVT, dl, Chain, Offset,
3505 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3506 Chain = Offset.getValue(1);
3507
3508 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3509 Offset = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Offset, PICLabel);
3510
3511 Offset = DAG.getLoad(
3512 PtrVT, dl, Chain, Offset,
3513 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3514 } else {
3515 // local exec model
3516 assert(model == TLSModel::LocalExec)(static_cast <bool> (model == TLSModel::LocalExec) ? void
(0) : __assert_fail ("model == TLSModel::LocalExec", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3516, __extension__ __PRETTY_FUNCTION__))
;
3517 ARMConstantPoolValue *CPV =
3518 ARMConstantPoolConstant::Create(GV, ARMCP::TPOFF);
3519 Offset = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3520 Offset = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, Offset);
3521 Offset = DAG.getLoad(
3522 PtrVT, dl, Chain, Offset,
3523 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3524 }
3525
3526 // The address of the thread local variable is the add of the thread
3527 // pointer with the offset of the variable.
3528 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
3529}
3530
3531SDValue
3532ARMTargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
3533 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
3534 if (DAG.getTarget().useEmulatedTLS())
3535 return LowerToTLSEmulatedModel(GA, DAG);
3536
3537 if (Subtarget->isTargetDarwin())
3538 return LowerGlobalTLSAddressDarwin(Op, DAG);
3539
3540 if (Subtarget->isTargetWindows())
3541 return LowerGlobalTLSAddressWindows(Op, DAG);
3542
3543 // TODO: implement the "local dynamic" model
3544 assert(Subtarget->isTargetELF() && "Only ELF implemented here")(static_cast <bool> (Subtarget->isTargetELF() &&
"Only ELF implemented here") ? void (0) : __assert_fail ("Subtarget->isTargetELF() && \"Only ELF implemented here\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3544, __extension__ __PRETTY_FUNCTION__))
;
3545 TLSModel::Model model = getTargetMachine().getTLSModel(GA->getGlobal());
3546
3547 switch (model) {
3548 case TLSModel::GeneralDynamic:
3549 case TLSModel::LocalDynamic:
3550 return LowerToTLSGeneralDynamicModel(GA, DAG);
3551 case TLSModel::InitialExec:
3552 case TLSModel::LocalExec:
3553 return LowerToTLSExecModels(GA, DAG, model);
3554 }
3555 llvm_unreachable("bogus TLS model")::llvm::llvm_unreachable_internal("bogus TLS model", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3555)
;
3556}
3557
3558/// Return true if all users of V are within function F, looking through
3559/// ConstantExprs.
3560static bool allUsersAreInFunction(const Value *V, const Function *F) {
3561 SmallVector<const User*,4> Worklist(V->users());
3562 while (!Worklist.empty()) {
3563 auto *U = Worklist.pop_back_val();
3564 if (isa<ConstantExpr>(U)) {
3565 append_range(Worklist, U->users());
3566 continue;
3567 }
3568
3569 auto *I = dyn_cast<Instruction>(U);
3570 if (!I || I->getParent()->getParent() != F)
3571 return false;
3572 }
3573 return true;
3574}
3575
3576static SDValue promoteToConstantPool(const ARMTargetLowering *TLI,
3577 const GlobalValue *GV, SelectionDAG &DAG,
3578 EVT PtrVT, const SDLoc &dl) {
3579 // If we're creating a pool entry for a constant global with unnamed address,
3580 // and the global is small enough, we can emit it inline into the constant pool
3581 // to save ourselves an indirection.
3582 //
3583 // This is a win if the constant is only used in one function (so it doesn't
3584 // need to be duplicated) or duplicating the constant wouldn't increase code
3585 // size (implying the constant is no larger than 4 bytes).
3586 const Function &F = DAG.getMachineFunction().getFunction();
3587
3588 // We rely on this decision to inline being idemopotent and unrelated to the
3589 // use-site. We know that if we inline a variable at one use site, we'll
3590 // inline it elsewhere too (and reuse the constant pool entry). Fast-isel
3591 // doesn't know about this optimization, so bail out if it's enabled else
3592 // we could decide to inline here (and thus never emit the GV) but require
3593 // the GV from fast-isel generated code.
3594 if (!EnableConstpoolPromotion ||
3595 DAG.getMachineFunction().getTarget().Options.EnableFastISel)
3596 return SDValue();
3597
3598 auto *GVar = dyn_cast<GlobalVariable>(GV);
3599 if (!GVar || !GVar->hasInitializer() ||
3600 !GVar->isConstant() || !GVar->hasGlobalUnnamedAddr() ||
3601 !GVar->hasLocalLinkage())
3602 return SDValue();
3603
3604 // If we inline a value that contains relocations, we move the relocations
3605 // from .data to .text. This is not allowed in position-independent code.
3606 auto *Init = GVar->getInitializer();
3607 if ((TLI->isPositionIndependent() || TLI->getSubtarget()->isROPI()) &&
3608 Init->needsDynamicRelocation())
3609 return SDValue();
3610
3611 // The constant islands pass can only really deal with alignment requests
3612 // <= 4 bytes and cannot pad constants itself. Therefore we cannot promote
3613 // any type wanting greater alignment requirements than 4 bytes. We also
3614 // can only promote constants that are multiples of 4 bytes in size or
3615 // are paddable to a multiple of 4. Currently we only try and pad constants
3616 // that are strings for simplicity.
3617 auto *CDAInit = dyn_cast<ConstantDataArray>(Init);
3618 unsigned Size = DAG.getDataLayout().getTypeAllocSize(Init->getType());
3619 Align PrefAlign = DAG.getDataLayout().getPreferredAlign(GVar);
3620 unsigned RequiredPadding = 4 - (Size % 4);
3621 bool PaddingPossible =
3622 RequiredPadding == 4 || (CDAInit && CDAInit->isString());
3623 if (!PaddingPossible || PrefAlign > 4 || Size > ConstpoolPromotionMaxSize ||
3624 Size == 0)
3625 return SDValue();
3626
3627 unsigned PaddedSize = Size + ((RequiredPadding == 4) ? 0 : RequiredPadding);
3628 MachineFunction &MF = DAG.getMachineFunction();
3629 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3630
3631 // We can't bloat the constant pool too much, else the ConstantIslands pass
3632 // may fail to converge. If we haven't promoted this global yet (it may have
3633 // multiple uses), and promoting it would increase the constant pool size (Sz
3634 // > 4), ensure we have space to do so up to MaxTotal.
3635 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar) && Size > 4)
3636 if (AFI->getPromotedConstpoolIncrease() + PaddedSize - 4 >=
3637 ConstpoolPromotionMaxTotal)
3638 return SDValue();
3639
3640 // This is only valid if all users are in a single function; we can't clone
3641 // the constant in general. The LLVM IR unnamed_addr allows merging
3642 // constants, but not cloning them.
3643 //
3644 // We could potentially allow cloning if we could prove all uses of the
3645 // constant in the current function don't care about the address, like
3646 // printf format strings. But that isn't implemented for now.
3647 if (!allUsersAreInFunction(GVar, &F))
3648 return SDValue();
3649
3650 // We're going to inline this global. Pad it out if needed.
3651 if (RequiredPadding != 4) {
3652 StringRef S = CDAInit->getAsString();
3653
3654 SmallVector<uint8_t,16> V(S.size());
3655 std::copy(S.bytes_begin(), S.bytes_end(), V.begin());
3656 while (RequiredPadding--)
3657 V.push_back(0);
3658 Init = ConstantDataArray::get(*DAG.getContext(), V);
3659 }
3660
3661 auto CPVal = ARMConstantPoolConstant::Create(GVar, Init);
3662 SDValue CPAddr = DAG.getTargetConstantPool(CPVal, PtrVT, Align(4));
3663 if (!AFI->getGlobalsPromotedToConstantPool().count(GVar)) {
3664 AFI->markGlobalAsPromotedToConstantPool(GVar);
3665 AFI->setPromotedConstpoolIncrease(AFI->getPromotedConstpoolIncrease() +
3666 PaddedSize - 4);
3667 }
3668 ++NumConstpoolPromoted;
3669 return DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3670}
3671
3672bool ARMTargetLowering::isReadOnly(const GlobalValue *GV) const {
3673 if (const GlobalAlias *GA = dyn_cast<GlobalAlias>(GV))
3674 if (!(GV = GA->getBaseObject()))
3675 return false;
3676 if (const auto *V = dyn_cast<GlobalVariable>(GV))
3677 return V->isConstant();
3678 return isa<Function>(GV);
3679}
3680
3681SDValue ARMTargetLowering::LowerGlobalAddress(SDValue Op,
3682 SelectionDAG &DAG) const {
3683 switch (Subtarget->getTargetTriple().getObjectFormat()) {
3684 default: llvm_unreachable("unknown object format")::llvm::llvm_unreachable_internal("unknown object format", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3684)
;
3685 case Triple::COFF:
3686 return LowerGlobalAddressWindows(Op, DAG);
3687 case Triple::ELF:
3688 return LowerGlobalAddressELF(Op, DAG);
3689 case Triple::MachO:
3690 return LowerGlobalAddressDarwin(Op, DAG);
3691 }
3692}
3693
3694SDValue ARMTargetLowering::LowerGlobalAddressELF(SDValue Op,
3695 SelectionDAG &DAG) const {
3696 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3697 SDLoc dl(Op);
3698 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3699 const TargetMachine &TM = getTargetMachine();
3700 bool IsRO = isReadOnly(GV);
3701
3702 // promoteToConstantPool only if not generating XO text section
3703 if (TM.shouldAssumeDSOLocal(*GV->getParent(), GV) && !Subtarget->genExecuteOnly())
3704 if (SDValue V = promoteToConstantPool(this, GV, DAG, PtrVT, dl))
3705 return V;
3706
3707 if (isPositionIndependent()) {
3708 bool UseGOT_PREL = !TM.shouldAssumeDSOLocal(*GV->getParent(), GV);
3709 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0,
3710 UseGOT_PREL ? ARMII::MO_GOT : 0);
3711 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3712 if (UseGOT_PREL)
3713 Result =
3714 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3715 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3716 return Result;
3717 } else if (Subtarget->isROPI() && IsRO) {
3718 // PC-relative.
3719 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT);
3720 SDValue Result = DAG.getNode(ARMISD::WrapperPIC, dl, PtrVT, G);
3721 return Result;
3722 } else if (Subtarget->isRWPI() && !IsRO) {
3723 // SB-relative.
3724 SDValue RelAddr;
3725 if (Subtarget->useMovt()) {
3726 ++NumMovwMovt;
3727 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_SBREL);
3728 RelAddr = DAG.getNode(ARMISD::Wrapper, dl, PtrVT, G);
3729 } else { // use literal pool for address constant
3730 ARMConstantPoolValue *CPV =
3731 ARMConstantPoolConstant::Create(GV, ARMCP::SBREL);
3732 SDValue CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3733 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3734 RelAddr = DAG.getLoad(
3735 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3736 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3737 }
3738 SDValue SB = DAG.getCopyFromReg(DAG.getEntryNode(), dl, ARM::R9, PtrVT);
3739 SDValue Result = DAG.getNode(ISD::ADD, dl, PtrVT, SB, RelAddr);
3740 return Result;
3741 }
3742
3743 // If we have T2 ops, we can materialize the address directly via movt/movw
3744 // pair. This is always cheaper.
3745 if (Subtarget->useMovt()) {
3746 ++NumMovwMovt;
3747 // FIXME: Once remat is capable of dealing with instructions with register
3748 // operands, expand this into two nodes.
3749 return DAG.getNode(ARMISD::Wrapper, dl, PtrVT,
3750 DAG.getTargetGlobalAddress(GV, dl, PtrVT));
3751 } else {
3752 SDValue CPAddr = DAG.getTargetConstantPool(GV, PtrVT, Align(4));
3753 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3754 return DAG.getLoad(
3755 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3756 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3757 }
3758}
3759
3760SDValue ARMTargetLowering::LowerGlobalAddressDarwin(SDValue Op,
3761 SelectionDAG &DAG) const {
3762 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported for Darwin"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported for Darwin\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3763, __extension__ __PRETTY_FUNCTION__))
3763 "ROPI/RWPI not currently supported for Darwin")(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported for Darwin"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported for Darwin\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3763, __extension__ __PRETTY_FUNCTION__))
;
3764 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3765 SDLoc dl(Op);
3766 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3767
3768 if (Subtarget->useMovt())
3769 ++NumMovwMovt;
3770
3771 // FIXME: Once remat is capable of dealing with instructions with register
3772 // operands, expand this into multiple nodes
3773 unsigned Wrapper =
3774 isPositionIndependent() ? ARMISD::WrapperPIC : ARMISD::Wrapper;
3775
3776 SDValue G = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, ARMII::MO_NONLAZY);
3777 SDValue Result = DAG.getNode(Wrapper, dl, PtrVT, G);
3778
3779 if (Subtarget->isGVIndirectSymbol(GV))
3780 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
3781 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3782 return Result;
3783}
3784
3785SDValue ARMTargetLowering::LowerGlobalAddressWindows(SDValue Op,
3786 SelectionDAG &DAG) const {
3787 assert(Subtarget->isTargetWindows() && "non-Windows COFF is not supported")(static_cast <bool> (Subtarget->isTargetWindows() &&
"non-Windows COFF is not supported") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"non-Windows COFF is not supported\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3787, __extension__ __PRETTY_FUNCTION__))
;
3788 assert(Subtarget->useMovt() &&(static_cast <bool> (Subtarget->useMovt() &&
"Windows on ARM expects to use movw/movt") ? void (0) : __assert_fail
("Subtarget->useMovt() && \"Windows on ARM expects to use movw/movt\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3789, __extension__ __PRETTY_FUNCTION__))
3789 "Windows on ARM expects to use movw/movt")(static_cast <bool> (Subtarget->useMovt() &&
"Windows on ARM expects to use movw/movt") ? void (0) : __assert_fail
("Subtarget->useMovt() && \"Windows on ARM expects to use movw/movt\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3789, __extension__ __PRETTY_FUNCTION__))
;
3790 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported for Windows"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported for Windows\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3791, __extension__ __PRETTY_FUNCTION__))
3791 "ROPI/RWPI not currently supported for Windows")(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported for Windows"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported for Windows\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3791, __extension__ __PRETTY_FUNCTION__))
;
3792
3793 const TargetMachine &TM = getTargetMachine();
3794 const GlobalValue *GV = cast<GlobalAddressSDNode>(Op)->getGlobal();
3795 ARMII::TOF TargetFlags = ARMII::MO_NO_FLAG;
3796 if (GV->hasDLLImportStorageClass())
3797 TargetFlags = ARMII::MO_DLLIMPORT;
3798 else if (!TM.shouldAssumeDSOLocal(*GV->getParent(), GV))
3799 TargetFlags = ARMII::MO_COFFSTUB;
3800 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3801 SDValue Result;
3802 SDLoc DL(Op);
3803
3804 ++NumMovwMovt;
3805
3806 // FIXME: Once remat is capable of dealing with instructions with register
3807 // operands, expand this into two nodes.
3808 Result = DAG.getNode(ARMISD::Wrapper, DL, PtrVT,
3809 DAG.getTargetGlobalAddress(GV, DL, PtrVT, /*offset=*/0,
3810 TargetFlags));
3811 if (TargetFlags & (ARMII::MO_DLLIMPORT | ARMII::MO_COFFSTUB))
3812 Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result,
3813 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
3814 return Result;
3815}
3816
3817SDValue
3818ARMTargetLowering::LowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const {
3819 SDLoc dl(Op);
3820 SDValue Val = DAG.getConstant(0, dl, MVT::i32);
3821 return DAG.getNode(ARMISD::EH_SJLJ_SETJMP, dl,
3822 DAG.getVTList(MVT::i32, MVT::Other), Op.getOperand(0),
3823 Op.getOperand(1), Val);
3824}
3825
3826SDValue
3827ARMTargetLowering::LowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const {
3828 SDLoc dl(Op);
3829 return DAG.getNode(ARMISD::EH_SJLJ_LONGJMP, dl, MVT::Other, Op.getOperand(0),
3830 Op.getOperand(1), DAG.getConstant(0, dl, MVT::i32));
3831}
3832
3833SDValue ARMTargetLowering::LowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
3834 SelectionDAG &DAG) const {
3835 SDLoc dl(Op);
3836 return DAG.getNode(ARMISD::EH_SJLJ_SETUP_DISPATCH, dl, MVT::Other,
3837 Op.getOperand(0));
3838}
3839
3840SDValue ARMTargetLowering::LowerINTRINSIC_VOID(
3841 SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget) const {
3842 unsigned IntNo =
3843 cast<ConstantSDNode>(
3844 Op.getOperand(Op.getOperand(0).getValueType() == MVT::Other))
3845 ->getZExtValue();
3846 switch (IntNo) {
3847 default:
3848 return SDValue(); // Don't custom lower most intrinsics.
3849 case Intrinsic::arm_gnu_eabi_mcount: {
3850 MachineFunction &MF = DAG.getMachineFunction();
3851 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3852 SDLoc dl(Op);
3853 SDValue Chain = Op.getOperand(0);
3854 // call "\01__gnu_mcount_nc"
3855 const ARMBaseRegisterInfo *ARI = Subtarget->getRegisterInfo();
3856 const uint32_t *Mask =
3857 ARI->getCallPreservedMask(DAG.getMachineFunction(), CallingConv::C);
3858 assert(Mask && "Missing call preserved mask for calling convention")(static_cast <bool> (Mask && "Missing call preserved mask for calling convention"
) ? void (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 3858, __extension__ __PRETTY_FUNCTION__))
;
3859 // Mark LR an implicit live-in.
3860 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
3861 SDValue ReturnAddress =
3862 DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, PtrVT);
3863 constexpr EVT ResultTys[] = {MVT::Other, MVT::Glue};
3864 SDValue Callee =
3865 DAG.getTargetExternalSymbol("\01__gnu_mcount_nc", PtrVT, 0);
3866 SDValue RegisterMask = DAG.getRegisterMask(Mask);
3867 if (Subtarget->isThumb())
3868 return SDValue(
3869 DAG.getMachineNode(
3870 ARM::tBL_PUSHLR, dl, ResultTys,
3871 {ReturnAddress, DAG.getTargetConstant(ARMCC::AL, dl, PtrVT),
3872 DAG.getRegister(0, PtrVT), Callee, RegisterMask, Chain}),
3873 0);
3874 return SDValue(
3875 DAG.getMachineNode(ARM::BL_PUSHLR, dl, ResultTys,
3876 {ReturnAddress, Callee, RegisterMask, Chain}),
3877 0);
3878 }
3879 }
3880}
3881
3882SDValue
3883ARMTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG,
3884 const ARMSubtarget *Subtarget) const {
3885 unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3886 SDLoc dl(Op);
3887 switch (IntNo) {
3888 default: return SDValue(); // Don't custom lower most intrinsics.
3889 case Intrinsic::thread_pointer: {
3890 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3891 return DAG.getNode(ARMISD::THREAD_POINTER, dl, PtrVT);
3892 }
3893 case Intrinsic::arm_cls: {
3894 const SDValue &Operand = Op.getOperand(1);
3895 const EVT VTy = Op.getValueType();
3896 SDValue SRA =
3897 DAG.getNode(ISD::SRA, dl, VTy, Operand, DAG.getConstant(31, dl, VTy));
3898 SDValue XOR = DAG.getNode(ISD::XOR, dl, VTy, SRA, Operand);
3899 SDValue SHL =
3900 DAG.getNode(ISD::SHL, dl, VTy, XOR, DAG.getConstant(1, dl, VTy));
3901 SDValue OR =
3902 DAG.getNode(ISD::OR, dl, VTy, SHL, DAG.getConstant(1, dl, VTy));
3903 SDValue Result = DAG.getNode(ISD::CTLZ, dl, VTy, OR);
3904 return Result;
3905 }
3906 case Intrinsic::arm_cls64: {
3907 // cls(x) = if cls(hi(x)) != 31 then cls(hi(x))
3908 // else 31 + clz(if hi(x) == 0 then lo(x) else not(lo(x)))
3909 const SDValue &Operand = Op.getOperand(1);
3910 const EVT VTy = Op.getValueType();
3911
3912 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3913 DAG.getConstant(1, dl, VTy));
3914 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VTy, Operand,
3915 DAG.getConstant(0, dl, VTy));
3916 SDValue Constant0 = DAG.getConstant(0, dl, VTy);
3917 SDValue Constant1 = DAG.getConstant(1, dl, VTy);
3918 SDValue Constant31 = DAG.getConstant(31, dl, VTy);
3919 SDValue SRAHi = DAG.getNode(ISD::SRA, dl, VTy, Hi, Constant31);
3920 SDValue XORHi = DAG.getNode(ISD::XOR, dl, VTy, SRAHi, Hi);
3921 SDValue SHLHi = DAG.getNode(ISD::SHL, dl, VTy, XORHi, Constant1);
3922 SDValue ORHi = DAG.getNode(ISD::OR, dl, VTy, SHLHi, Constant1);
3923 SDValue CLSHi = DAG.getNode(ISD::CTLZ, dl, VTy, ORHi);
3924 SDValue CheckLo =
3925 DAG.getSetCC(dl, MVT::i1, CLSHi, Constant31, ISD::CondCode::SETEQ);
3926 SDValue HiIsZero =
3927 DAG.getSetCC(dl, MVT::i1, Hi, Constant0, ISD::CondCode::SETEQ);
3928 SDValue AdjustedLo =
3929 DAG.getSelect(dl, VTy, HiIsZero, Lo, DAG.getNOT(dl, Lo, VTy));
3930 SDValue CLZAdjustedLo = DAG.getNode(ISD::CTLZ, dl, VTy, AdjustedLo);
3931 SDValue Result =
3932 DAG.getSelect(dl, VTy, CheckLo,
3933 DAG.getNode(ISD::ADD, dl, VTy, CLZAdjustedLo, Constant31), CLSHi);
3934 return Result;
3935 }
3936 case Intrinsic::eh_sjlj_lsda: {
3937 MachineFunction &MF = DAG.getMachineFunction();
3938 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
3939 unsigned ARMPCLabelIndex = AFI->createPICLabelUId();
3940 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3941 SDValue CPAddr;
3942 bool IsPositionIndependent = isPositionIndependent();
3943 unsigned PCAdj = IsPositionIndependent ? (Subtarget->isThumb() ? 4 : 8) : 0;
3944 ARMConstantPoolValue *CPV =
3945 ARMConstantPoolConstant::Create(&MF.getFunction(), ARMPCLabelIndex,
3946 ARMCP::CPLSDA, PCAdj);
3947 CPAddr = DAG.getTargetConstantPool(CPV, PtrVT, Align(4));
3948 CPAddr = DAG.getNode(ARMISD::Wrapper, dl, MVT::i32, CPAddr);
3949 SDValue Result = DAG.getLoad(
3950 PtrVT, dl, DAG.getEntryNode(), CPAddr,
3951 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
3952
3953 if (IsPositionIndependent) {
3954 SDValue PICLabel = DAG.getConstant(ARMPCLabelIndex, dl, MVT::i32);
3955 Result = DAG.getNode(ARMISD::PIC_ADD, dl, PtrVT, Result, PICLabel);
3956 }
3957 return Result;
3958 }
3959 case Intrinsic::arm_neon_vabs:
3960 return DAG.getNode(ISD::ABS, SDLoc(Op), Op.getValueType(),
3961 Op.getOperand(1));
3962 case Intrinsic::arm_neon_vmulls:
3963 case Intrinsic::arm_neon_vmullu: {
3964 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmulls)
3965 ? ARMISD::VMULLs : ARMISD::VMULLu;
3966 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3967 Op.getOperand(1), Op.getOperand(2));
3968 }
3969 case Intrinsic::arm_neon_vminnm:
3970 case Intrinsic::arm_neon_vmaxnm: {
3971 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminnm)
3972 ? ISD::FMINNUM : ISD::FMAXNUM;
3973 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3974 Op.getOperand(1), Op.getOperand(2));
3975 }
3976 case Intrinsic::arm_neon_vminu:
3977 case Intrinsic::arm_neon_vmaxu: {
3978 if (Op.getValueType().isFloatingPoint())
3979 return SDValue();
3980 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vminu)
3981 ? ISD::UMIN : ISD::UMAX;
3982 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3983 Op.getOperand(1), Op.getOperand(2));
3984 }
3985 case Intrinsic::arm_neon_vmins:
3986 case Intrinsic::arm_neon_vmaxs: {
3987 // v{min,max}s is overloaded between signed integers and floats.
3988 if (!Op.getValueType().isFloatingPoint()) {
3989 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3990 ? ISD::SMIN : ISD::SMAX;
3991 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3992 Op.getOperand(1), Op.getOperand(2));
3993 }
3994 unsigned NewOpc = (IntNo == Intrinsic::arm_neon_vmins)
3995 ? ISD::FMINIMUM : ISD::FMAXIMUM;
3996 return DAG.getNode(NewOpc, SDLoc(Op), Op.getValueType(),
3997 Op.getOperand(1), Op.getOperand(2));
3998 }
3999 case Intrinsic::arm_neon_vtbl1:
4000 return DAG.getNode(ARMISD::VTBL1, SDLoc(Op), Op.getValueType(),
4001 Op.getOperand(1), Op.getOperand(2));
4002 case Intrinsic::arm_neon_vtbl2:
4003 return DAG.getNode(ARMISD::VTBL2, SDLoc(Op), Op.getValueType(),
4004 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4005 case Intrinsic::arm_mve_pred_i2v:
4006 case Intrinsic::arm_mve_pred_v2i:
4007 return DAG.getNode(ARMISD::PREDICATE_CAST, SDLoc(Op), Op.getValueType(),
4008 Op.getOperand(1));
4009 case Intrinsic::arm_mve_vreinterpretq:
4010 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(Op), Op.getValueType(),
4011 Op.getOperand(1));
4012 case Intrinsic::arm_mve_lsll:
4013 return DAG.getNode(ARMISD::LSLL, SDLoc(Op), Op->getVTList(),
4014 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4015 case Intrinsic::arm_mve_asrl:
4016 return DAG.getNode(ARMISD::ASRL, SDLoc(Op), Op->getVTList(),
4017 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
4018 }
4019}
4020
4021static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
4022 const ARMSubtarget *Subtarget) {
4023 SDLoc dl(Op);
4024 ConstantSDNode *SSIDNode = cast<ConstantSDNode>(Op.getOperand(2));
4025 auto SSID = static_cast<SyncScope::ID>(SSIDNode->getZExtValue());
4026 if (SSID == SyncScope::SingleThread)
4027 return Op;
4028
4029 if (!Subtarget->hasDataBarrier()) {
4030 // Some ARMv6 cpus can support data barriers with an mcr instruction.
4031 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
4032 // here.
4033 assert(Subtarget->hasV6Ops() && !Subtarget->isThumb() &&(static_cast <bool> (Subtarget->hasV6Ops() &&
!Subtarget->isThumb() && "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!"
) ? void (0) : __assert_fail ("Subtarget->hasV6Ops() && !Subtarget->isThumb() && \"Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4034, __extension__ __PRETTY_FUNCTION__))
4034 "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!")(static_cast <bool> (Subtarget->hasV6Ops() &&
!Subtarget->isThumb() && "Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!"
) ? void (0) : __assert_fail ("Subtarget->hasV6Ops() && !Subtarget->isThumb() && \"Unexpected ISD::ATOMIC_FENCE encountered. Should be libcall!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4034, __extension__ __PRETTY_FUNCTION__))
;
4035 return DAG.getNode(ARMISD::MEMBARRIER_MCR, dl, MVT::Other, Op.getOperand(0),
4036 DAG.getConstant(0, dl, MVT::i32));
4037 }
4038
4039 ConstantSDNode *OrdN = cast<ConstantSDNode>(Op.getOperand(1));
4040 AtomicOrdering Ord = static_cast<AtomicOrdering>(OrdN->getZExtValue());
4041 ARM_MB::MemBOpt Domain = ARM_MB::ISH;
4042 if (Subtarget->isMClass()) {
4043 // Only a full system barrier exists in the M-class architectures.
4044 Domain = ARM_MB::SY;
4045 } else if (Subtarget->preferISHSTBarriers() &&
4046 Ord == AtomicOrdering::Release) {
4047 // Swift happens to implement ISHST barriers in a way that's compatible with
4048 // Release semantics but weaker than ISH so we'd be fools not to use
4049 // it. Beware: other processors probably don't!
4050 Domain = ARM_MB::ISHST;
4051 }
4052
4053 return DAG.getNode(ISD::INTRINSIC_VOID, dl, MVT::Other, Op.getOperand(0),
4054 DAG.getConstant(Intrinsic::arm_dmb, dl, MVT::i32),
4055 DAG.getConstant(Domain, dl, MVT::i32));
4056}
4057
4058static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG,
4059 const ARMSubtarget *Subtarget) {
4060 // ARM pre v5TE and Thumb1 does not have preload instructions.
4061 if (!(Subtarget->isThumb2() ||
4062 (!Subtarget->isThumb1Only() && Subtarget->hasV5TEOps())))
4063 // Just preserve the chain.
4064 return Op.getOperand(0);
4065
4066 SDLoc dl(Op);
4067 unsigned isRead = ~cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue() & 1;
4068 if (!isRead &&
4069 (!Subtarget->hasV7Ops() || !Subtarget->hasMPExtension()))
4070 // ARMv7 with MP extension has PLDW.
4071 return Op.getOperand(0);
4072
4073 unsigned isData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
4074 if (Subtarget->isThumb()) {
4075 // Invert the bits.
4076 isRead = ~isRead & 1;
4077 isData = ~isData & 1;
4078 }
4079
4080 return DAG.getNode(ARMISD::PRELOAD, dl, MVT::Other, Op.getOperand(0),
4081 Op.getOperand(1), DAG.getConstant(isRead, dl, MVT::i32),
4082 DAG.getConstant(isData, dl, MVT::i32));
4083}
4084
4085static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) {
4086 MachineFunction &MF = DAG.getMachineFunction();
4087 ARMFunctionInfo *FuncInfo = MF.getInfo<ARMFunctionInfo>();
4088
4089 // vastart just stores the address of the VarArgsFrameIndex slot into the
4090 // memory location argument.
4091 SDLoc dl(Op);
4092 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
4093 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
4094 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4095 return DAG.getStore(Op.getOperand(0), dl, FR, Op.getOperand(1),
4096 MachinePointerInfo(SV));
4097}
4098
4099SDValue ARMTargetLowering::GetF64FormalArgument(CCValAssign &VA,
4100 CCValAssign &NextVA,
4101 SDValue &Root,
4102 SelectionDAG &DAG,
4103 const SDLoc &dl) const {
4104 MachineFunction &MF = DAG.getMachineFunction();
4105 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4106
4107 const TargetRegisterClass *RC;
4108 if (AFI->isThumb1OnlyFunction())
4109 RC = &ARM::tGPRRegClass;
4110 else
4111 RC = &ARM::GPRRegClass;
4112
4113 // Transform the arguments stored in physical registers into virtual ones.
4114 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
4115 SDValue ArgValue = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4116
4117 SDValue ArgValue2;
4118 if (NextVA.isMemLoc()) {
4119 MachineFrameInfo &MFI = MF.getFrameInfo();
4120 int FI = MFI.CreateFixedObject(4, NextVA.getLocMemOffset(), true);
4121
4122 // Create load node to retrieve arguments from the stack.
4123 SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4124 ArgValue2 = DAG.getLoad(
4125 MVT::i32, dl, Root, FIN,
4126 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4127 } else {
4128 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
4129 ArgValue2 = DAG.getCopyFromReg(Root, dl, Reg, MVT::i32);
4130 }
4131 if (!Subtarget->isLittle())
4132 std::swap (ArgValue, ArgValue2);
4133 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, ArgValue, ArgValue2);
4134}
4135
4136// The remaining GPRs hold either the beginning of variable-argument
4137// data, or the beginning of an aggregate passed by value (usually
4138// byval). Either way, we allocate stack slots adjacent to the data
4139// provided by our caller, and store the unallocated registers there.
4140// If this is a variadic function, the va_list pointer will begin with
4141// these values; otherwise, this reassembles a (byval) structure that
4142// was split between registers and memory.
4143// Return: The frame index registers were stored into.
4144int ARMTargetLowering::StoreByValRegs(CCState &CCInfo, SelectionDAG &DAG,
4145 const SDLoc &dl, SDValue &Chain,
4146 const Value *OrigArg,
4147 unsigned InRegsParamRecordIdx,
4148 int ArgOffset, unsigned ArgSize) const {
4149 // Currently, two use-cases possible:
4150 // Case #1. Non-var-args function, and we meet first byval parameter.
4151 // Setup first unallocated register as first byval register;
4152 // eat all remained registers
4153 // (these two actions are performed by HandleByVal method).
4154 // Then, here, we initialize stack frame with
4155 // "store-reg" instructions.
4156 // Case #2. Var-args function, that doesn't contain byval parameters.
4157 // The same: eat all remained unallocated registers,
4158 // initialize stack frame.
4159
4160 MachineFunction &MF = DAG.getMachineFunction();
4161 MachineFrameInfo &MFI = MF.getFrameInfo();
4162 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4163 unsigned RBegin, REnd;
4164 if (InRegsParamRecordIdx < CCInfo.getInRegsParamsCount()) {
4165 CCInfo.getInRegsParamInfo(InRegsParamRecordIdx, RBegin, REnd);
4166 } else {
4167 unsigned RBeginIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4168 RBegin = RBeginIdx == 4 ? (unsigned)ARM::R4 : GPRArgRegs[RBeginIdx];
4169 REnd = ARM::R4;
4170 }
4171
4172 if (REnd != RBegin)
4173 ArgOffset = -4 * (ARM::R4 - RBegin);
4174
4175 auto PtrVT = getPointerTy(DAG.getDataLayout());
4176 int FrameIndex = MFI.CreateFixedObject(ArgSize, ArgOffset, false);
4177 SDValue FIN = DAG.getFrameIndex(FrameIndex, PtrVT);
4178
4179 SmallVector<SDValue, 4> MemOps;
4180 const TargetRegisterClass *RC =
4181 AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
4182
4183 for (unsigned Reg = RBegin, i = 0; Reg < REnd; ++Reg, ++i) {
4184 unsigned VReg = MF.addLiveIn(Reg, RC);
4185 SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
4186 SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
4187 MachinePointerInfo(OrigArg, 4 * i));
4188 MemOps.push_back(Store);
4189 FIN = DAG.getNode(ISD::ADD, dl, PtrVT, FIN, DAG.getConstant(4, dl, PtrVT));
4190 }
4191
4192 if (!MemOps.empty())
4193 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
4194 return FrameIndex;
4195}
4196
4197// Setup stack frame, the va_list pointer will start from.
4198void ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
4199 const SDLoc &dl, SDValue &Chain,
4200 unsigned ArgOffset,
4201 unsigned TotalArgRegsSaveSize,
4202 bool ForceMutable) const {
4203 MachineFunction &MF = DAG.getMachineFunction();
4204 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4205
4206 // Try to store any remaining integer argument regs
4207 // to their spots on the stack so that they may be loaded by dereferencing
4208 // the result of va_next.
4209 // If there is no regs to be stored, just point address after last
4210 // argument passed via stack.
4211 int FrameIndex = StoreByValRegs(CCInfo, DAG, dl, Chain, nullptr,
4212 CCInfo.getInRegsParamsCount(),
4213 CCInfo.getNextStackOffset(),
4214 std::max(4U, TotalArgRegsSaveSize));
4215 AFI->setVarArgsFrameIndex(FrameIndex);
4216}
4217
4218bool ARMTargetLowering::splitValueIntoRegisterParts(
4219 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
4220 unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
4221 bool IsABIRegCopy = CC.hasValue();
4222 EVT ValueVT = Val.getValueType();
4223 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
4224 PartVT == MVT::f32) {
4225 unsigned ValueBits = ValueVT.getSizeInBits();
4226 unsigned PartBits = PartVT.getSizeInBits();
4227 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(ValueBits), Val);
4228 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::getIntegerVT(PartBits), Val);
4229 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
4230 Parts[0] = Val;
4231 return true;
4232 }
4233 return false;
4234}
4235
4236SDValue ARMTargetLowering::joinRegisterPartsIntoValue(
4237 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
4238 MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
4239 bool IsABIRegCopy = CC.hasValue();
4240 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
4241 PartVT == MVT::f32) {
4242 unsigned ValueBits = ValueVT.getSizeInBits();
4243 unsigned PartBits = PartVT.getSizeInBits();
4244 SDValue Val = Parts[0];
4245
4246 Val = DAG.getNode(ISD::BITCAST, DL, MVT::getIntegerVT(PartBits), Val);
4247 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::getIntegerVT(ValueBits), Val);
4248 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
4249 return Val;
4250 }
4251 return SDValue();
4252}
4253
4254SDValue ARMTargetLowering::LowerFormalArguments(
4255 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4256 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4257 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4258 MachineFunction &MF = DAG.getMachineFunction();
4259 MachineFrameInfo &MFI = MF.getFrameInfo();
4260
4261 ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
4262
4263 // Assign locations to all of the incoming arguments.
4264 SmallVector<CCValAssign, 16> ArgLocs;
4265 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4266 *DAG.getContext());
4267 CCInfo.AnalyzeFormalArguments(Ins, CCAssignFnForCall(CallConv, isVarArg));
4268
4269 SmallVector<SDValue, 16> ArgValues;
4270 SDValue ArgValue;
4271 Function::const_arg_iterator CurOrigArg = MF.getFunction().arg_begin();
4272 unsigned CurArgIdx = 0;
4273
4274 // Initially ArgRegsSaveSize is zero.
4275 // Then we increase this value each time we meet byval parameter.
4276 // We also increase this value in case of varargs function.
4277 AFI->setArgRegsSaveSize(0);
4278
4279 // Calculate the amount of stack space that we need to allocate to store
4280 // byval and variadic arguments that are passed in registers.
4281 // We need to know this before we allocate the first byval or variadic
4282 // argument, as they will be allocated a stack slot below the CFA (Canonical
4283 // Frame Address, the stack pointer at entry to the function).
4284 unsigned ArgRegBegin = ARM::R4;
4285 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4286 if (CCInfo.getInRegsParamsProcessed() >= CCInfo.getInRegsParamsCount())
4287 break;
4288
4289 CCValAssign &VA = ArgLocs[i];
4290 unsigned Index = VA.getValNo();
4291 ISD::ArgFlagsTy Flags = Ins[Index].Flags;
4292 if (!Flags.isByVal())
4293 continue;
4294
4295 assert(VA.isMemLoc() && "unexpected byval pointer in reg")(static_cast <bool> (VA.isMemLoc() && "unexpected byval pointer in reg"
) ? void (0) : __assert_fail ("VA.isMemLoc() && \"unexpected byval pointer in reg\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4295, __extension__ __PRETTY_FUNCTION__))
;
4296 unsigned RBegin, REnd;
4297 CCInfo.getInRegsParamInfo(CCInfo.getInRegsParamsProcessed(), RBegin, REnd);
4298 ArgRegBegin = std::min(ArgRegBegin, RBegin);
4299
4300 CCInfo.nextInRegsParam();
4301 }
4302 CCInfo.rewindByValRegsInfo();
4303
4304 int lastInsIndex = -1;
4305 if (isVarArg && MFI.hasVAStart()) {
4306 unsigned RegIdx = CCInfo.getFirstUnallocated(GPRArgRegs);
4307 if (RegIdx != array_lengthof(GPRArgRegs))
4308 ArgRegBegin = std::min(ArgRegBegin, (unsigned)GPRArgRegs[RegIdx]);
4309 }
4310
4311 unsigned TotalArgRegsSaveSize = 4 * (ARM::R4 - ArgRegBegin);
4312 AFI->setArgRegsSaveSize(TotalArgRegsSaveSize);
4313 auto PtrVT = getPointerTy(DAG.getDataLayout());
4314
4315 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4316 CCValAssign &VA = ArgLocs[i];
4317 if (Ins[VA.getValNo()].isOrigArg()) {
4318 std::advance(CurOrigArg,
4319 Ins[VA.getValNo()].getOrigArgIndex() - CurArgIdx);
4320 CurArgIdx = Ins[VA.getValNo()].getOrigArgIndex();
4321 }
4322 // Arguments stored in registers.
4323 if (VA.isRegLoc()) {
4324 EVT RegVT = VA.getLocVT();
4325
4326 if (VA.needsCustom() && VA.getLocVT() == MVT::v2f64) {
4327 // f64 and vector types are split up into multiple registers or
4328 // combinations of registers and stack slots.
4329 SDValue ArgValue1 =
4330 GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4331 VA = ArgLocs[++i]; // skip ahead to next loc
4332 SDValue ArgValue2;
4333 if (VA.isMemLoc()) {
4334 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), true);
4335 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4336 ArgValue2 = DAG.getLoad(
4337 MVT::f64, dl, Chain, FIN,
4338 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
4339 } else {
4340 ArgValue2 = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4341 }
4342 ArgValue = DAG.getNode(ISD::UNDEF, dl, MVT::v2f64);
4343 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4344 ArgValue1, DAG.getIntPtrConstant(0, dl));
4345 ArgValue = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, ArgValue,
4346 ArgValue2, DAG.getIntPtrConstant(1, dl));
4347 } else if (VA.needsCustom() && VA.getLocVT() == MVT::f64) {
4348 ArgValue = GetF64FormalArgument(VA, ArgLocs[++i], Chain, DAG, dl);
4349 } else {
4350 const TargetRegisterClass *RC;
4351
4352 if (RegVT == MVT::f16 || RegVT == MVT::bf16)
4353 RC = &ARM::HPRRegClass;
4354 else if (RegVT == MVT::f32)
4355 RC = &ARM::SPRRegClass;
4356 else if (RegVT == MVT::f64 || RegVT == MVT::v4f16 ||
4357 RegVT == MVT::v4bf16)
4358 RC = &ARM::DPRRegClass;
4359 else if (RegVT == MVT::v2f64 || RegVT == MVT::v8f16 ||
4360 RegVT == MVT::v8bf16)
4361 RC = &ARM::QPRRegClass;
4362 else if (RegVT == MVT::i32)
4363 RC = AFI->isThumb1OnlyFunction() ? &ARM::tGPRRegClass
4364 : &ARM::GPRRegClass;
4365 else
4366 llvm_unreachable("RegVT not supported by FORMAL_ARGUMENTS Lowering")::llvm::llvm_unreachable_internal("RegVT not supported by FORMAL_ARGUMENTS Lowering"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4366)
;
4367
4368 // Transform the arguments in physical registers into virtual ones.
4369 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
4370 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
4371
4372 // If this value is passed in r0 and has the returned attribute (e.g.
4373 // C++ 'structors), record this fact for later use.
4374 if (VA.getLocReg() == ARM::R0 && Ins[VA.getValNo()].Flags.isReturned()) {
4375 AFI->setPreservesR0();
4376 }
4377 }
4378
4379 // If this is an 8 or 16-bit value, it is really passed promoted
4380 // to 32 bits. Insert an assert[sz]ext to capture this, then
4381 // truncate to the right size.
4382 switch (VA.getLocInfo()) {
4383 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4383)
;
4384 case CCValAssign::Full: break;
4385 case CCValAssign::BCvt:
4386 ArgValue = DAG.getNode(ISD::BITCAST, dl, VA.getValVT(), ArgValue);
4387 break;
4388 case CCValAssign::SExt:
4389 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
4390 DAG.getValueType(VA.getValVT()));
4391 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
4392 break;
4393 case CCValAssign::ZExt:
4394 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
4395 DAG.getValueType(VA.getValVT()));
4396 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
4397 break;
4398 }
4399
4400 // f16 arguments have their size extended to 4 bytes and passed as if they
4401 // had been copied to the LSBs of a 32-bit register.
4402 // For that, it's passed extended to i32 (soft ABI) or to f32 (hard ABI)
4403 if (VA.needsCustom() &&
4404 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
4405 ArgValue = MoveToHPR(dl, DAG, VA.getLocVT(), VA.getValVT(), ArgValue);
4406
4407 InVals.push_back(ArgValue);
4408 } else { // VA.isRegLoc()
4409 // sanity check
4410 assert(VA.isMemLoc())(static_cast <bool> (VA.isMemLoc()) ? void (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4410, __extension__ __PRETTY_FUNCTION__))
;
4411 assert(VA.getValVT() != MVT::i64 && "i64 should already be lowered")(static_cast <bool> (VA.getValVT() != MVT::i64 &&
"i64 should already be lowered") ? void (0) : __assert_fail (
"VA.getValVT() != MVT::i64 && \"i64 should already be lowered\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4411, __extension__ __PRETTY_FUNCTION__))
;
4412
4413 int index = VA.getValNo();
4414
4415 // Some Ins[] entries become multiple ArgLoc[] entries.
4416 // Process them only once.
4417 if (index != lastInsIndex)
4418 {
4419 ISD::ArgFlagsTy Flags = Ins[index].Flags;
4420 // FIXME: For now, all byval parameter objects are marked mutable.
4421 // This can be changed with more analysis.
4422 // In case of tail call optimization mark all arguments mutable.
4423 // Since they could be overwritten by lowering of arguments in case of
4424 // a tail call.
4425 if (Flags.isByVal()) {
4426 assert(Ins[index].isOrigArg() &&(static_cast <bool> (Ins[index].isOrigArg() && "Byval arguments cannot be implicit"
) ? void (0) : __assert_fail ("Ins[index].isOrigArg() && \"Byval arguments cannot be implicit\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4427, __extension__ __PRETTY_FUNCTION__))
4427 "Byval arguments cannot be implicit")(static_cast <bool> (Ins[index].isOrigArg() && "Byval arguments cannot be implicit"
) ? void (0) : __assert_fail ("Ins[index].isOrigArg() && \"Byval arguments cannot be implicit\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4427, __extension__ __PRETTY_FUNCTION__))
;
4428 unsigned CurByValIndex = CCInfo.getInRegsParamsProcessed();
4429
4430 int FrameIndex = StoreByValRegs(
4431 CCInfo, DAG, dl, Chain, &*CurOrigArg, CurByValIndex,
4432 VA.getLocMemOffset(), Flags.getByValSize());
4433 InVals.push_back(DAG.getFrameIndex(FrameIndex, PtrVT));
4434 CCInfo.nextInRegsParam();
4435 } else {
4436 unsigned FIOffset = VA.getLocMemOffset();
4437 int FI = MFI.CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
4438 FIOffset, true);
4439
4440 // Create load nodes to retrieve arguments from the stack.
4441 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
4442 InVals.push_back(DAG.getLoad(VA.getValVT(), dl, Chain, FIN,
4443 MachinePointerInfo::getFixedStack(
4444 DAG.getMachineFunction(), FI)));
4445 }
4446 lastInsIndex = index;
4447 }
4448 }
4449 }
4450
4451 // varargs
4452 if (isVarArg && MFI.hasVAStart()) {
4453 VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getNextStackOffset(),
4454 TotalArgRegsSaveSize);
4455 if (AFI->isCmseNSEntryFunction()) {
4456 DiagnosticInfoUnsupported Diag(
4457 DAG.getMachineFunction().getFunction(),
4458 "secure entry function must not be variadic", dl.getDebugLoc());
4459 DAG.getContext()->diagnose(Diag);
4460 }
4461 }
4462
4463 AFI->setArgumentStackSize(CCInfo.getNextStackOffset());
4464
4465 if (CCInfo.getNextStackOffset() > 0 && AFI->isCmseNSEntryFunction()) {
4466 DiagnosticInfoUnsupported Diag(
4467 DAG.getMachineFunction().getFunction(),
4468 "secure entry function requires arguments on stack", dl.getDebugLoc());
4469 DAG.getContext()->diagnose(Diag);
4470 }
4471
4472 return Chain;
4473}
4474
4475/// isFloatingPointZero - Return true if this is +0.0.
4476static bool isFloatingPointZero(SDValue Op) {
4477 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Op))
4478 return CFP->getValueAPF().isPosZero();
4479 else if (ISD::isEXTLoad(Op.getNode()) || ISD::isNON_EXTLoad(Op.getNode())) {
4480 // Maybe this has already been legalized into the constant pool?
4481 if (Op.getOperand(1).getOpcode() == ARMISD::Wrapper) {
4482 SDValue WrapperOp = Op.getOperand(1).getOperand(0);
4483 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(WrapperOp))
4484 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CP->getConstVal()))
4485 return CFP->getValueAPF().isPosZero();
4486 }
4487 } else if (Op->getOpcode() == ISD::BITCAST &&
4488 Op->getValueType(0) == MVT::f64) {
4489 // Handle (ISD::BITCAST (ARMISD::VMOVIMM (ISD::TargetConstant 0)) MVT::f64)
4490 // created by LowerConstantFP().
4491 SDValue BitcastOp = Op->getOperand(0);
4492 if (BitcastOp->getOpcode() == ARMISD::VMOVIMM &&
4493 isNullConstant(BitcastOp->getOperand(0)))
4494 return true;
4495 }
4496 return false;
4497}
4498
4499/// Returns appropriate ARM CMP (cmp) and corresponding condition code for
4500/// the given operands.
4501SDValue ARMTargetLowering::getARMCmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
4502 SDValue &ARMcc, SelectionDAG &DAG,
4503 const SDLoc &dl) const {
4504 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
4505 unsigned C = RHSC->getZExtValue();
4506 if (!isLegalICmpImmediate((int32_t)C)) {
4507 // Constant does not fit, try adjusting it by one.
4508 switch (CC) {
4509 default: break;
4510 case ISD::SETLT:
4511 case ISD::SETGE:
4512 if (C != 0x80000000 && isLegalICmpImmediate(C-1)) {
4513 CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
4514 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4515 }
4516 break;
4517 case ISD::SETULT:
4518 case ISD::SETUGE:
4519 if (C != 0 && isLegalICmpImmediate(C-1)) {
4520 CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
4521 RHS = DAG.getConstant(C - 1, dl, MVT::i32);
4522 }
4523 break;
4524 case ISD::SETLE:
4525 case ISD::SETGT:
4526 if (C != 0x7fffffff && isLegalICmpImmediate(C+1)) {
4527 CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
4528 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4529 }
4530 break;
4531 case ISD::SETULE:
4532 case ISD::SETUGT:
4533 if (C != 0xffffffff && isLegalICmpImmediate(C+1)) {
4534 CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
4535 RHS = DAG.getConstant(C + 1, dl, MVT::i32);
4536 }
4537 break;
4538 }
4539 }
4540 } else if ((ARM_AM::getShiftOpcForNode(LHS.getOpcode()) != ARM_AM::no_shift) &&
4541 (ARM_AM::getShiftOpcForNode(RHS.getOpcode()) == ARM_AM::no_shift)) {
4542 // In ARM and Thumb-2, the compare instructions can shift their second
4543 // operand.
4544 CC = ISD::getSetCCSwappedOperands(CC);
4545 std::swap(LHS, RHS);
4546 }
4547
4548 // Thumb1 has very limited immediate modes, so turning an "and" into a
4549 // shift can save multiple instructions.
4550 //
4551 // If we have (x & C1), and C1 is an appropriate mask, we can transform it
4552 // into "((x << n) >> n)". But that isn't necessarily profitable on its
4553 // own. If it's the operand to an unsigned comparison with an immediate,
4554 // we can eliminate one of the shifts: we transform
4555 // "((x << n) >> n) == C2" to "(x << n) == (C2 << n)".
4556 //
4557 // We avoid transforming cases which aren't profitable due to encoding
4558 // details:
4559 //
4560 // 1. C2 fits into the immediate field of a cmp, and the transformed version
4561 // would not; in that case, we're essentially trading one immediate load for
4562 // another.
4563 // 2. C1 is 255 or 65535, so we can use uxtb or uxth.
4564 // 3. C2 is zero; we have other code for this special case.
4565 //
4566 // FIXME: Figure out profitability for Thumb2; we usually can't save an
4567 // instruction, since the AND is always one instruction anyway, but we could
4568 // use narrow instructions in some cases.
4569 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::AND &&
4570 LHS->hasOneUse() && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4571 LHS.getValueType() == MVT::i32 && isa<ConstantSDNode>(RHS) &&
4572 !isSignedIntSetCC(CC)) {
4573 unsigned Mask = cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue();
4574 auto *RHSC = cast<ConstantSDNode>(RHS.getNode());
4575 uint64_t RHSV = RHSC->getZExtValue();
4576 if (isMask_32(Mask) && (RHSV & ~Mask) == 0 && Mask != 255 && Mask != 65535) {
4577 unsigned ShiftBits = countLeadingZeros(Mask);
4578 if (RHSV && (RHSV > 255 || (RHSV << ShiftBits) <= 255)) {
4579 SDValue ShiftAmt = DAG.getConstant(ShiftBits, dl, MVT::i32);
4580 LHS = DAG.getNode(ISD::SHL, dl, MVT::i32, LHS.getOperand(0), ShiftAmt);
4581 RHS = DAG.getConstant(RHSV << ShiftBits, dl, MVT::i32);
4582 }
4583 }
4584 }
4585
4586 // The specific comparison "(x<<c) > 0x80000000U" can be optimized to a
4587 // single "lsls x, c+1". The shift sets the "C" and "Z" flags the same
4588 // way a cmp would.
4589 // FIXME: Add support for ARM/Thumb2; this would need isel patterns, and
4590 // some tweaks to the heuristics for the previous and->shift transform.
4591 // FIXME: Optimize cases where the LHS isn't a shift.
4592 if (Subtarget->isThumb1Only() && LHS->getOpcode() == ISD::SHL &&
4593 isa<ConstantSDNode>(RHS) &&
4594 cast<ConstantSDNode>(RHS)->getZExtValue() == 0x80000000U &&
4595 CC == ISD::SETUGT && isa<ConstantSDNode>(LHS.getOperand(1)) &&
4596 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() < 31) {
4597 unsigned ShiftAmt =
4598 cast<ConstantSDNode>(LHS.getOperand(1))->getZExtValue() + 1;
4599 SDValue Shift = DAG.getNode(ARMISD::LSLS, dl,
4600 DAG.getVTList(MVT::i32, MVT::i32),
4601 LHS.getOperand(0),
4602 DAG.getConstant(ShiftAmt, dl, MVT::i32));
4603 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
4604 Shift.getValue(1), SDValue());
4605 ARMcc = DAG.getConstant(ARMCC::HI, dl, MVT::i32);
4606 return Chain.getValue(1);
4607 }
4608
4609 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
4610
4611 // If the RHS is a constant zero then the V (overflow) flag will never be
4612 // set. This can allow us to simplify GE to PL or LT to MI, which can be
4613 // simpler for other passes (like the peephole optimiser) to deal with.
4614 if (isNullConstant(RHS)) {
4615 switch (CondCode) {
4616 default: break;
4617 case ARMCC::GE:
4618 CondCode = ARMCC::PL;
4619 break;
4620 case ARMCC::LT:
4621 CondCode = ARMCC::MI;
4622 break;
4623 }
4624 }
4625
4626 ARMISD::NodeType CompareType;
4627 switch (CondCode) {
4628 default:
4629 CompareType = ARMISD::CMP;
4630 break;
4631 case ARMCC::EQ:
4632 case ARMCC::NE:
4633 // Uses only Z Flag
4634 CompareType = ARMISD::CMPZ;
4635 break;
4636 }
4637 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
4638 return DAG.getNode(CompareType, dl, MVT::Glue, LHS, RHS);
4639}
4640
4641/// Returns a appropriate VFP CMP (fcmp{s|d}+fmstat) for the given operands.
4642SDValue ARMTargetLowering::getVFPCmp(SDValue LHS, SDValue RHS,
4643 SelectionDAG &DAG, const SDLoc &dl,
4644 bool Signaling) const {
4645 assert(Subtarget->hasFP64() || RHS.getValueType() != MVT::f64)(static_cast <bool> (Subtarget->hasFP64() || RHS.getValueType
() != MVT::f64) ? void (0) : __assert_fail ("Subtarget->hasFP64() || RHS.getValueType() != MVT::f64"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4645, __extension__ __PRETTY_FUNCTION__))
;
4646 SDValue Cmp;
4647 if (!isFloatingPointZero(RHS))
4648 Cmp = DAG.getNode(Signaling ? ARMISD::CMPFPE : ARMISD::CMPFP,
4649 dl, MVT::Glue, LHS, RHS);
4650 else
4651 Cmp = DAG.getNode(Signaling ? ARMISD::CMPFPEw0 : ARMISD::CMPFPw0,
4652 dl, MVT::Glue, LHS);
4653 return DAG.getNode(ARMISD::FMSTAT, dl, MVT::Glue, Cmp);
4654}
4655
4656/// duplicateCmp - Glue values can have only one use, so this function
4657/// duplicates a comparison node.
4658SDValue
4659ARMTargetLowering::duplicateCmp(SDValue Cmp, SelectionDAG &DAG) const {
4660 unsigned Opc = Cmp.getOpcode();
4661 SDLoc DL(Cmp);
4662 if (Opc == ARMISD::CMP || Opc == ARMISD::CMPZ)
4663 return DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
4664
4665 assert(Opc == ARMISD::FMSTAT && "unexpected comparison operation")(static_cast <bool> (Opc == ARMISD::FMSTAT && "unexpected comparison operation"
) ? void (0) : __assert_fail ("Opc == ARMISD::FMSTAT && \"unexpected comparison operation\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4665, __extension__ __PRETTY_FUNCTION__))
;
4666 Cmp = Cmp.getOperand(0);
4667 Opc = Cmp.getOpcode();
4668 if (Opc == ARMISD::CMPFP)
4669 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0),Cmp.getOperand(1));
4670 else {
4671 assert(Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT")(static_cast <bool> (Opc == ARMISD::CMPFPw0 && "unexpected operand of FMSTAT"
) ? void (0) : __assert_fail ("Opc == ARMISD::CMPFPw0 && \"unexpected operand of FMSTAT\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4671, __extension__ __PRETTY_FUNCTION__))
;
4672 Cmp = DAG.getNode(Opc, DL, MVT::Glue, Cmp.getOperand(0));
4673 }
4674 return DAG.getNode(ARMISD::FMSTAT, DL, MVT::Glue, Cmp);
4675}
4676
4677// This function returns three things: the arithmetic computation itself
4678// (Value), a comparison (OverflowCmp), and a condition code (ARMcc). The
4679// comparison and the condition code define the case in which the arithmetic
4680// computation *does not* overflow.
4681std::pair<SDValue, SDValue>
4682ARMTargetLowering::getARMXALUOOp(SDValue Op, SelectionDAG &DAG,
4683 SDValue &ARMcc) const {
4684 assert(Op.getValueType() == MVT::i32 && "Unsupported value type")(static_cast <bool> (Op.getValueType() == MVT::i32 &&
"Unsupported value type") ? void (0) : __assert_fail ("Op.getValueType() == MVT::i32 && \"Unsupported value type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4684, __extension__ __PRETTY_FUNCTION__))
;
4685
4686 SDValue Value, OverflowCmp;
4687 SDValue LHS = Op.getOperand(0);
4688 SDValue RHS = Op.getOperand(1);
4689 SDLoc dl(Op);
4690
4691 // FIXME: We are currently always generating CMPs because we don't support
4692 // generating CMN through the backend. This is not as good as the natural
4693 // CMP case because it causes a register dependency and cannot be folded
4694 // later.
4695
4696 switch (Op.getOpcode()) {
4697 default:
4698 llvm_unreachable("Unknown overflow instruction!")::llvm::llvm_unreachable_internal("Unknown overflow instruction!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4698)
;
4699 case ISD::SADDO:
4700 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4701 Value = DAG.getNode(ISD::ADD, dl, Op.getValueType(), LHS, RHS);
4702 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
4703 break;
4704 case ISD::UADDO:
4705 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4706 // We use ADDC here to correspond to its use in LowerUnsignedALUO.
4707 // We do not use it in the USUBO case as Value may not be used.
4708 Value = DAG.getNode(ARMISD::ADDC, dl,
4709 DAG.getVTList(Op.getValueType(), MVT::i32), LHS, RHS)
4710 .getValue(0);
4711 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value, LHS);
4712 break;
4713 case ISD::SSUBO:
4714 ARMcc = DAG.getConstant(ARMCC::VC, dl, MVT::i32);
4715 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4716 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
4717 break;
4718 case ISD::USUBO:
4719 ARMcc = DAG.getConstant(ARMCC::HS, dl, MVT::i32);
4720 Value = DAG.getNode(ISD::SUB, dl, Op.getValueType(), LHS, RHS);
4721 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, LHS, RHS);
4722 break;
4723 case ISD::UMULO:
4724 // We generate a UMUL_LOHI and then check if the high word is 0.
4725 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4726 Value = DAG.getNode(ISD::UMUL_LOHI, dl,
4727 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4728 LHS, RHS);
4729 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
4730 DAG.getConstant(0, dl, MVT::i32));
4731 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4732 break;
4733 case ISD::SMULO:
4734 // We generate a SMUL_LOHI and then check if all the bits of the high word
4735 // are the same as the sign bit of the low word.
4736 ARMcc = DAG.getConstant(ARMCC::EQ, dl, MVT::i32);
4737 Value = DAG.getNode(ISD::SMUL_LOHI, dl,
4738 DAG.getVTList(Op.getValueType(), Op.getValueType()),
4739 LHS, RHS);
4740 OverflowCmp = DAG.getNode(ARMISD::CMP, dl, MVT::Glue, Value.getValue(1),
4741 DAG.getNode(ISD::SRA, dl, Op.getValueType(),
4742 Value.getValue(0),
4743 DAG.getConstant(31, dl, MVT::i32)));
4744 Value = Value.getValue(0); // We only want the low 32 bits for the result.
4745 break;
4746 } // switch (...)
4747
4748 return std::make_pair(Value, OverflowCmp);
4749}
4750
4751SDValue
4752ARMTargetLowering::LowerSignedALUO(SDValue Op, SelectionDAG &DAG) const {
4753 // Let legalize expand this if it isn't a legal type yet.
4754 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
4755 return SDValue();
4756
4757 SDValue Value, OverflowCmp;
4758 SDValue ARMcc;
4759 std::tie(Value, OverflowCmp) = getARMXALUOOp(Op, DAG, ARMcc);
4760 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4761 SDLoc dl(Op);
4762 // We use 0 and 1 as false and true values.
4763 SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
4764 SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
4765 EVT VT = Op.getValueType();
4766
4767 SDValue Overflow = DAG.getNode(ARMISD::CMOV, dl, VT, TVal, FVal,
4768 ARMcc, CCR, OverflowCmp);
4769
4770 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
4771 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
4772}
4773
4774static SDValue ConvertBooleanCarryToCarryFlag(SDValue BoolCarry,
4775 SelectionDAG &DAG) {
4776 SDLoc DL(BoolCarry);
4777 EVT CarryVT = BoolCarry.getValueType();
4778
4779 // This converts the boolean value carry into the carry flag by doing
4780 // ARMISD::SUBC Carry, 1
4781 SDValue Carry = DAG.getNode(ARMISD::SUBC, DL,
4782 DAG.getVTList(CarryVT, MVT::i32),
4783 BoolCarry, DAG.getConstant(1, DL, CarryVT));
4784 return Carry.getValue(1);
4785}
4786
4787static SDValue ConvertCarryFlagToBooleanCarry(SDValue Flags, EVT VT,
4788 SelectionDAG &DAG) {
4789 SDLoc DL(Flags);
4790
4791 // Now convert the carry flag into a boolean carry. We do this
4792 // using ARMISD:ADDE 0, 0, Carry
4793 return DAG.getNode(ARMISD::ADDE, DL, DAG.getVTList(VT, MVT::i32),
4794 DAG.getConstant(0, DL, MVT::i32),
4795 DAG.getConstant(0, DL, MVT::i32), Flags);
4796}
4797
4798SDValue ARMTargetLowering::LowerUnsignedALUO(SDValue Op,
4799 SelectionDAG &DAG) const {
4800 // Let legalize expand this if it isn't a legal type yet.
4801 if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
4802 return SDValue();
4803
4804 SDValue LHS = Op.getOperand(0);
4805 SDValue RHS = Op.getOperand(1);
4806 SDLoc dl(Op);
4807
4808 EVT VT = Op.getValueType();
4809 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
4810 SDValue Value;
4811 SDValue Overflow;
4812 switch (Op.getOpcode()) {
4813 default:
4814 llvm_unreachable("Unknown overflow instruction!")::llvm::llvm_unreachable_internal("Unknown overflow instruction!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4814)
;
4815 case ISD::UADDO:
4816 Value = DAG.getNode(ARMISD::ADDC, dl, VTs, LHS, RHS);
4817 // Convert the carry flag into a boolean value.
4818 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
4819 break;
4820 case ISD::USUBO: {
4821 Value = DAG.getNode(ARMISD::SUBC, dl, VTs, LHS, RHS);
4822 // Convert the carry flag into a boolean value.
4823 Overflow = ConvertCarryFlagToBooleanCarry(Value.getValue(1), VT, DAG);
4824 // ARMISD::SUBC returns 0 when we have to borrow, so make it an overflow
4825 // value. So compute 1 - C.
4826 Overflow = DAG.getNode(ISD::SUB, dl, MVT::i32,
4827 DAG.getConstant(1, dl, MVT::i32), Overflow);
4828 break;
4829 }
4830 }
4831
4832 return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
4833}
4834
4835static SDValue LowerSADDSUBSAT(SDValue Op, SelectionDAG &DAG,
4836 const ARMSubtarget *Subtarget) {
4837 EVT VT = Op.getValueType();
4838 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
4839 return SDValue();
4840 if (!VT.isSimple())
4841 return SDValue();
4842
4843 unsigned NewOpcode;
4844 bool IsAdd = Op->getOpcode() == ISD::SADDSAT;
4845 switch (VT.getSimpleVT().SimpleTy) {
4846 default:
4847 return SDValue();
4848 case MVT::i8:
4849 NewOpcode = IsAdd ? ARMISD::QADD8b : ARMISD::QSUB8b;
4850 break;
4851 case MVT::i16:
4852 NewOpcode = IsAdd ? ARMISD::QADD16b : ARMISD::QSUB16b;
4853 break;
4854 }
4855
4856 SDLoc dl(Op);
4857 SDValue Add =
4858 DAG.getNode(NewOpcode, dl, MVT::i32,
4859 DAG.getSExtOrTrunc(Op->getOperand(0), dl, MVT::i32),
4860 DAG.getSExtOrTrunc(Op->getOperand(1), dl, MVT::i32));
4861 return DAG.getNode(ISD::TRUNCATE, dl, VT, Add);
4862}
4863
4864SDValue ARMTargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
4865 SDValue Cond = Op.getOperand(0);
4866 SDValue SelectTrue = Op.getOperand(1);
4867 SDValue SelectFalse = Op.getOperand(2);
4868 SDLoc dl(Op);
4869 unsigned Opc = Cond.getOpcode();
4870
4871 if (Cond.getResNo() == 1 &&
4872 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
4873 Opc == ISD::USUBO)) {
4874 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
4875 return SDValue();
4876
4877 SDValue Value, OverflowCmp;
4878 SDValue ARMcc;
4879 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
4880 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
4881 EVT VT = Op.getValueType();
4882
4883 return getCMOV(dl, VT, SelectTrue, SelectFalse, ARMcc, CCR,
4884 OverflowCmp, DAG);
4885 }
4886
4887 // Convert:
4888 //
4889 // (select (cmov 1, 0, cond), t, f) -> (cmov t, f, cond)
4890 // (select (cmov 0, 1, cond), t, f) -> (cmov f, t, cond)
4891 //
4892 if (Cond.getOpcode() == ARMISD::CMOV && Cond.hasOneUse()) {
4893 const ConstantSDNode *CMOVTrue =
4894 dyn_cast<ConstantSDNode>(Cond.getOperand(0));
4895 const ConstantSDNode *CMOVFalse =
4896 dyn_cast<ConstantSDNode>(Cond.getOperand(1));
4897
4898 if (CMOVTrue && CMOVFalse) {
4899 unsigned CMOVTrueVal = CMOVTrue->getZExtValue();
4900 unsigned CMOVFalseVal = CMOVFalse->getZExtValue();
4901
4902 SDValue True;
4903 SDValue False;
4904 if (CMOVTrueVal == 1 && CMOVFalseVal == 0) {
4905 True = SelectTrue;
4906 False = SelectFalse;
4907 } else if (CMOVTrueVal == 0 && CMOVFalseVal == 1) {
4908 True = SelectFalse;
4909 False = SelectTrue;
4910 }
4911
4912 if (True.getNode() && False.getNode()) {
4913 EVT VT = Op.getValueType();
4914 SDValue ARMcc = Cond.getOperand(2);
4915 SDValue CCR = Cond.getOperand(3);
4916 SDValue Cmp = duplicateCmp(Cond.getOperand(4), DAG);
4917 assert(True.getValueType() == VT)(static_cast <bool> (True.getValueType() == VT) ? void (
0) : __assert_fail ("True.getValueType() == VT", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 4917, __extension__ __PRETTY_FUNCTION__))
;
4918 return getCMOV(dl, VT, True, False, ARMcc, CCR, Cmp, DAG);
4919 }
4920 }
4921 }
4922
4923 // ARM's BooleanContents value is UndefinedBooleanContent. Mask out the
4924 // undefined bits before doing a full-word comparison with zero.
4925 Cond = DAG.getNode(ISD::AND, dl, Cond.getValueType(), Cond,
4926 DAG.getConstant(1, dl, Cond.getValueType()));
4927
4928 return DAG.getSelectCC(dl, Cond,
4929 DAG.getConstant(0, dl, Cond.getValueType()),
4930 SelectTrue, SelectFalse, ISD::SETNE);
4931}
4932
4933static void checkVSELConstraints(ISD::CondCode CC, ARMCC::CondCodes &CondCode,
4934 bool &swpCmpOps, bool &swpVselOps) {
4935 // Start by selecting the GE condition code for opcodes that return true for
4936 // 'equality'
4937 if (CC == ISD::SETUGE || CC == ISD::SETOGE || CC == ISD::SETOLE ||
4938 CC == ISD::SETULE || CC == ISD::SETGE || CC == ISD::SETLE)
4939 CondCode = ARMCC::GE;
4940
4941 // and GT for opcodes that return false for 'equality'.
4942 else if (CC == ISD::SETUGT || CC == ISD::SETOGT || CC == ISD::SETOLT ||
4943 CC == ISD::SETULT || CC == ISD::SETGT || CC == ISD::SETLT)
4944 CondCode = ARMCC::GT;
4945
4946 // Since we are constrained to GE/GT, if the opcode contains 'less', we need
4947 // to swap the compare operands.
4948 if (CC == ISD::SETOLE || CC == ISD::SETULE || CC == ISD::SETOLT ||
4949 CC == ISD::SETULT || CC == ISD::SETLE || CC == ISD::SETLT)
4950 swpCmpOps = true;
4951
4952 // Both GT and GE are ordered comparisons, and return false for 'unordered'.
4953 // If we have an unordered opcode, we need to swap the operands to the VSEL
4954 // instruction (effectively negating the condition).
4955 //
4956 // This also has the effect of swapping which one of 'less' or 'greater'
4957 // returns true, so we also swap the compare operands. It also switches
4958 // whether we return true for 'equality', so we compensate by picking the
4959 // opposite condition code to our original choice.
4960 if (CC == ISD::SETULE || CC == ISD::SETULT || CC == ISD::SETUGE ||
4961 CC == ISD::SETUGT) {
4962 swpCmpOps = !swpCmpOps;
4963 swpVselOps = !swpVselOps;
4964 CondCode = CondCode == ARMCC::GT ? ARMCC::GE : ARMCC::GT;
4965 }
4966
4967 // 'ordered' is 'anything but unordered', so use the VS condition code and
4968 // swap the VSEL operands.
4969 if (CC == ISD::SETO) {
4970 CondCode = ARMCC::VS;
4971 swpVselOps = true;
4972 }
4973
4974 // 'unordered or not equal' is 'anything but equal', so use the EQ condition
4975 // code and swap the VSEL operands. Also do this if we don't care about the
4976 // unordered case.
4977 if (CC == ISD::SETUNE || CC == ISD::SETNE) {
4978 CondCode = ARMCC::EQ;
4979 swpVselOps = true;
4980 }
4981}
4982
4983SDValue ARMTargetLowering::getCMOV(const SDLoc &dl, EVT VT, SDValue FalseVal,
4984 SDValue TrueVal, SDValue ARMcc, SDValue CCR,
4985 SDValue Cmp, SelectionDAG &DAG) const {
4986 if (!Subtarget->hasFP64() && VT == MVT::f64) {
4987 FalseVal = DAG.getNode(ARMISD::VMOVRRD, dl,
4988 DAG.getVTList(MVT::i32, MVT::i32), FalseVal);
4989 TrueVal = DAG.getNode(ARMISD::VMOVRRD, dl,
4990 DAG.getVTList(MVT::i32, MVT::i32), TrueVal);
4991
4992 SDValue TrueLow = TrueVal.getValue(0);
4993 SDValue TrueHigh = TrueVal.getValue(1);
4994 SDValue FalseLow = FalseVal.getValue(0);
4995 SDValue FalseHigh = FalseVal.getValue(1);
4996
4997 SDValue Low = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseLow, TrueLow,
4998 ARMcc, CCR, Cmp);
4999 SDValue High = DAG.getNode(ARMISD::CMOV, dl, MVT::i32, FalseHigh, TrueHigh,
5000 ARMcc, CCR, duplicateCmp(Cmp, DAG));
5001
5002 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Low, High);
5003 } else {
5004 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal, ARMcc, CCR,
5005 Cmp);
5006 }
5007}
5008
5009static bool isGTorGE(ISD::CondCode CC) {
5010 return CC == ISD::SETGT || CC == ISD::SETGE;
5011}
5012
5013static bool isLTorLE(ISD::CondCode CC) {
5014 return CC == ISD::SETLT || CC == ISD::SETLE;
5015}
5016
5017// See if a conditional (LHS CC RHS ? TrueVal : FalseVal) is lower-saturating.
5018// All of these conditions (and their <= and >= counterparts) will do:
5019// x < k ? k : x
5020// x > k ? x : k
5021// k < x ? x : k
5022// k > x ? k : x
5023static bool isLowerSaturate(const SDValue LHS, const SDValue RHS,
5024 const SDValue TrueVal, const SDValue FalseVal,
5025 const ISD::CondCode CC, const SDValue K) {
5026 return (isGTorGE(CC) &&
5027 ((K == LHS && K == TrueVal) || (K == RHS && K == FalseVal))) ||
5028 (isLTorLE(CC) &&
5029 ((K == RHS && K == TrueVal) || (K == LHS && K == FalseVal)));
5030}
5031
5032// Check if two chained conditionals could be converted into SSAT or USAT.
5033//
5034// SSAT can replace a set of two conditional selectors that bound a number to an
5035// interval of type [k, ~k] when k + 1 is a power of 2. Here are some examples:
5036//
5037// x < -k ? -k : (x > k ? k : x)
5038// x < -k ? -k : (x < k ? x : k)
5039// x > -k ? (x > k ? k : x) : -k
5040// x < k ? (x < -k ? -k : x) : k
5041// etc.
5042//
5043// LLVM canonicalizes these to either a min(max()) or a max(min())
5044// pattern. This function tries to match one of these and will return a SSAT
5045// node if successful.
5046//
5047// USAT works similarily to SSAT but bounds on the interval [0, k] where k + 1
5048// is a power of 2.
5049static SDValue LowerSaturatingConditional(SDValue Op, SelectionDAG &DAG) {
5050 EVT VT = Op.getValueType();
5051 SDValue V1 = Op.getOperand(0);
5052 SDValue K1 = Op.getOperand(1);
5053 SDValue TrueVal1 = Op.getOperand(2);
5054 SDValue FalseVal1 = Op.getOperand(3);
5055 ISD::CondCode CC1 = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5056
5057 const SDValue Op2 = isa<ConstantSDNode>(TrueVal1) ? FalseVal1 : TrueVal1;
5058 if (Op2.getOpcode() != ISD::SELECT_CC)
5059 return SDValue();
5060
5061 SDValue V2 = Op2.getOperand(0);
5062 SDValue K2 = Op2.getOperand(1);
5063 SDValue TrueVal2 = Op2.getOperand(2);
5064 SDValue FalseVal2 = Op2.getOperand(3);
5065 ISD::CondCode CC2 = cast<CondCodeSDNode>(Op2.getOperand(4))->get();
5066
5067 SDValue V1Tmp = V1;
5068 SDValue V2Tmp = V2;
5069
5070 // Check that the registers and the constants match a max(min()) or min(max())
5071 // pattern
5072 if (V1Tmp != TrueVal1 || V2Tmp != TrueVal2 || K1 != FalseVal1 ||
5073 K2 != FalseVal2 ||
5074 !((isGTorGE(CC1) && isLTorLE(CC2)) || (isLTorLE(CC1) && isGTorGE(CC2))))
5075 return SDValue();
5076
5077 // Check that the constant in the lower-bound check is
5078 // the opposite of the constant in the upper-bound check
5079 // in 1's complement.
5080 if (!isa<ConstantSDNode>(K1) || !isa<ConstantSDNode>(K2))
5081 return SDValue();
5082
5083 int64_t Val1 = cast<ConstantSDNode>(K1)->getSExtValue();
5084 int64_t Val2 = cast<ConstantSDNode>(K2)->getSExtValue();
5085 int64_t PosVal = std::max(Val1, Val2);
5086 int64_t NegVal = std::min(Val1, Val2);
5087
5088 if (!((Val1 > Val2 && isLTorLE(CC1)) || (Val1 < Val2 && isLTorLE(CC2))) ||
5089 !isPowerOf2_64(PosVal + 1))
5090 return SDValue();
5091
5092 // Handle the difference between USAT (unsigned) and SSAT (signed)
5093 // saturation
5094 // At this point, PosVal is guaranteed to be positive
5095 uint64_t K = PosVal;
5096 SDLoc dl(Op);
5097 if (Val1 == ~Val2)
5098 return DAG.getNode(ARMISD::SSAT, dl, VT, V2Tmp,
5099 DAG.getConstant(countTrailingOnes(K), dl, VT));
5100 if (NegVal == 0)
5101 return DAG.getNode(ARMISD::USAT, dl, VT, V2Tmp,
5102 DAG.getConstant(countTrailingOnes(K), dl, VT));
5103
5104 return SDValue();
5105}
5106
5107// Check if a condition of the type x < k ? k : x can be converted into a
5108// bit operation instead of conditional moves.
5109// Currently this is allowed given:
5110// - The conditions and values match up
5111// - k is 0 or -1 (all ones)
5112// This function will not check the last condition, thats up to the caller
5113// It returns true if the transformation can be made, and in such case
5114// returns x in V, and k in SatK.
5115static bool isLowerSaturatingConditional(const SDValue &Op, SDValue &V,
5116 SDValue &SatK)
5117{
5118 SDValue LHS = Op.getOperand(0);
5119 SDValue RHS = Op.getOperand(1);
5120 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5121 SDValue TrueVal = Op.getOperand(2);
5122 SDValue FalseVal = Op.getOperand(3);
5123
5124 SDValue *K = isa<ConstantSDNode>(LHS) ? &LHS : isa<ConstantSDNode>(RHS)
5125 ? &RHS
5126 : nullptr;
5127
5128 // No constant operation in comparison, early out
5129 if (!K)
5130 return false;
5131
5132 SDValue KTmp = isa<ConstantSDNode>(TrueVal) ? TrueVal : FalseVal;
5133 V = (KTmp == TrueVal) ? FalseVal : TrueVal;
5134 SDValue VTmp = (K && *K == LHS) ? RHS : LHS;
5135
5136 // If the constant on left and right side, or variable on left and right,
5137 // does not match, early out
5138 if (*K != KTmp || V != VTmp)
5139 return false;
5140
5141 if (isLowerSaturate(LHS, RHS, TrueVal, FalseVal, CC, *K)) {
5142 SatK = *K;
5143 return true;
5144 }
5145
5146 return false;
5147}
5148
5149bool ARMTargetLowering::isUnsupportedFloatingType(EVT VT) const {
5150 if (VT == MVT::f32)
5151 return !Subtarget->hasVFP2Base();
5152 if (VT == MVT::f64)
5153 return !Subtarget->hasFP64();
5154 if (VT == MVT::f16)
5155 return !Subtarget->hasFullFP16();
5156 return false;
5157}
5158
5159SDValue ARMTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
5160 EVT VT = Op.getValueType();
5161 SDLoc dl(Op);
5162
5163 // Try to convert two saturating conditional selects into a single SSAT
5164 if ((!Subtarget->isThumb() && Subtarget->hasV6Ops()) || Subtarget->isThumb2())
5165 if (SDValue SatValue = LowerSaturatingConditional(Op, DAG))
5166 return SatValue;
5167
5168 // Try to convert expressions of the form x < k ? k : x (and similar forms)
5169 // into more efficient bit operations, which is possible when k is 0 or -1
5170 // On ARM and Thumb-2 which have flexible operand 2 this will result in
5171 // single instructions. On Thumb the shift and the bit operation will be two
5172 // instructions.
5173 // Only allow this transformation on full-width (32-bit) operations
5174 SDValue LowerSatConstant;
5175 SDValue SatValue;
5176 if (VT == MVT::i32 &&
5177 isLowerSaturatingConditional(Op, SatValue, LowerSatConstant)) {
5178 SDValue ShiftV = DAG.getNode(ISD::SRA, dl, VT, SatValue,
5179 DAG.getConstant(31, dl, VT));
5180 if (isNullConstant(LowerSatConstant)) {
5181 SDValue NotShiftV = DAG.getNode(ISD::XOR, dl, VT, ShiftV,
5182 DAG.getAllOnesConstant(dl, VT));
5183 return DAG.getNode(ISD::AND, dl, VT, SatValue, NotShiftV);
5184 } else if (isAllOnesConstant(LowerSatConstant))
5185 return DAG.getNode(ISD::OR, dl, VT, SatValue, ShiftV);
5186 }
5187
5188 SDValue LHS = Op.getOperand(0);
5189 SDValue RHS = Op.getOperand(1);
5190 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
5191 SDValue TrueVal = Op.getOperand(2);
5192 SDValue FalseVal = Op.getOperand(3);
5193 ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FalseVal);
5194 ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TrueVal);
5195
5196 if (Subtarget->hasV8_1MMainlineOps() && CFVal && CTVal &&
5197 LHS.getValueType() == MVT::i32 && RHS.getValueType() == MVT::i32) {
5198 unsigned TVal = CTVal->getZExtValue();
5199 unsigned FVal = CFVal->getZExtValue();
5200 unsigned Opcode = 0;
5201
5202 if (TVal == ~FVal) {
5203 Opcode = ARMISD::CSINV;
5204 } else if (TVal == ~FVal + 1) {
5205 Opcode = ARMISD::CSNEG;
5206 } else if (TVal + 1 == FVal) {
5207 Opcode = ARMISD::CSINC;
5208 } else if (TVal == FVal + 1) {
5209 Opcode = ARMISD::CSINC;
5210 std::swap(TrueVal, FalseVal);
5211 std::swap(TVal, FVal);
5212 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5213 }
5214
5215 if (Opcode) {
5216 // If one of the constants is cheaper than another, materialise the
5217 // cheaper one and let the csel generate the other.
5218 if (Opcode != ARMISD::CSINC &&
5219 HasLowerConstantMaterializationCost(FVal, TVal, Subtarget)) {
5220 std::swap(TrueVal, FalseVal);
5221 std::swap(TVal, FVal);
5222 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5223 }
5224
5225 // Attempt to use ZR checking TVal is 0, possibly inverting the condition
5226 // to get there. CSINC not is invertable like the other two (~(~a) == a,
5227 // -(-a) == a, but (a+1)+1 != a).
5228 if (FVal == 0 && Opcode != ARMISD::CSINC) {
5229 std::swap(TrueVal, FalseVal);
5230 std::swap(TVal, FVal);
5231 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5232 }
5233
5234 // Drops F's value because we can get it by inverting/negating TVal.
5235 FalseVal = TrueVal;
5236
5237 SDValue ARMcc;
5238 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5239 EVT VT = TrueVal.getValueType();
5240 return DAG.getNode(Opcode, dl, VT, TrueVal, FalseVal, ARMcc, Cmp);
5241 }
5242 }
5243
5244 if (isUnsupportedFloatingType(LHS.getValueType())) {
5245 DAG.getTargetLoweringInfo().softenSetCCOperands(
5246 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5247
5248 // If softenSetCCOperands only returned one value, we should compare it to
5249 // zero.
5250 if (!RHS.getNode()) {
5251 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5252 CC = ISD::SETNE;
5253 }
5254 }
5255
5256 if (LHS.getValueType() == MVT::i32) {
5257 // Try to generate VSEL on ARMv8.
5258 // The VSEL instruction can't use all the usual ARM condition
5259 // codes: it only has two bits to select the condition code, so it's
5260 // constrained to use only GE, GT, VS and EQ.
5261 //
5262 // To implement all the various ISD::SETXXX opcodes, we sometimes need to
5263 // swap the operands of the previous compare instruction (effectively
5264 // inverting the compare condition, swapping 'less' and 'greater') and
5265 // sometimes need to swap the operands to the VSEL (which inverts the
5266 // condition in the sense of firing whenever the previous condition didn't)
5267 if (Subtarget->hasFPARMv8Base() && (TrueVal.getValueType() == MVT::f16 ||
5268 TrueVal.getValueType() == MVT::f32 ||
5269 TrueVal.getValueType() == MVT::f64)) {
5270 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5271 if (CondCode == ARMCC::LT || CondCode == ARMCC::LE ||
5272 CondCode == ARMCC::VC || CondCode == ARMCC::NE) {
5273 CC = ISD::getSetCCInverse(CC, LHS.getValueType());
5274 std::swap(TrueVal, FalseVal);
5275 }
5276 }
5277
5278 SDValue ARMcc;
5279 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5280 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5281 // Choose GE over PL, which vsel does now support
5282 if (cast<ConstantSDNode>(ARMcc)->getZExtValue() == ARMCC::PL)
5283 ARMcc = DAG.getConstant(ARMCC::GE, dl, MVT::i32);
5284 return getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5285 }
5286
5287 ARMCC::CondCodes CondCode, CondCode2;
5288 FPCCToARMCC(CC, CondCode, CondCode2);
5289
5290 // Normalize the fp compare. If RHS is zero we prefer to keep it there so we
5291 // match CMPFPw0 instead of CMPFP, though we don't do this for f16 because we
5292 // must use VSEL (limited condition codes), due to not having conditional f16
5293 // moves.
5294 if (Subtarget->hasFPARMv8Base() &&
5295 !(isFloatingPointZero(RHS) && TrueVal.getValueType() != MVT::f16) &&
5296 (TrueVal.getValueType() == MVT::f16 ||
5297 TrueVal.getValueType() == MVT::f32 ||
5298 TrueVal.getValueType() == MVT::f64)) {
5299 bool swpCmpOps = false;
5300 bool swpVselOps = false;
5301 checkVSELConstraints(CC, CondCode, swpCmpOps, swpVselOps);
5302
5303 if (CondCode == ARMCC::GT || CondCode == ARMCC::GE ||
5304 CondCode == ARMCC::VS || CondCode == ARMCC::EQ) {
5305 if (swpCmpOps)
5306 std::swap(LHS, RHS);
5307 if (swpVselOps)
5308 std::swap(TrueVal, FalseVal);
5309 }
5310 }
5311
5312 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5313 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5314 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5315 SDValue Result = getCMOV(dl, VT, FalseVal, TrueVal, ARMcc, CCR, Cmp, DAG);
5316 if (CondCode2 != ARMCC::AL) {
5317 SDValue ARMcc2 = DAG.getConstant(CondCode2, dl, MVT::i32);
5318 // FIXME: Needs another CMP because flag can have but one use.
5319 SDValue Cmp2 = getVFPCmp(LHS, RHS, DAG, dl);
5320 Result = getCMOV(dl, VT, Result, TrueVal, ARMcc2, CCR, Cmp2, DAG);
5321 }
5322 return Result;
5323}
5324
5325/// canChangeToInt - Given the fp compare operand, return true if it is suitable
5326/// to morph to an integer compare sequence.
5327static bool canChangeToInt(SDValue Op, bool &SeenZero,
5328 const ARMSubtarget *Subtarget) {
5329 SDNode *N = Op.getNode();
5330 if (!N->hasOneUse())
5331 // Otherwise it requires moving the value from fp to integer registers.
5332 return false;
5333 if (!N->getNumValues())
5334 return false;
5335 EVT VT = Op.getValueType();
5336 if (VT != MVT::f32 && !Subtarget->isFPBrccSlow())
5337 // f32 case is generally profitable. f64 case only makes sense when vcmpe +
5338 // vmrs are very slow, e.g. cortex-a8.
5339 return false;
5340
5341 if (isFloatingPointZero(Op)) {
5342 SeenZero = true;
5343 return true;
5344 }
5345 return ISD::isNormalLoad(N);
5346}
5347
5348static SDValue bitcastf32Toi32(SDValue Op, SelectionDAG &DAG) {
5349 if (isFloatingPointZero(Op))
5350 return DAG.getConstant(0, SDLoc(Op), MVT::i32);
5351
5352 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op))
5353 return DAG.getLoad(MVT::i32, SDLoc(Op), Ld->getChain(), Ld->getBasePtr(),
5354 Ld->getPointerInfo(), Ld->getAlignment(),
5355 Ld->getMemOperand()->getFlags());
5356
5357 llvm_unreachable("Unknown VFP cmp argument!")::llvm::llvm_unreachable_internal("Unknown VFP cmp argument!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5357)
;
5358}
5359
5360static void expandf64Toi32(SDValue Op, SelectionDAG &DAG,
5361 SDValue &RetVal1, SDValue &RetVal2) {
5362 SDLoc dl(Op);
5363
5364 if (isFloatingPointZero(Op)) {
5365 RetVal1 = DAG.getConstant(0, dl, MVT::i32);
5366 RetVal2 = DAG.getConstant(0, dl, MVT::i32);
5367 return;
5368 }
5369
5370 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Op)) {
5371 SDValue Ptr = Ld->getBasePtr();
5372 RetVal1 =
5373 DAG.getLoad(MVT::i32, dl, Ld->getChain(), Ptr, Ld->getPointerInfo(),
5374 Ld->getAlignment(), Ld->getMemOperand()->getFlags());
5375
5376 EVT PtrType = Ptr.getValueType();
5377 unsigned NewAlign = MinAlign(Ld->getAlignment(), 4);
5378 SDValue NewPtr = DAG.getNode(ISD::ADD, dl,
5379 PtrType, Ptr, DAG.getConstant(4, dl, PtrType));
5380 RetVal2 = DAG.getLoad(MVT::i32, dl, Ld->getChain(), NewPtr,
5381 Ld->getPointerInfo().getWithOffset(4), NewAlign,
5382 Ld->getMemOperand()->getFlags());
5383 return;
5384 }
5385
5386 llvm_unreachable("Unknown VFP cmp argument!")::llvm::llvm_unreachable_internal("Unknown VFP cmp argument!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5386)
;
5387}
5388
5389/// OptimizeVFPBrcond - With -enable-unsafe-fp-math, it's legal to optimize some
5390/// f32 and even f64 comparisons to integer ones.
5391SDValue
5392ARMTargetLowering::OptimizeVFPBrcond(SDValue Op, SelectionDAG &DAG) const {
5393 SDValue Chain = Op.getOperand(0);
5394 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
5395 SDValue LHS = Op.getOperand(2);
5396 SDValue RHS = Op.getOperand(3);
5397 SDValue Dest = Op.getOperand(4);
5398 SDLoc dl(Op);
5399
5400 bool LHSSeenZero = false;
5401 bool LHSOk = canChangeToInt(LHS, LHSSeenZero, Subtarget);
5402 bool RHSSeenZero = false;
5403 bool RHSOk = canChangeToInt(RHS, RHSSeenZero, Subtarget);
5404 if (LHSOk && RHSOk && (LHSSeenZero || RHSSeenZero)) {
5405 // If unsafe fp math optimization is enabled and there are no other uses of
5406 // the CMP operands, and the condition code is EQ or NE, we can optimize it
5407 // to an integer comparison.
5408 if (CC == ISD::SETOEQ)
5409 CC = ISD::SETEQ;
5410 else if (CC == ISD::SETUNE)
5411 CC = ISD::SETNE;
5412
5413 SDValue Mask = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5414 SDValue ARMcc;
5415 if (LHS.getValueType() == MVT::f32) {
5416 LHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5417 bitcastf32Toi32(LHS, DAG), Mask);
5418 RHS = DAG.getNode(ISD::AND, dl, MVT::i32,
5419 bitcastf32Toi32(RHS, DAG), Mask);
5420 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5421 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5422 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
5423 Chain, Dest, ARMcc, CCR, Cmp);
5424 }
5425
5426 SDValue LHS1, LHS2;
5427 SDValue RHS1, RHS2;
5428 expandf64Toi32(LHS, DAG, LHS1, LHS2);
5429 expandf64Toi32(RHS, DAG, RHS1, RHS2);
5430 LHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, LHS2, Mask);
5431 RHS2 = DAG.getNode(ISD::AND, dl, MVT::i32, RHS2, Mask);
5432 ARMCC::CondCodes CondCode = IntCCToARMCC(CC);
5433 ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5434 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
5435 SDValue Ops[] = { Chain, ARMcc, LHS1, LHS2, RHS1, RHS2, Dest };
5436 return DAG.getNode(ARMISD::BCC_i64, dl, VTList, Ops);
5437 }
5438
5439 return SDValue();
5440}
5441
5442SDValue ARMTargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
5443 SDValue Chain = Op.getOperand(0);
5444 SDValue Cond = Op.getOperand(1);
5445 SDValue Dest = Op.getOperand(2);
5446 SDLoc dl(Op);
5447
5448 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5449 // instruction.
5450 unsigned Opc = Cond.getOpcode();
5451 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5452 !Subtarget->isThumb1Only();
5453 if (Cond.getResNo() == 1 &&
5454 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5455 Opc == ISD::USUBO || OptimizeMul)) {
5456 // Only lower legal XALUO ops.
5457 if (!DAG.getTargetLoweringInfo().isTypeLegal(Cond->getValueType(0)))
5458 return SDValue();
5459
5460 // The actual operation with overflow check.
5461 SDValue Value, OverflowCmp;
5462 SDValue ARMcc;
5463 std::tie(Value, OverflowCmp) = getARMXALUOOp(Cond, DAG, ARMcc);
5464
5465 // Reverse the condition code.
5466 ARMCC::CondCodes CondCode =
5467 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
5468 CondCode = ARMCC::getOppositeCondition(CondCode);
5469 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5470 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5471
5472 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
5473 OverflowCmp);
5474 }
5475
5476 return SDValue();
5477}
5478
5479SDValue ARMTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
5480 SDValue Chain = Op.getOperand(0);
5481 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
5482 SDValue LHS = Op.getOperand(2);
5483 SDValue RHS = Op.getOperand(3);
5484 SDValue Dest = Op.getOperand(4);
5485 SDLoc dl(Op);
5486
5487 if (isUnsupportedFloatingType(LHS.getValueType())) {
5488 DAG.getTargetLoweringInfo().softenSetCCOperands(
5489 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS);
5490
5491 // If softenSetCCOperands only returned one value, we should compare it to
5492 // zero.
5493 if (!RHS.getNode()) {
5494 RHS = DAG.getConstant(0, dl, LHS.getValueType());
5495 CC = ISD::SETNE;
5496 }
5497 }
5498
5499 // Optimize {s|u}{add|sub|mul}.with.overflow feeding into a branch
5500 // instruction.
5501 unsigned Opc = LHS.getOpcode();
5502 bool OptimizeMul = (Opc == ISD::SMULO || Opc == ISD::UMULO) &&
5503 !Subtarget->isThumb1Only();
5504 if (LHS.getResNo() == 1 && (isOneConstant(RHS) || isNullConstant(RHS)) &&
5505 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
5506 Opc == ISD::USUBO || OptimizeMul) &&
5507 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
5508 // Only lower legal XALUO ops.
5509 if (!DAG.getTargetLoweringInfo().isTypeLegal(LHS->getValueType(0)))
5510 return SDValue();
5511
5512 // The actual operation with overflow check.
5513 SDValue Value, OverflowCmp;
5514 SDValue ARMcc;
5515 std::tie(Value, OverflowCmp) = getARMXALUOOp(LHS.getValue(0), DAG, ARMcc);
5516
5517 if ((CC == ISD::SETNE) != isOneConstant(RHS)) {
5518 // Reverse the condition code.
5519 ARMCC::CondCodes CondCode =
5520 (ARMCC::CondCodes)cast<const ConstantSDNode>(ARMcc)->getZExtValue();
5521 CondCode = ARMCC::getOppositeCondition(CondCode);
5522 ARMcc = DAG.getConstant(CondCode, SDLoc(ARMcc), MVT::i32);
5523 }
5524 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5525
5526 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other, Chain, Dest, ARMcc, CCR,
5527 OverflowCmp);
5528 }
5529
5530 if (LHS.getValueType() == MVT::i32) {
5531 SDValue ARMcc;
5532 SDValue Cmp = getARMCmp(LHS, RHS, CC, ARMcc, DAG, dl);
5533 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5534 return DAG.getNode(ARMISD::BRCOND, dl, MVT::Other,
5535 Chain, Dest, ARMcc, CCR, Cmp);
5536 }
5537
5538 if (getTargetMachine().Options.UnsafeFPMath &&
5539 (CC == ISD::SETEQ || CC == ISD::SETOEQ ||
5540 CC == ISD::SETNE || CC == ISD::SETUNE)) {
5541 if (SDValue Result = OptimizeVFPBrcond(Op, DAG))
5542 return Result;
5543 }
5544
5545 ARMCC::CondCodes CondCode, CondCode2;
5546 FPCCToARMCC(CC, CondCode, CondCode2);
5547
5548 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
5549 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl);
5550 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
5551 SDVTList VTList = DAG.getVTList(MVT::Other, MVT::Glue);
5552 SDValue Ops[] = { Chain, Dest, ARMcc, CCR, Cmp };
5553 SDValue Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5554 if (CondCode2 != ARMCC::AL) {
5555 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
5556 SDValue Ops[] = { Res, Dest, ARMcc, CCR, Res.getValue(1) };
5557 Res = DAG.getNode(ARMISD::BRCOND, dl, VTList, Ops);
5558 }
5559 return Res;
5560}
5561
5562SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
5563 SDValue Chain = Op.getOperand(0);
5564 SDValue Table = Op.getOperand(1);
5565 SDValue Index = Op.getOperand(2);
5566 SDLoc dl(Op);
5567
5568 EVT PTy = getPointerTy(DAG.getDataLayout());
5569 JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
5570 SDValue JTI = DAG.getTargetJumpTable(JT->getIndex(), PTy);
5571 Table = DAG.getNode(ARMISD::WrapperJT, dl, MVT::i32, JTI);
5572 Index = DAG.getNode(ISD::MUL, dl, PTy, Index, DAG.getConstant(4, dl, PTy));
5573 SDValue Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Index);
5574 if (Subtarget->isThumb2() || (Subtarget->hasV8MBaselineOps() && Subtarget->isThumb())) {
5575 // Thumb2 and ARMv8-M use a two-level jump. That is, it jumps into the jump table
5576 // which does another jump to the destination. This also makes it easier
5577 // to translate it to TBB / TBH later (Thumb2 only).
5578 // FIXME: This might not work if the function is extremely large.
5579 return DAG.getNode(ARMISD::BR2_JT, dl, MVT::Other, Chain,
5580 Addr, Op.getOperand(2), JTI);
5581 }
5582 if (isPositionIndependent() || Subtarget->isROPI()) {
5583 Addr =
5584 DAG.getLoad((EVT)MVT::i32, dl, Chain, Addr,
5585 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5586 Chain = Addr.getValue(1);
5587 Addr = DAG.getNode(ISD::ADD, dl, PTy, Table, Addr);
5588 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5589 } else {
5590 Addr =
5591 DAG.getLoad(PTy, dl, Chain, Addr,
5592 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()));
5593 Chain = Addr.getValue(1);
5594 return DAG.getNode(ARMISD::BR_JT, dl, MVT::Other, Chain, Addr, JTI);
5595 }
5596}
5597
5598static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
5599 EVT VT = Op.getValueType();
5600 SDLoc dl(Op);
5601
5602 if (Op.getValueType().getVectorElementType() == MVT::i32) {
5603 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::f32)
5604 return Op;
5605 return DAG.UnrollVectorOp(Op.getNode());
5606 }
5607
5608 const bool HasFullFP16 =
5609 static_cast<const ARMSubtarget&>(DAG.getSubtarget()).hasFullFP16();
5610
5611 EVT NewTy;
5612 const EVT OpTy = Op.getOperand(0).getValueType();
5613 if (OpTy == MVT::v4f32)
5614 NewTy = MVT::v4i32;
5615 else if (OpTy == MVT::v4f16 && HasFullFP16)
5616 NewTy = MVT::v4i16;
5617 else if (OpTy == MVT::v8f16 && HasFullFP16)
5618 NewTy = MVT::v8i16;
5619 else
5620 llvm_unreachable("Invalid type for custom lowering!")::llvm::llvm_unreachable_internal("Invalid type for custom lowering!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5620)
;
5621
5622 if (VT != MVT::v4i16 && VT != MVT::v8i16)
5623 return DAG.UnrollVectorOp(Op.getNode());
5624
5625 Op = DAG.getNode(Op.getOpcode(), dl, NewTy, Op.getOperand(0));
5626 return DAG.getNode(ISD::TRUNCATE, dl, VT, Op);
5627}
5628
5629SDValue ARMTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
5630 EVT VT = Op.getValueType();
5631 if (VT.isVector())
5632 return LowerVectorFP_TO_INT(Op, DAG);
5633
5634 bool IsStrict = Op->isStrictFPOpcode();
5635 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
5636
5637 if (isUnsupportedFloatingType(SrcVal.getValueType())) {
5638 RTLIB::Libcall LC;
5639 if (Op.getOpcode() == ISD::FP_TO_SINT ||
5640 Op.getOpcode() == ISD::STRICT_FP_TO_SINT)
5641 LC = RTLIB::getFPTOSINT(SrcVal.getValueType(),
5642 Op.getValueType());
5643 else
5644 LC = RTLIB::getFPTOUINT(SrcVal.getValueType(),
5645 Op.getValueType());
5646 SDLoc Loc(Op);
5647 MakeLibCallOptions CallOptions;
5648 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
5649 SDValue Result;
5650 std::tie(Result, Chain) = makeLibCall(DAG, LC, Op.getValueType(), SrcVal,
5651 CallOptions, Loc, Chain);
5652 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
5653 }
5654
5655 // FIXME: Remove this when we have strict fp instruction selection patterns
5656 if (IsStrict) {
5657 SDLoc Loc(Op);
5658 SDValue Result =
5659 DAG.getNode(Op.getOpcode() == ISD::STRICT_FP_TO_SINT ? ISD::FP_TO_SINT
5660 : ISD::FP_TO_UINT,
5661 Loc, Op.getValueType(), SrcVal);
5662 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
5663 }
5664
5665 return Op;
5666}
5667
5668static SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
5669 EVT VT = Op.getValueType();
5670 SDLoc dl(Op);
5671
5672 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i32) {
5673 if (VT.getVectorElementType() == MVT::f32)
5674 return Op;
5675 return DAG.UnrollVectorOp(Op.getNode());
5676 }
5677
5678 assert((Op.getOperand(0).getValueType() == MVT::v4i16 ||(static_cast <bool> ((Op.getOperand(0).getValueType() ==
MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16)
&& "Invalid type for custom lowering!") ? void (0) :
__assert_fail ("(Op.getOperand(0).getValueType() == MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16) && \"Invalid type for custom lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5680, __extension__ __PRETTY_FUNCTION__))
5679 Op.getOperand(0).getValueType() == MVT::v8i16) &&(static_cast <bool> ((Op.getOperand(0).getValueType() ==
MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16)
&& "Invalid type for custom lowering!") ? void (0) :
__assert_fail ("(Op.getOperand(0).getValueType() == MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16) && \"Invalid type for custom lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5680, __extension__ __PRETTY_FUNCTION__))
5680 "Invalid type for custom lowering!")(static_cast <bool> ((Op.getOperand(0).getValueType() ==
MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16)
&& "Invalid type for custom lowering!") ? void (0) :
__assert_fail ("(Op.getOperand(0).getValueType() == MVT::v4i16 || Op.getOperand(0).getValueType() == MVT::v8i16) && \"Invalid type for custom lowering!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5680, __extension__ __PRETTY_FUNCTION__))
;
5681
5682 const bool HasFullFP16 =
5683 static_cast<const ARMSubtarget&>(DAG.getSubtarget()).hasFullFP16();
5684
5685 EVT DestVecType;
5686 if (VT == MVT::v4f32)
5687 DestVecType = MVT::v4i32;
5688 else if (VT == MVT::v4f16 && HasFullFP16)
5689 DestVecType = MVT::v4i16;
5690 else if (VT == MVT::v8f16 && HasFullFP16)
5691 DestVecType = MVT::v8i16;
5692 else
5693 return DAG.UnrollVectorOp(Op.getNode());
5694
5695 unsigned CastOpc;
5696 unsigned Opc;
5697 switch (Op.getOpcode()) {
5698 default: llvm_unreachable("Invalid opcode!")::llvm::llvm_unreachable_internal("Invalid opcode!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5698)
;
5699 case ISD::SINT_TO_FP:
5700 CastOpc = ISD::SIGN_EXTEND;
5701 Opc = ISD::SINT_TO_FP;
5702 break;
5703 case ISD::UINT_TO_FP:
5704 CastOpc = ISD::ZERO_EXTEND;
5705 Opc = ISD::UINT_TO_FP;
5706 break;
5707 }
5708
5709 Op = DAG.getNode(CastOpc, dl, DestVecType, Op.getOperand(0));
5710 return DAG.getNode(Opc, dl, VT, Op);
5711}
5712
5713SDValue ARMTargetLowering::LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const {
5714 EVT VT = Op.getValueType();
5715 if (VT.isVector())
5716 return LowerVectorINT_TO_FP(Op, DAG);
5717 if (isUnsupportedFloatingType(VT)) {
5718 RTLIB::Libcall LC;
5719 if (Op.getOpcode() == ISD::SINT_TO_FP)
5720 LC = RTLIB::getSINTTOFP(Op.getOperand(0).getValueType(),
5721 Op.getValueType());
5722 else
5723 LC = RTLIB::getUINTTOFP(Op.getOperand(0).getValueType(),
5724 Op.getValueType());
5725 MakeLibCallOptions CallOptions;
5726 return makeLibCall(DAG, LC, Op.getValueType(), Op.getOperand(0),
5727 CallOptions, SDLoc(Op)).first;
5728 }
5729
5730 return Op;
5731}
5732
5733SDValue ARMTargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
5734 // Implement fcopysign with a fabs and a conditional fneg.
5735 SDValue Tmp0 = Op.getOperand(0);
5736 SDValue Tmp1 = Op.getOperand(1);
5737 SDLoc dl(Op);
5738 EVT VT = Op.getValueType();
5739 EVT SrcVT = Tmp1.getValueType();
5740 bool InGPR = Tmp0.getOpcode() == ISD::BITCAST ||
5741 Tmp0.getOpcode() == ARMISD::VMOVDRR;
5742 bool UseNEON = !InGPR && Subtarget->hasNEON();
5743
5744 if (UseNEON) {
5745 // Use VBSL to copy the sign bit.
5746 unsigned EncodedVal = ARM_AM::createVMOVModImm(0x6, 0x80);
5747 SDValue Mask = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v2i32,
5748 DAG.getTargetConstant(EncodedVal, dl, MVT::i32));
5749 EVT OpVT = (VT == MVT::f32) ? MVT::v2i32 : MVT::v1i64;
5750 if (VT == MVT::f64)
5751 Mask = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
5752 DAG.getNode(ISD::BITCAST, dl, OpVT, Mask),
5753 DAG.getConstant(32, dl, MVT::i32));
5754 else /*if (VT == MVT::f32)*/
5755 Tmp0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp0);
5756 if (SrcVT == MVT::f32) {
5757 Tmp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f32, Tmp1);
5758 if (VT == MVT::f64)
5759 Tmp1 = DAG.getNode(ARMISD::VSHLIMM, dl, OpVT,
5760 DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1),
5761 DAG.getConstant(32, dl, MVT::i32));
5762 } else if (VT == MVT::f32)
5763 Tmp1 = DAG.getNode(ARMISD::VSHRuIMM, dl, MVT::v1i64,
5764 DAG.getNode(ISD::BITCAST, dl, MVT::v1i64, Tmp1),
5765 DAG.getConstant(32, dl, MVT::i32));
5766 Tmp0 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp0);
5767 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OpVT, Tmp1);
5768
5769 SDValue AllOnes = DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff),
5770 dl, MVT::i32);
5771 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v8i8, AllOnes);
5772 SDValue MaskNot = DAG.getNode(ISD::XOR, dl, OpVT, Mask,
5773 DAG.getNode(ISD::BITCAST, dl, OpVT, AllOnes));
5774
5775 SDValue Res = DAG.getNode(ISD::OR, dl, OpVT,
5776 DAG.getNode(ISD::AND, dl, OpVT, Tmp1, Mask),
5777 DAG.getNode(ISD::AND, dl, OpVT, Tmp0, MaskNot));
5778 if (VT == MVT::f32) {
5779 Res = DAG.getNode(ISD::BITCAST, dl, MVT::v2f32, Res);
5780 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, Res,
5781 DAG.getConstant(0, dl, MVT::i32));
5782 } else {
5783 Res = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Res);
5784 }
5785
5786 return Res;
5787 }
5788
5789 // Bitcast operand 1 to i32.
5790 if (SrcVT == MVT::f64)
5791 Tmp1 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
5792 Tmp1).getValue(1);
5793 Tmp1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp1);
5794
5795 // Or in the signbit with integer operations.
5796 SDValue Mask1 = DAG.getConstant(0x80000000, dl, MVT::i32);
5797 SDValue Mask2 = DAG.getConstant(0x7fffffff, dl, MVT::i32);
5798 Tmp1 = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp1, Mask1);
5799 if (VT == MVT::f32) {
5800 Tmp0 = DAG.getNode(ISD::AND, dl, MVT::i32,
5801 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Tmp0), Mask2);
5802 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
5803 DAG.getNode(ISD::OR, dl, MVT::i32, Tmp0, Tmp1));
5804 }
5805
5806 // f64: Or the high part with signbit and then combine two parts.
5807 Tmp0 = DAG.getNode(ARMISD::VMOVRRD, dl, DAG.getVTList(MVT::i32, MVT::i32),
5808 Tmp0);
5809 SDValue Lo = Tmp0.getValue(0);
5810 SDValue Hi = DAG.getNode(ISD::AND, dl, MVT::i32, Tmp0.getValue(1), Mask2);
5811 Hi = DAG.getNode(ISD::OR, dl, MVT::i32, Hi, Tmp1);
5812 return DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi);
5813}
5814
5815SDValue ARMTargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const{
5816 MachineFunction &MF = DAG.getMachineFunction();
5817 MachineFrameInfo &MFI = MF.getFrameInfo();
5818 MFI.setReturnAddressIsTaken(true);
5819
5820 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
5821 return SDValue();
5822
5823 EVT VT = Op.getValueType();
5824 SDLoc dl(Op);
5825 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5826 if (Depth) {
5827 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
5828 SDValue Offset = DAG.getConstant(4, dl, MVT::i32);
5829 return DAG.getLoad(VT, dl, DAG.getEntryNode(),
5830 DAG.getNode(ISD::ADD, dl, VT, FrameAddr, Offset),
5831 MachinePointerInfo());
5832 }
5833
5834 // Return LR, which contains the return address. Mark it an implicit live-in.
5835 unsigned Reg = MF.addLiveIn(ARM::LR, getRegClassFor(MVT::i32));
5836 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
5837}
5838
5839SDValue ARMTargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
5840 const ARMBaseRegisterInfo &ARI =
5841 *static_cast<const ARMBaseRegisterInfo*>(RegInfo);
5842 MachineFunction &MF = DAG.getMachineFunction();
5843 MachineFrameInfo &MFI = MF.getFrameInfo();
5844 MFI.setFrameAddressIsTaken(true);
5845
5846 EVT VT = Op.getValueType();
5847 SDLoc dl(Op); // FIXME probably not meaningful
5848 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
5849 Register FrameReg = ARI.getFrameRegister(MF);
5850 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
5851 while (Depth--)
5852 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
5853 MachinePointerInfo());
5854 return FrameAddr;
5855}
5856
5857// FIXME? Maybe this could be a TableGen attribute on some registers and
5858// this table could be generated automatically from RegInfo.
5859Register ARMTargetLowering::getRegisterByName(const char* RegName, LLT VT,
5860 const MachineFunction &MF) const {
5861 Register Reg = StringSwitch<unsigned>(RegName)
5862 .Case("sp", ARM::SP)
5863 .Default(0);
5864 if (Reg)
5865 return Reg;
5866 report_fatal_error(Twine("Invalid register name \""
5867 + StringRef(RegName) + "\"."));
5868}
5869
5870// Result is 64 bit value so split into two 32 bit values and return as a
5871// pair of values.
5872static void ExpandREAD_REGISTER(SDNode *N, SmallVectorImpl<SDValue> &Results,
5873 SelectionDAG &DAG) {
5874 SDLoc DL(N);
5875
5876 // This function is only supposed to be called for i64 type destination.
5877 assert(N->getValueType(0) == MVT::i64(static_cast <bool> (N->getValueType(0) == MVT::i64 &&
"ExpandREAD_REGISTER called for non-i64 type result.") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"ExpandREAD_REGISTER called for non-i64 type result.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5878, __extension__ __PRETTY_FUNCTION__))
5878 && "ExpandREAD_REGISTER called for non-i64 type result.")(static_cast <bool> (N->getValueType(0) == MVT::i64 &&
"ExpandREAD_REGISTER called for non-i64 type result.") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"ExpandREAD_REGISTER called for non-i64 type result.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 5878, __extension__ __PRETTY_FUNCTION__))
;
5879
5880 SDValue Read = DAG.getNode(ISD::READ_REGISTER, DL,
5881 DAG.getVTList(MVT::i32, MVT::i32, MVT::Other),
5882 N->getOperand(0),
5883 N->getOperand(1));
5884
5885 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Read.getValue(0),
5886 Read.getValue(1)));
5887 Results.push_back(Read.getOperand(0));
5888}
5889
5890/// \p BC is a bitcast that is about to be turned into a VMOVDRR.
5891/// When \p DstVT, the destination type of \p BC, is on the vector
5892/// register bank and the source of bitcast, \p Op, operates on the same bank,
5893/// it might be possible to combine them, such that everything stays on the
5894/// vector register bank.
5895/// \p return The node that would replace \p BT, if the combine
5896/// is possible.
5897static SDValue CombineVMOVDRRCandidateWithVecOp(const SDNode *BC,
5898 SelectionDAG &DAG) {
5899 SDValue Op = BC->getOperand(0);
5900 EVT DstVT = BC->getValueType(0);
5901
5902 // The only vector instruction that can produce a scalar (remember,
5903 // since the bitcast was about to be turned into VMOVDRR, the source
5904 // type is i64) from a vector is EXTRACT_VECTOR_ELT.
5905 // Moreover, we can do this combine only if there is one use.
5906 // Finally, if the destination type is not a vector, there is not
5907 // much point on forcing everything on the vector bank.
5908 if (!DstVT.isVector() || Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5909 !Op.hasOneUse())
5910 return SDValue();
5911
5912 // If the index is not constant, we will introduce an additional
5913 // multiply that will stick.
5914 // Give up in that case.
5915 ConstantSDNode *Index = dyn_cast<ConstantSDNode>(Op.getOperand(1));
5916 if (!Index)
5917 return SDValue();
5918 unsigned DstNumElt = DstVT.getVectorNumElements();
5919
5920 // Compute the new index.
5921 const APInt &APIntIndex = Index->getAPIntValue();
5922 APInt NewIndex(APIntIndex.getBitWidth(), DstNumElt);
5923 NewIndex *= APIntIndex;
5924 // Check if the new constant index fits into i32.
5925 if (NewIndex.getBitWidth() > 32)
5926 return SDValue();
5927
5928 // vMTy bitcast(i64 extractelt vNi64 src, i32 index) ->
5929 // vMTy extractsubvector vNxMTy (bitcast vNi64 src), i32 index*M)
5930 SDLoc dl(Op);
5931 SDValue ExtractSrc = Op.getOperand(0);
5932 EVT VecVT = EVT::getVectorVT(
5933 *DAG.getContext(), DstVT.getScalarType(),
5934 ExtractSrc.getValueType().getVectorNumElements() * DstNumElt);
5935 SDValue BitCast = DAG.getNode(ISD::BITCAST, dl, VecVT, ExtractSrc);
5936 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DstVT, BitCast,
5937 DAG.getConstant(NewIndex.getZExtValue(), dl, MVT::i32));
5938}
5939
5940/// ExpandBITCAST - If the target supports VFP, this function is called to
5941/// expand a bit convert where either the source or destination type is i64 to
5942/// use a VMOVDRR or VMOVRRD node. This should not be done when the non-i64
5943/// operand type is illegal (e.g., v2f32 for a target that doesn't support
5944/// vectors), since the legalizer won't know what to do with that.
5945SDValue ARMTargetLowering::ExpandBITCAST(SDNode *N, SelectionDAG &DAG,
5946 const ARMSubtarget *Subtarget) const {
5947 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5948 SDLoc dl(N);
5949 SDValue Op = N->getOperand(0);
5950
5951 // This function is only supposed to be called for i16 and i64 types, either
5952 // as the source or destination of the bit convert.
5953 EVT SrcVT = Op.getValueType();
5954 EVT DstVT = N->getValueType(0);
5955
5956 if ((SrcVT == MVT::i16 || SrcVT == MVT::i32) &&
5957 (DstVT == MVT::f16 || DstVT == MVT::bf16))
5958 return MoveToHPR(SDLoc(N), DAG, MVT::i32, DstVT.getSimpleVT(),
5959 DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), MVT::i32, Op));
5960
5961 if ((DstVT == MVT::i16 || DstVT == MVT::i32) &&
5962 (SrcVT == MVT::f16 || SrcVT == MVT::bf16))
5963 return DAG.getNode(
5964 ISD::TRUNCATE, SDLoc(N), DstVT,
5965 MoveFromHPR(SDLoc(N), DAG, MVT::i32, SrcVT.getSimpleVT(), Op));
5966
5967 if (!(SrcVT == MVT::i64 || DstVT == MVT::i64))
5968 return SDValue();
5969
5970 // Turn i64->f64 into VMOVDRR.
5971 if (SrcVT == MVT::i64 && TLI.isTypeLegal(DstVT)) {
5972 // Do not force values to GPRs (this is what VMOVDRR does for the inputs)
5973 // if we can combine the bitcast with its source.
5974 if (SDValue Val = CombineVMOVDRRCandidateWithVecOp(N, DAG))
5975 return Val;
5976
5977 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
5978 DAG.getConstant(0, dl, MVT::i32));
5979 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Op,
5980 DAG.getConstant(1, dl, MVT::i32));
5981 return DAG.getNode(ISD::BITCAST, dl, DstVT,
5982 DAG.getNode(ARMISD::VMOVDRR, dl, MVT::f64, Lo, Hi));
5983 }
5984
5985 // Turn f64->i64 into VMOVRRD.
5986 if (DstVT == MVT::i64 && TLI.isTypeLegal(SrcVT)) {
5987 SDValue Cvt;
5988 if (DAG.getDataLayout().isBigEndian() && SrcVT.isVector() &&
5989 SrcVT.getVectorNumElements() > 1)
5990 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
5991 DAG.getVTList(MVT::i32, MVT::i32),
5992 DAG.getNode(ARMISD::VREV64, dl, SrcVT, Op));
5993 else
5994 Cvt = DAG.getNode(ARMISD::VMOVRRD, dl,
5995 DAG.getVTList(MVT::i32, MVT::i32), Op);
5996 // Merge the pieces into a single i64 value.
5997 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Cvt, Cvt.getValue(1));
5998 }
5999
6000 return SDValue();
6001}
6002
6003/// getZeroVector - Returns a vector of specified type with all zero elements.
6004/// Zero vectors are used to represent vector negation and in those cases
6005/// will be implemented with the NEON VNEG instruction. However, VNEG does
6006/// not support i64 elements, so sometimes the zero vectors will need to be
6007/// explicitly constructed. Regardless, use a canonical VMOV to create the
6008/// zero vector.
6009static SDValue getZeroVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
6010 assert(VT.isVector() && "Expected a vector type")(static_cast <bool> (VT.isVector() && "Expected a vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"Expected a vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6010, __extension__ __PRETTY_FUNCTION__))
;
6011 // The canonical modified immediate encoding of a zero vector is....0!
6012 SDValue EncodedVal = DAG.getTargetConstant(0, dl, MVT::i32);
6013 EVT VmovVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
6014 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, EncodedVal);
6015 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
6016}
6017
6018/// LowerShiftRightParts - Lower SRA_PARTS, which returns two
6019/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6020SDValue ARMTargetLowering::LowerShiftRightParts(SDValue Op,
6021 SelectionDAG &DAG) const {
6022 assert(Op.getNumOperands() == 3 && "Not a double-shift!")(static_cast <bool> (Op.getNumOperands() == 3 &&
"Not a double-shift!") ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6022, __extension__ __PRETTY_FUNCTION__))
;
6023 EVT VT = Op.getValueType();
6024 unsigned VTBits = VT.getSizeInBits();
6025 SDLoc dl(Op);
6026 SDValue ShOpLo = Op.getOperand(0);
6027 SDValue ShOpHi = Op.getOperand(1);
6028 SDValue ShAmt = Op.getOperand(2);
6029 SDValue ARMcc;
6030 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6031 unsigned Opc = (Op.getOpcode() == ISD::SRA_PARTS) ? ISD::SRA : ISD::SRL;
6032
6033 assert(Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS)(static_cast <bool> (Op.getOpcode() == ISD::SRA_PARTS ||
Op.getOpcode() == ISD::SRL_PARTS) ? void (0) : __assert_fail
("Op.getOpcode() == ISD::SRA_PARTS || Op.getOpcode() == ISD::SRL_PARTS"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6033, __extension__ __PRETTY_FUNCTION__))
;
6034
6035 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6036 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6037 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, ShAmt);
6038 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6039 DAG.getConstant(VTBits, dl, MVT::i32));
6040 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, RevShAmt);
6041 SDValue LoSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6042 SDValue LoBigShift = DAG.getNode(Opc, dl, VT, ShOpHi, ExtraShAmt);
6043 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6044 ISD::SETGE, ARMcc, DAG, dl);
6045 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift, LoBigShift,
6046 ARMcc, CCR, CmpLo);
6047
6048 SDValue HiSmallShift = DAG.getNode(Opc, dl, VT, ShOpHi, ShAmt);
6049 SDValue HiBigShift = Opc == ISD::SRA
6050 ? DAG.getNode(Opc, dl, VT, ShOpHi,
6051 DAG.getConstant(VTBits - 1, dl, VT))
6052 : DAG.getConstant(0, dl, VT);
6053 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6054 ISD::SETGE, ARMcc, DAG, dl);
6055 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6056 ARMcc, CCR, CmpHi);
6057
6058 SDValue Ops[2] = { Lo, Hi };
6059 return DAG.getMergeValues(Ops, dl);
6060}
6061
6062/// LowerShiftLeftParts - Lower SHL_PARTS, which returns two
6063/// i32 values and take a 2 x i32 value to shift plus a shift amount.
6064SDValue ARMTargetLowering::LowerShiftLeftParts(SDValue Op,
6065 SelectionDAG &DAG) const {
6066 assert(Op.getNumOperands() == 3 && "Not a double-shift!")(static_cast <bool> (Op.getNumOperands() == 3 &&
"Not a double-shift!") ? void (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6066, __extension__ __PRETTY_FUNCTION__))
;
6067 EVT VT = Op.getValueType();
6068 unsigned VTBits = VT.getSizeInBits();
6069 SDLoc dl(Op);
6070 SDValue ShOpLo = Op.getOperand(0);
6071 SDValue ShOpHi = Op.getOperand(1);
6072 SDValue ShAmt = Op.getOperand(2);
6073 SDValue ARMcc;
6074 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6075
6076 assert(Op.getOpcode() == ISD::SHL_PARTS)(static_cast <bool> (Op.getOpcode() == ISD::SHL_PARTS) ?
void (0) : __assert_fail ("Op.getOpcode() == ISD::SHL_PARTS"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6076, __extension__ __PRETTY_FUNCTION__))
;
6077 SDValue RevShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6078 DAG.getConstant(VTBits, dl, MVT::i32), ShAmt);
6079 SDValue Tmp1 = DAG.getNode(ISD::SRL, dl, VT, ShOpLo, RevShAmt);
6080 SDValue Tmp2 = DAG.getNode(ISD::SHL, dl, VT, ShOpHi, ShAmt);
6081 SDValue HiSmallShift = DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2);
6082
6083 SDValue ExtraShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32, ShAmt,
6084 DAG.getConstant(VTBits, dl, MVT::i32));
6085 SDValue HiBigShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ExtraShAmt);
6086 SDValue CmpHi = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6087 ISD::SETGE, ARMcc, DAG, dl);
6088 SDValue Hi = DAG.getNode(ARMISD::CMOV, dl, VT, HiSmallShift, HiBigShift,
6089 ARMcc, CCR, CmpHi);
6090
6091 SDValue CmpLo = getARMCmp(ExtraShAmt, DAG.getConstant(0, dl, MVT::i32),
6092 ISD::SETGE, ARMcc, DAG, dl);
6093 SDValue LoSmallShift = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, ShAmt);
6094 SDValue Lo = DAG.getNode(ARMISD::CMOV, dl, VT, LoSmallShift,
6095 DAG.getConstant(0, dl, VT), ARMcc, CCR, CmpLo);
6096
6097 SDValue Ops[2] = { Lo, Hi };
6098 return DAG.getMergeValues(Ops, dl);
6099}
6100
6101SDValue ARMTargetLowering::LowerFLT_ROUNDS_(SDValue Op,
6102 SelectionDAG &DAG) const {
6103 // The rounding mode is in bits 23:22 of the FPSCR.
6104 // The ARM rounding mode value to FLT_ROUNDS mapping is 0->1, 1->2, 2->3, 3->0
6105 // The formula we use to implement this is (((FPSCR + 1 << 22) >> 22) & 3)
6106 // so that the shift + and get folded into a bitfield extract.
6107 SDLoc dl(Op);
6108 SDValue Chain = Op.getOperand(0);
6109 SDValue Ops[] = {Chain,
6110 DAG.getConstant(Intrinsic::arm_get_fpscr, dl, MVT::i32)};
6111
6112 SDValue FPSCR =
6113 DAG.getNode(ISD::INTRINSIC_W_CHAIN, dl, {MVT::i32, MVT::Other}, Ops);
6114 Chain = FPSCR.getValue(1);
6115 SDValue FltRounds = DAG.getNode(ISD::ADD, dl, MVT::i32, FPSCR,
6116 DAG.getConstant(1U << 22, dl, MVT::i32));
6117 SDValue RMODE = DAG.getNode(ISD::SRL, dl, MVT::i32, FltRounds,
6118 DAG.getConstant(22, dl, MVT::i32));
6119 SDValue And = DAG.getNode(ISD::AND, dl, MVT::i32, RMODE,
6120 DAG.getConstant(3, dl, MVT::i32));
6121 return DAG.getMergeValues({And, Chain}, dl);
6122}
6123
6124SDValue ARMTargetLowering::LowerSET_ROUNDING(SDValue Op,
6125 SelectionDAG &DAG) const {
6126 SDLoc DL(Op);
6127 SDValue Chain = Op->getOperand(0);
6128 SDValue RMValue = Op->getOperand(1);
6129
6130 // The rounding mode is in bits 23:22 of the FPSCR.
6131 // The llvm.set.rounding argument value to ARM rounding mode value mapping
6132 // is 0->3, 1->0, 2->1, 3->2. The formula we use to implement this is
6133 // ((arg - 1) & 3) << 22).
6134 //
6135 // It is expected that the argument of llvm.set.rounding is within the
6136 // segment [0, 3], so NearestTiesToAway (4) is not handled here. It is
6137 // responsibility of the code generated llvm.set.rounding to ensure this
6138 // condition.
6139
6140 // Calculate new value of FPSCR[23:22].
6141 RMValue = DAG.getNode(ISD::SUB, DL, MVT::i32, RMValue,
6142 DAG.getConstant(1, DL, MVT::i32));
6143 RMValue = DAG.getNode(ISD::AND, DL, MVT::i32, RMValue,
6144 DAG.getConstant(0x3, DL, MVT::i32));
6145 RMValue = DAG.getNode(ISD::SHL, DL, MVT::i32, RMValue,
6146 DAG.getConstant(ARM::RoundingBitsPos, DL, MVT::i32));
6147
6148 // Get current value of FPSCR.
6149 SDValue Ops[] = {Chain,
6150 DAG.getConstant(Intrinsic::arm_get_fpscr, DL, MVT::i32)};
6151 SDValue FPSCR =
6152 DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL, {MVT::i32, MVT::Other}, Ops);
6153 Chain = FPSCR.getValue(1);
6154 FPSCR = FPSCR.getValue(0);
6155
6156 // Put new rounding mode into FPSCR[23:22].
6157 const unsigned RMMask = ~(ARM::Rounding::rmMask << ARM::RoundingBitsPos);
6158 FPSCR = DAG.getNode(ISD::AND, DL, MVT::i32, FPSCR,
6159 DAG.getConstant(RMMask, DL, MVT::i32));
6160 FPSCR = DAG.getNode(ISD::OR, DL, MVT::i32, FPSCR, RMValue);
6161 SDValue Ops2[] = {
6162 Chain, DAG.getConstant(Intrinsic::arm_set_fpscr, DL, MVT::i32), FPSCR};
6163 return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, Ops2);
6164}
6165
6166static SDValue LowerCTTZ(SDNode *N, SelectionDAG &DAG,
6167 const ARMSubtarget *ST) {
6168 SDLoc dl(N);
6169 EVT VT = N->getValueType(0);
6170 if (VT.isVector() && ST->hasNEON()) {
6171
6172 // Compute the least significant set bit: LSB = X & -X
6173 SDValue X = N->getOperand(0);
6174 SDValue NX = DAG.getNode(ISD::SUB, dl, VT, getZeroVector(VT, DAG, dl), X);
6175 SDValue LSB = DAG.getNode(ISD::AND, dl, VT, X, NX);
6176
6177 EVT ElemTy = VT.getVectorElementType();
6178
6179 if (ElemTy == MVT::i8) {
6180 // Compute with: cttz(x) = ctpop(lsb - 1)
6181 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6182 DAG.getTargetConstant(1, dl, ElemTy));
6183 SDValue Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6184 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6185 }
6186
6187 if ((ElemTy == MVT::i16 || ElemTy == MVT::i32) &&
6188 (N->getOpcode() == ISD::CTTZ_ZERO_UNDEF)) {
6189 // Compute with: cttz(x) = (width - 1) - ctlz(lsb), if x != 0
6190 unsigned NumBits = ElemTy.getSizeInBits();
6191 SDValue WidthMinus1 =
6192 DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6193 DAG.getTargetConstant(NumBits - 1, dl, ElemTy));
6194 SDValue CTLZ = DAG.getNode(ISD::CTLZ, dl, VT, LSB);
6195 return DAG.getNode(ISD::SUB, dl, VT, WidthMinus1, CTLZ);
6196 }
6197
6198 // Compute with: cttz(x) = ctpop(lsb - 1)
6199
6200 // Compute LSB - 1.
6201 SDValue Bits;
6202 if (ElemTy == MVT::i64) {
6203 // Load constant 0xffff'ffff'ffff'ffff to register.
6204 SDValue FF = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6205 DAG.getTargetConstant(0x1eff, dl, MVT::i32));
6206 Bits = DAG.getNode(ISD::ADD, dl, VT, LSB, FF);
6207 } else {
6208 SDValue One = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
6209 DAG.getTargetConstant(1, dl, ElemTy));
6210 Bits = DAG.getNode(ISD::SUB, dl, VT, LSB, One);
6211 }
6212 return DAG.getNode(ISD::CTPOP, dl, VT, Bits);
6213 }
6214
6215 if (!ST->hasV6T2Ops())
6216 return SDValue();
6217
6218 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, VT, N->getOperand(0));
6219 return DAG.getNode(ISD::CTLZ, dl, VT, rbit);
6220}
6221
6222static SDValue LowerCTPOP(SDNode *N, SelectionDAG &DAG,
6223 const ARMSubtarget *ST) {
6224 EVT VT = N->getValueType(0);
6225 SDLoc DL(N);
6226
6227 assert(ST->hasNEON() && "Custom ctpop lowering requires NEON.")(static_cast <bool> (ST->hasNEON() && "Custom ctpop lowering requires NEON."
) ? void (0) : __assert_fail ("ST->hasNEON() && \"Custom ctpop lowering requires NEON.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6227, __extension__ __PRETTY_FUNCTION__))
;
6228 assert((VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 ||(static_cast <bool> ((VT == MVT::v1i64 || VT == MVT::v2i64
|| VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 ||
VT == MVT::v8i16) && "Unexpected type for custom ctpop lowering"
) ? void (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6230, __extension__ __PRETTY_FUNCTION__))
6229 VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) &&(static_cast <bool> ((VT == MVT::v1i64 || VT == MVT::v2i64
|| VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 ||
VT == MVT::v8i16) && "Unexpected type for custom ctpop lowering"
) ? void (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6230, __extension__ __PRETTY_FUNCTION__))
6230 "Unexpected type for custom ctpop lowering")(static_cast <bool> ((VT == MVT::v1i64 || VT == MVT::v2i64
|| VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 ||
VT == MVT::v8i16) && "Unexpected type for custom ctpop lowering"
) ? void (0) : __assert_fail ("(VT == MVT::v1i64 || VT == MVT::v2i64 || VT == MVT::v2i32 || VT == MVT::v4i32 || VT == MVT::v4i16 || VT == MVT::v8i16) && \"Unexpected type for custom ctpop lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6230, __extension__ __PRETTY_FUNCTION__))
;
6231
6232 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6233 EVT VT8Bit = VT.is64BitVector() ? MVT::v8i8 : MVT::v16i8;
6234 SDValue Res = DAG.getBitcast(VT8Bit, N->getOperand(0));
6235 Res = DAG.getNode(ISD::CTPOP, DL, VT8Bit, Res);
6236
6237 // Widen v8i8/v16i8 CTPOP result to VT by repeatedly widening pairwise adds.
6238 unsigned EltSize = 8;
6239 unsigned NumElts = VT.is64BitVector() ? 8 : 16;
6240 while (EltSize != VT.getScalarSizeInBits()) {
6241 SmallVector<SDValue, 8> Ops;
6242 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddlu, DL,
6243 TLI.getPointerTy(DAG.getDataLayout())));
6244 Ops.push_back(Res);
6245
6246 EltSize *= 2;
6247 NumElts /= 2;
6248 MVT WidenVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize), NumElts);
6249 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, WidenVT, Ops);
6250 }
6251
6252 return Res;
6253}
6254
6255/// Getvshiftimm - Check if this is a valid build_vector for the immediate
6256/// operand of a vector shift operation, where all the elements of the
6257/// build_vector must have the same constant integer value.
6258static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) {
6259 // Ignore bit_converts.
6260 while (Op.getOpcode() == ISD::BITCAST)
6261 Op = Op.getOperand(0);
6262 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Op.getNode());
6263 APInt SplatBits, SplatUndef;
6264 unsigned SplatBitSize;
6265 bool HasAnyUndefs;
6266 if (!BVN ||
6267 !BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs,
6268 ElementBits) ||
6269 SplatBitSize > ElementBits)
6270 return false;
6271 Cnt = SplatBits.getSExtValue();
6272 return true;
6273}
6274
6275/// isVShiftLImm - Check if this is a valid build_vector for the immediate
6276/// operand of a vector shift left operation. That value must be in the range:
6277/// 0 <= Value < ElementBits for a left shift; or
6278/// 0 <= Value <= ElementBits for a long left shift.
6279static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) {
6280 assert(VT.isVector() && "vector shift count is not a vector type")(static_cast <bool> (VT.isVector() && "vector shift count is not a vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"vector shift count is not a vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6280, __extension__ __PRETTY_FUNCTION__))
;
6281 int64_t ElementBits = VT.getScalarSizeInBits();
6282 if (!getVShiftImm(Op, ElementBits, Cnt))
6283 return false;
6284 return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits);
6285}
6286
6287/// isVShiftRImm - Check if this is a valid build_vector for the immediate
6288/// operand of a vector shift right operation. For a shift opcode, the value
6289/// is positive, but for an intrinsic the value count must be negative. The
6290/// absolute value must be in the range:
6291/// 1 <= |Value| <= ElementBits for a right shift; or
6292/// 1 <= |Value| <= ElementBits/2 for a narrow right shift.
6293static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, bool isIntrinsic,
6294 int64_t &Cnt) {
6295 assert(VT.isVector() && "vector shift count is not a vector type")(static_cast <bool> (VT.isVector() && "vector shift count is not a vector type"
) ? void (0) : __assert_fail ("VT.isVector() && \"vector shift count is not a vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6295, __extension__ __PRETTY_FUNCTION__))
;
6296 int64_t ElementBits = VT.getScalarSizeInBits();
6297 if (!getVShiftImm(Op, ElementBits, Cnt))
6298 return false;
6299 if (!isIntrinsic)
6300 return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits));
6301 if (Cnt >= -(isNarrow ? ElementBits / 2 : ElementBits) && Cnt <= -1) {
6302 Cnt = -Cnt;
6303 return true;
6304 }
6305 return false;
6306}
6307
6308static SDValue LowerShift(SDNode *N, SelectionDAG &DAG,
6309 const ARMSubtarget *ST) {
6310 EVT VT = N->getValueType(0);
6311 SDLoc dl(N);
6312 int64_t Cnt;
6313
6314 if (!VT.isVector())
6315 return SDValue();
6316
6317 // We essentially have two forms here. Shift by an immediate and shift by a
6318 // vector register (there are also shift by a gpr, but that is just handled
6319 // with a tablegen pattern). We cannot easily match shift by an immediate in
6320 // tablegen so we do that here and generate a VSHLIMM/VSHRsIMM/VSHRuIMM.
6321 // For shifting by a vector, we don't have VSHR, only VSHL (which can be
6322 // signed or unsigned, and a negative shift indicates a shift right).
6323 if (N->getOpcode() == ISD::SHL) {
6324 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt))
6325 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
6326 DAG.getConstant(Cnt, dl, MVT::i32));
6327 return DAG.getNode(ARMISD::VSHLu, dl, VT, N->getOperand(0),
6328 N->getOperand(1));
6329 }
6330
6331 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&(static_cast <bool> ((N->getOpcode() == ISD::SRA || N
->getOpcode() == ISD::SRL) && "unexpected vector shift opcode"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"unexpected vector shift opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6332, __extension__ __PRETTY_FUNCTION__))
6332 "unexpected vector shift opcode")(static_cast <bool> ((N->getOpcode() == ISD::SRA || N
->getOpcode() == ISD::SRL) && "unexpected vector shift opcode"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"unexpected vector shift opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6332, __extension__ __PRETTY_FUNCTION__))
;
6333
6334 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
6335 unsigned VShiftOpc =
6336 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
6337 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
6338 DAG.getConstant(Cnt, dl, MVT::i32));
6339 }
6340
6341 // Other right shifts we don't have operations for (we use a shift left by a
6342 // negative number).
6343 EVT ShiftVT = N->getOperand(1).getValueType();
6344 SDValue NegatedCount = DAG.getNode(
6345 ISD::SUB, dl, ShiftVT, getZeroVector(ShiftVT, DAG, dl), N->getOperand(1));
6346 unsigned VShiftOpc =
6347 (N->getOpcode() == ISD::SRA ? ARMISD::VSHLs : ARMISD::VSHLu);
6348 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0), NegatedCount);
6349}
6350
6351static SDValue Expand64BitShift(SDNode *N, SelectionDAG &DAG,
6352 const ARMSubtarget *ST) {
6353 EVT VT = N->getValueType(0);
6354 SDLoc dl(N);
6355
6356 // We can get here for a node like i32 = ISD::SHL i32, i64
6357 if (VT != MVT::i64)
6358 return SDValue();
6359
6360 assert((N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA ||(static_cast <bool> ((N->getOpcode() == ISD::SRL || N
->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL
) && "Unknown shift to lower!") ? void (0) : __assert_fail
("(N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL) && \"Unknown shift to lower!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6362, __extension__ __PRETTY_FUNCTION__))
6361 N->getOpcode() == ISD::SHL) &&(static_cast <bool> ((N->getOpcode() == ISD::SRL || N
->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL
) && "Unknown shift to lower!") ? void (0) : __assert_fail
("(N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL) && \"Unknown shift to lower!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6362, __extension__ __PRETTY_FUNCTION__))
6362 "Unknown shift to lower!")(static_cast <bool> ((N->getOpcode() == ISD::SRL || N
->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL
) && "Unknown shift to lower!") ? void (0) : __assert_fail
("(N->getOpcode() == ISD::SRL || N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SHL) && \"Unknown shift to lower!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6362, __extension__ __PRETTY_FUNCTION__))
;
6363
6364 unsigned ShOpc = N->getOpcode();
6365 if (ST->hasMVEIntegerOps()) {
6366 SDValue ShAmt = N->getOperand(1);
6367 unsigned ShPartsOpc = ARMISD::LSLL;
6368 ConstantSDNode *Con = dyn_cast<ConstantSDNode>(ShAmt);
6369
6370 // If the shift amount is greater than 32 or has a greater bitwidth than 64
6371 // then do the default optimisation
6372 if (ShAmt->getValueType(0).getSizeInBits() > 64 ||
6373 (Con && (Con->getZExtValue() == 0 || Con->getZExtValue() >= 32)))
6374 return SDValue();
6375
6376 // Extract the lower 32 bits of the shift amount if it's not an i32
6377 if (ShAmt->getValueType(0) != MVT::i32)
6378 ShAmt = DAG.getZExtOrTrunc(ShAmt, dl, MVT::i32);
6379
6380 if (ShOpc == ISD::SRL) {
6381 if (!Con)
6382 // There is no t2LSRLr instruction so negate and perform an lsll if the
6383 // shift amount is in a register, emulating a right shift.
6384 ShAmt = DAG.getNode(ISD::SUB, dl, MVT::i32,
6385 DAG.getConstant(0, dl, MVT::i32), ShAmt);
6386 else
6387 // Else generate an lsrl on the immediate shift amount
6388 ShPartsOpc = ARMISD::LSRL;
6389 } else if (ShOpc == ISD::SRA)
6390 ShPartsOpc = ARMISD::ASRL;
6391
6392 // Lower 32 bits of the destination/source
6393 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6394 DAG.getConstant(0, dl, MVT::i32));
6395 // Upper 32 bits of the destination/source
6396 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6397 DAG.getConstant(1, dl, MVT::i32));
6398
6399 // Generate the shift operation as computed above
6400 Lo = DAG.getNode(ShPartsOpc, dl, DAG.getVTList(MVT::i32, MVT::i32), Lo, Hi,
6401 ShAmt);
6402 // The upper 32 bits come from the second return value of lsll
6403 Hi = SDValue(Lo.getNode(), 1);
6404 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6405 }
6406
6407 // We only lower SRA, SRL of 1 here, all others use generic lowering.
6408 if (!isOneConstant(N->getOperand(1)) || N->getOpcode() == ISD::SHL)
6409 return SDValue();
6410
6411 // If we are in thumb mode, we don't have RRX.
6412 if (ST->isThumb1Only())
6413 return SDValue();
6414
6415 // Okay, we have a 64-bit SRA or SRL of 1. Lower this to an RRX expr.
6416 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6417 DAG.getConstant(0, dl, MVT::i32));
6418 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, N->getOperand(0),
6419 DAG.getConstant(1, dl, MVT::i32));
6420
6421 // First, build a SRA_FLAG/SRL_FLAG op, which shifts the top part by one and
6422 // captures the result into a carry flag.
6423 unsigned Opc = N->getOpcode() == ISD::SRL ? ARMISD::SRL_FLAG:ARMISD::SRA_FLAG;
6424 Hi = DAG.getNode(Opc, dl, DAG.getVTList(MVT::i32, MVT::Glue), Hi);
6425
6426 // The low part is an ARMISD::RRX operand, which shifts the carry in.
6427 Lo = DAG.getNode(ARMISD::RRX, dl, MVT::i32, Lo, Hi.getValue(1));
6428
6429 // Merge the pieces into a single i64 value.
6430 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
6431}
6432
6433static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG,
6434 const ARMSubtarget *ST) {
6435 bool Invert = false;
6436 bool Swap = false;
6437 unsigned Opc = ARMCC::AL;
6438
6439 SDValue Op0 = Op.getOperand(0);
6440 SDValue Op1 = Op.getOperand(1);
6441 SDValue CC = Op.getOperand(2);
6442 EVT VT = Op.getValueType();
6443 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
6444 SDLoc dl(Op);
6445
6446 EVT CmpVT;
6447 if (ST->hasNEON())
6448 CmpVT = Op0.getValueType().changeVectorElementTypeToInteger();
6449 else {
6450 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"No hardware support for integer vector comparison!") ? void
(0) : __assert_fail ("ST->hasMVEIntegerOps() && \"No hardware support for integer vector comparison!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6451, __extension__ __PRETTY_FUNCTION__))
6451 "No hardware support for integer vector comparison!")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"No hardware support for integer vector comparison!") ? void
(0) : __assert_fail ("ST->hasMVEIntegerOps() && \"No hardware support for integer vector comparison!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6451, __extension__ __PRETTY_FUNCTION__))
;
6452
6453 if (Op.getValueType().getVectorElementType() != MVT::i1)
6454 return SDValue();
6455
6456 // Make sure we expand floating point setcc to scalar if we do not have
6457 // mve.fp, so that we can handle them from there.
6458 if (Op0.getValueType().isFloatingPoint() && !ST->hasMVEFloatOps())
6459 return SDValue();
6460
6461 CmpVT = VT;
6462 }
6463
6464 if (Op0.getValueType().getVectorElementType() == MVT::i64 &&
6465 (SetCCOpcode == ISD::SETEQ || SetCCOpcode == ISD::SETNE)) {
6466 // Special-case integer 64-bit equality comparisons. They aren't legal,
6467 // but they can be lowered with a few vector instructions.
6468 unsigned CmpElements = CmpVT.getVectorNumElements() * 2;
6469 EVT SplitVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, CmpElements);
6470 SDValue CastOp0 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op0);
6471 SDValue CastOp1 = DAG.getNode(ISD::BITCAST, dl, SplitVT, Op1);
6472 SDValue Cmp = DAG.getNode(ISD::SETCC, dl, SplitVT, CastOp0, CastOp1,
6473 DAG.getCondCode(ISD::SETEQ));
6474 SDValue Reversed = DAG.getNode(ARMISD::VREV64, dl, SplitVT, Cmp);
6475 SDValue Merged = DAG.getNode(ISD::AND, dl, SplitVT, Cmp, Reversed);
6476 Merged = DAG.getNode(ISD::BITCAST, dl, CmpVT, Merged);
6477 if (SetCCOpcode == ISD::SETNE)
6478 Merged = DAG.getNOT(dl, Merged, CmpVT);
6479 Merged = DAG.getSExtOrTrunc(Merged, dl, VT);
6480 return Merged;
6481 }
6482
6483 if (CmpVT.getVectorElementType() == MVT::i64)
6484 // 64-bit comparisons are not legal in general.
6485 return SDValue();
6486
6487 if (Op1.getValueType().isFloatingPoint()) {
6488 switch (SetCCOpcode) {
6489 default: llvm_unreachable("Illegal FP comparison")::llvm::llvm_unreachable_internal("Illegal FP comparison", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6489)
;
6490 case ISD::SETUNE:
6491 case ISD::SETNE:
6492 if (ST->hasMVEFloatOps()) {
6493 Opc = ARMCC::NE; break;
6494 } else {
6495 Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6496 }
6497 case ISD::SETOEQ:
6498 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6499 case ISD::SETOLT:
6500 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6501 case ISD::SETOGT:
6502 case ISD::SETGT: Opc = ARMCC::GT; break;
6503 case ISD::SETOLE:
6504 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6505 case ISD::SETOGE:
6506 case ISD::SETGE: Opc = ARMCC::GE; break;
6507 case ISD::SETUGE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6508 case ISD::SETULE: Invert = true; Opc = ARMCC::GT; break;
6509 case ISD::SETUGT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6510 case ISD::SETULT: Invert = true; Opc = ARMCC::GE; break;
6511 case ISD::SETUEQ: Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6512 case ISD::SETONE: {
6513 // Expand this to (OLT | OGT).
6514 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6515 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6516 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6517 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6518 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6519 if (Invert)
6520 Result = DAG.getNOT(dl, Result, VT);
6521 return Result;
6522 }
6523 case ISD::SETUO: Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6524 case ISD::SETO: {
6525 // Expand this to (OLT | OGE).
6526 SDValue TmpOp0 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op1, Op0,
6527 DAG.getConstant(ARMCC::GT, dl, MVT::i32));
6528 SDValue TmpOp1 = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6529 DAG.getConstant(ARMCC::GE, dl, MVT::i32));
6530 SDValue Result = DAG.getNode(ISD::OR, dl, CmpVT, TmpOp0, TmpOp1);
6531 if (Invert)
6532 Result = DAG.getNOT(dl, Result, VT);
6533 return Result;
6534 }
6535 }
6536 } else {
6537 // Integer comparisons.
6538 switch (SetCCOpcode) {
6539 default: llvm_unreachable("Illegal integer comparison")::llvm::llvm_unreachable_internal("Illegal integer comparison"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6539)
;
6540 case ISD::SETNE:
6541 if (ST->hasMVEIntegerOps()) {
6542 Opc = ARMCC::NE; break;
6543 } else {
6544 Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6545 }
6546 case ISD::SETEQ: Opc = ARMCC::EQ; break;
6547 case ISD::SETLT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6548 case ISD::SETGT: Opc = ARMCC::GT; break;
6549 case ISD::SETLE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6550 case ISD::SETGE: Opc = ARMCC::GE; break;
6551 case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6552 case ISD::SETUGT: Opc = ARMCC::HI; break;
6553 case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
6554 case ISD::SETUGE: Opc = ARMCC::HS; break;
6555 }
6556
6557 // Detect VTST (Vector Test Bits) = icmp ne (and (op0, op1), zero).
6558 if (ST->hasNEON() && Opc == ARMCC::EQ) {
6559 SDValue AndOp;
6560 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
6561 AndOp = Op0;
6562 else if (ISD::isBuildVectorAllZeros(Op0.getNode()))
6563 AndOp = Op1;
6564
6565 // Ignore bitconvert.
6566 if (AndOp.getNode() && AndOp.getOpcode() == ISD::BITCAST)
6567 AndOp = AndOp.getOperand(0);
6568
6569 if (AndOp.getNode() && AndOp.getOpcode() == ISD::AND) {
6570 Op0 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(0));
6571 Op1 = DAG.getNode(ISD::BITCAST, dl, CmpVT, AndOp.getOperand(1));
6572 SDValue Result = DAG.getNode(ARMISD::VTST, dl, CmpVT, Op0, Op1);
6573 if (!Invert)
6574 Result = DAG.getNOT(dl, Result, VT);
6575 return Result;
6576 }
6577 }
6578 }
6579
6580 if (Swap)
6581 std::swap(Op0, Op1);
6582
6583 // If one of the operands is a constant vector zero, attempt to fold the
6584 // comparison to a specialized compare-against-zero form.
6585 SDValue SingleOp;
6586 if (ISD::isBuildVectorAllZeros(Op1.getNode()))
6587 SingleOp = Op0;
6588 else if (ISD::isBuildVectorAllZeros(Op0.getNode())) {
6589 if (Opc == ARMCC::GE)
6590 Opc = ARMCC::LE;
6591 else if (Opc == ARMCC::GT)
6592 Opc = ARMCC::LT;
6593 SingleOp = Op1;
6594 }
6595
6596 SDValue Result;
6597 if (SingleOp.getNode()) {
6598 Result = DAG.getNode(ARMISD::VCMPZ, dl, CmpVT, SingleOp,
6599 DAG.getConstant(Opc, dl, MVT::i32));
6600 } else {
6601 Result = DAG.getNode(ARMISD::VCMP, dl, CmpVT, Op0, Op1,
6602 DAG.getConstant(Opc, dl, MVT::i32));
6603 }
6604
6605 Result = DAG.getSExtOrTrunc(Result, dl, VT);
6606
6607 if (Invert)
6608 Result = DAG.getNOT(dl, Result, VT);
6609
6610 return Result;
6611}
6612
6613static SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) {
6614 SDValue LHS = Op.getOperand(0);
6615 SDValue RHS = Op.getOperand(1);
6616 SDValue Carry = Op.getOperand(2);
6617 SDValue Cond = Op.getOperand(3);
6618 SDLoc DL(Op);
6619
6620 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.")(static_cast <bool> (LHS.getSimpleValueType().isInteger
() && "SETCCCARRY is integer only.") ? void (0) : __assert_fail
("LHS.getSimpleValueType().isInteger() && \"SETCCCARRY is integer only.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6620, __extension__ __PRETTY_FUNCTION__))
;
6621
6622 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
6623 // have to invert the carry first.
6624 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
6625 DAG.getConstant(1, DL, MVT::i32), Carry);
6626 // This converts the boolean value carry into the carry flag.
6627 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
6628
6629 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
6630 SDValue Cmp = DAG.getNode(ARMISD::SUBE, DL, VTs, LHS, RHS, Carry);
6631
6632 SDValue FVal = DAG.getConstant(0, DL, MVT::i32);
6633 SDValue TVal = DAG.getConstant(1, DL, MVT::i32);
6634 SDValue ARMcc = DAG.getConstant(
6635 IntCCToARMCC(cast<CondCodeSDNode>(Cond)->get()), DL, MVT::i32);
6636 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
6637 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), DL, ARM::CPSR,
6638 Cmp.getValue(1), SDValue());
6639 return DAG.getNode(ARMISD::CMOV, DL, Op.getValueType(), FVal, TVal, ARMcc,
6640 CCR, Chain.getValue(1));
6641}
6642
6643/// isVMOVModifiedImm - Check if the specified splat value corresponds to a
6644/// valid vector constant for a NEON or MVE instruction with a "modified
6645/// immediate" operand (e.g., VMOV). If so, return the encoded value.
6646static SDValue isVMOVModifiedImm(uint64_t SplatBits, uint64_t SplatUndef,
6647 unsigned SplatBitSize, SelectionDAG &DAG,
6648 const SDLoc &dl, EVT &VT, EVT VectorVT,
6649 VMOVModImmType type) {
6650 unsigned OpCmode, Imm;
6651 bool is128Bits = VectorVT.is128BitVector();
6652
6653 // SplatBitSize is set to the smallest size that splats the vector, so a
6654 // zero vector will always have SplatBitSize == 8. However, NEON modified
6655 // immediate instructions others than VMOV do not support the 8-bit encoding
6656 // of a zero vector, and the default encoding of zero is supposed to be the
6657 // 32-bit version.
6658 if (SplatBits == 0)
6659 SplatBitSize = 32;
6660
6661 switch (SplatBitSize) {
6662 case 8:
6663 if (type != VMOVModImm)
6664 return SDValue();
6665 // Any 1-byte value is OK. Op=0, Cmode=1110.
6666 assert((SplatBits & ~0xff) == 0 && "one byte splat value is too big")(static_cast <bool> ((SplatBits & ~0xff) == 0 &&
"one byte splat value is too big") ? void (0) : __assert_fail
("(SplatBits & ~0xff) == 0 && \"one byte splat value is too big\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6666, __extension__ __PRETTY_FUNCTION__))
;
6667 OpCmode = 0xe;
6668 Imm = SplatBits;
6669 VT = is128Bits ? MVT::v16i8 : MVT::v8i8;
6670 break;
6671
6672 case 16:
6673 // NEON's 16-bit VMOV supports splat values where only one byte is nonzero.
6674 VT = is128Bits ? MVT::v8i16 : MVT::v4i16;
6675 if ((SplatBits & ~0xff) == 0) {
6676 // Value = 0x00nn: Op=x, Cmode=100x.
6677 OpCmode = 0x8;
6678 Imm = SplatBits;
6679 break;
6680 }
6681 if ((SplatBits & ~0xff00) == 0) {
6682 // Value = 0xnn00: Op=x, Cmode=101x.
6683 OpCmode = 0xa;
6684 Imm = SplatBits >> 8;
6685 break;
6686 }
6687 return SDValue();
6688
6689 case 32:
6690 // NEON's 32-bit VMOV supports splat values where:
6691 // * only one byte is nonzero, or
6692 // * the least significant byte is 0xff and the second byte is nonzero, or
6693 // * the least significant 2 bytes are 0xff and the third is nonzero.
6694 VT = is128Bits ? MVT::v4i32 : MVT::v2i32;
6695 if ((SplatBits & ~0xff) == 0) {
6696 // Value = 0x000000nn: Op=x, Cmode=000x.
6697 OpCmode = 0;
6698 Imm = SplatBits;
6699 break;
6700 }
6701 if ((SplatBits & ~0xff00) == 0) {
6702 // Value = 0x0000nn00: Op=x, Cmode=001x.
6703 OpCmode = 0x2;
6704 Imm = SplatBits >> 8;
6705 break;
6706 }
6707 if ((SplatBits & ~0xff0000) == 0) {
6708 // Value = 0x00nn0000: Op=x, Cmode=010x.
6709 OpCmode = 0x4;
6710 Imm = SplatBits >> 16;
6711 break;
6712 }
6713 if ((SplatBits & ~0xff000000) == 0) {
6714 // Value = 0xnn000000: Op=x, Cmode=011x.
6715 OpCmode = 0x6;
6716 Imm = SplatBits >> 24;
6717 break;
6718 }
6719
6720 // cmode == 0b1100 and cmode == 0b1101 are not supported for VORR or VBIC
6721 if (type == OtherModImm) return SDValue();
6722
6723 if ((SplatBits & ~0xffff) == 0 &&
6724 ((SplatBits | SplatUndef) & 0xff) == 0xff) {
6725 // Value = 0x0000nnff: Op=x, Cmode=1100.
6726 OpCmode = 0xc;
6727 Imm = SplatBits >> 8;
6728 break;
6729 }
6730
6731 // cmode == 0b1101 is not supported for MVE VMVN
6732 if (type == MVEVMVNModImm)
6733 return SDValue();
6734
6735 if ((SplatBits & ~0xffffff) == 0 &&
6736 ((SplatBits | SplatUndef) & 0xffff) == 0xffff) {
6737 // Value = 0x00nnffff: Op=x, Cmode=1101.
6738 OpCmode = 0xd;
6739 Imm = SplatBits >> 16;
6740 break;
6741 }
6742
6743 // Note: there are a few 32-bit splat values (specifically: 00ffff00,
6744 // ff000000, ff0000ff, and ffff00ff) that are valid for VMOV.I64 but not
6745 // VMOV.I32. A (very) minor optimization would be to replicate the value
6746 // and fall through here to test for a valid 64-bit splat. But, then the
6747 // caller would also need to check and handle the change in size.
6748 return SDValue();
6749
6750 case 64: {
6751 if (type != VMOVModImm)
6752 return SDValue();
6753 // NEON has a 64-bit VMOV splat where each byte is either 0 or 0xff.
6754 uint64_t BitMask = 0xff;
6755 uint64_t Val = 0;
6756 unsigned ImmMask = 1;
6757 Imm = 0;
6758 for (int ByteNum = 0; ByteNum < 8; ++ByteNum) {
6759 if (((SplatBits | SplatUndef) & BitMask) == BitMask) {
6760 Val |= BitMask;
6761 Imm |= ImmMask;
6762 } else if ((SplatBits & BitMask) != 0) {
6763 return SDValue();
6764 }
6765 BitMask <<= 8;
6766 ImmMask <<= 1;
6767 }
6768
6769 if (DAG.getDataLayout().isBigEndian()) {
6770 // Reverse the order of elements within the vector.
6771 unsigned BytesPerElem = VectorVT.getScalarSizeInBits() / 8;
6772 unsigned Mask = (1 << BytesPerElem) - 1;
6773 unsigned NumElems = 8 / BytesPerElem;
6774 unsigned NewImm = 0;
6775 for (unsigned ElemNum = 0; ElemNum < NumElems; ++ElemNum) {
6776 unsigned Elem = ((Imm >> ElemNum * BytesPerElem) & Mask);
6777 NewImm |= Elem << (NumElems - ElemNum - 1) * BytesPerElem;
6778 }
6779 Imm = NewImm;
6780 }
6781
6782 // Op=1, Cmode=1110.
6783 OpCmode = 0x1e;
6784 VT = is128Bits ? MVT::v2i64 : MVT::v1i64;
6785 break;
6786 }
6787
6788 default:
6789 llvm_unreachable("unexpected size for isVMOVModifiedImm")::llvm::llvm_unreachable_internal("unexpected size for isVMOVModifiedImm"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6789)
;
6790 }
6791
6792 unsigned EncodedVal = ARM_AM::createVMOVModImm(OpCmode, Imm);
6793 return DAG.getTargetConstant(EncodedVal, dl, MVT::i32);
6794}
6795
6796SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
6797 const ARMSubtarget *ST) const {
6798 EVT VT = Op.getValueType();
6799 bool IsDouble = (VT == MVT::f64);
6800 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
6801 const APFloat &FPVal = CFP->getValueAPF();
6802
6803 // Prevent floating-point constants from using literal loads
6804 // when execute-only is enabled.
6805 if (ST->genExecuteOnly()) {
6806 // If we can represent the constant as an immediate, don't lower it
6807 if (isFPImmLegal(FPVal, VT))
6808 return Op;
6809 // Otherwise, construct as integer, and move to float register
6810 APInt INTVal = FPVal.bitcastToAPInt();
6811 SDLoc DL(CFP);
6812 switch (VT.getSimpleVT().SimpleTy) {
6813 default:
6814 llvm_unreachable("Unknown floating point type!")::llvm::llvm_unreachable_internal("Unknown floating point type!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 6814)
;
6815 break;
6816 case MVT::f64: {
6817 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
6818 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
6819 return DAG.getNode(ARMISD::VMOVDRR, DL, MVT::f64, Lo, Hi);
6820 }
6821 case MVT::f32:
6822 return DAG.getNode(ARMISD::VMOVSR, DL, VT,
6823 DAG.getConstant(INTVal, DL, MVT::i32));
6824 }
6825 }
6826
6827 if (!ST->hasVFP3Base())
6828 return SDValue();
6829
6830 // Use the default (constant pool) lowering for double constants when we have
6831 // an SP-only FPU
6832 if (IsDouble && !Subtarget->hasFP64())
6833 return SDValue();
6834
6835 // Try splatting with a VMOV.f32...
6836 int ImmVal = IsDouble ? ARM_AM::getFP64Imm(FPVal) : ARM_AM::getFP32Imm(FPVal);
6837
6838 if (ImmVal != -1) {
6839 if (IsDouble || !ST->useNEONForSinglePrecisionFP()) {
6840 // We have code in place to select a valid ConstantFP already, no need to
6841 // do any mangling.
6842 return Op;
6843 }
6844
6845 // It's a float and we are trying to use NEON operations where
6846 // possible. Lower it to a splat followed by an extract.
6847 SDLoc DL(Op);
6848 SDValue NewVal = DAG.getTargetConstant(ImmVal, DL, MVT::i32);
6849 SDValue VecConstant = DAG.getNode(ARMISD::VMOVFPIMM, DL, MVT::v2f32,
6850 NewVal);
6851 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecConstant,
6852 DAG.getConstant(0, DL, MVT::i32));
6853 }
6854
6855 // The rest of our options are NEON only, make sure that's allowed before
6856 // proceeding..
6857 if (!ST->hasNEON() || (!IsDouble && !ST->useNEONForSinglePrecisionFP()))
6858 return SDValue();
6859
6860 EVT VMovVT;
6861 uint64_t iVal = FPVal.bitcastToAPInt().getZExtValue();
6862
6863 // It wouldn't really be worth bothering for doubles except for one very
6864 // important value, which does happen to match: 0.0. So make sure we don't do
6865 // anything stupid.
6866 if (IsDouble && (iVal & 0xffffffff) != (iVal >> 32))
6867 return SDValue();
6868
6869 // Try a VMOV.i32 (FIXME: i8, i16, or i64 could work too).
6870 SDValue NewVal = isVMOVModifiedImm(iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op),
6871 VMovVT, VT, VMOVModImm);
6872 if (NewVal != SDValue()) {
6873 SDLoc DL(Op);
6874 SDValue VecConstant = DAG.getNode(ARMISD::VMOVIMM, DL, VMovVT,
6875 NewVal);
6876 if (IsDouble)
6877 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
6878
6879 // It's a float: cast and extract a vector element.
6880 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
6881 VecConstant);
6882 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
6883 DAG.getConstant(0, DL, MVT::i32));
6884 }
6885
6886 // Finally, try a VMVN.i32
6887 NewVal = isVMOVModifiedImm(~iVal & 0xffffffffU, 0, 32, DAG, SDLoc(Op), VMovVT,
6888 VT, VMVNModImm);
6889 if (NewVal != SDValue()) {
6890 SDLoc DL(Op);
6891 SDValue VecConstant = DAG.getNode(ARMISD::VMVNIMM, DL, VMovVT, NewVal);
6892
6893 if (IsDouble)
6894 return DAG.getNode(ISD::BITCAST, DL, MVT::f64, VecConstant);
6895
6896 // It's a float: cast and extract a vector element.
6897 SDValue VecFConstant = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32,
6898 VecConstant);
6899 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, VecFConstant,
6900 DAG.getConstant(0, DL, MVT::i32));
6901 }
6902
6903 return SDValue();
6904}
6905
6906// check if an VEXT instruction can handle the shuffle mask when the
6907// vector sources of the shuffle are the same.
6908static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
6909 unsigned NumElts = VT.getVectorNumElements();
6910
6911 // Assume that the first shuffle index is not UNDEF. Fail if it is.
6912 if (M[0] < 0)
6913 return false;
6914
6915 Imm = M[0];
6916
6917 // If this is a VEXT shuffle, the immediate value is the index of the first
6918 // element. The other shuffle indices must be the successive elements after
6919 // the first one.
6920 unsigned ExpectedElt = Imm;
6921 for (unsigned i = 1; i < NumElts; ++i) {
6922 // Increment the expected index. If it wraps around, just follow it
6923 // back to index zero and keep going.
6924 ++ExpectedElt;
6925 if (ExpectedElt == NumElts)
6926 ExpectedElt = 0;
6927
6928 if (M[i] < 0) continue; // ignore UNDEF indices
6929 if (ExpectedElt != static_cast<unsigned>(M[i]))
6930 return false;
6931 }
6932
6933 return true;
6934}
6935
6936static bool isVEXTMask(ArrayRef<int> M, EVT VT,
6937 bool &ReverseVEXT, unsigned &Imm) {
6938 unsigned NumElts = VT.getVectorNumElements();
6939 ReverseVEXT = false;
6940
6941 // Assume that the first shuffle index is not UNDEF. Fail if it is.
6942 if (M[0] < 0)
6943 return false;
6944
6945 Imm = M[0];
6946
6947 // If this is a VEXT shuffle, the immediate value is the index of the first
6948 // element. The other shuffle indices must be the successive elements after
6949 // the first one.
6950 unsigned ExpectedElt = Imm;
6951 for (unsigned i = 1; i < NumElts; ++i) {
6952 // Increment the expected index. If it wraps around, it may still be
6953 // a VEXT but the source vectors must be swapped.
6954 ExpectedElt += 1;
6955 if (ExpectedElt == NumElts * 2) {
6956 ExpectedElt = 0;
6957 ReverseVEXT = true;
6958 }
6959
6960 if (M[i] < 0) continue; // ignore UNDEF indices
6961 if (ExpectedElt != static_cast<unsigned>(M[i]))
6962 return false;
6963 }
6964
6965 // Adjust the index value if the source operands will be swapped.
6966 if (ReverseVEXT)
6967 Imm -= NumElts;
6968
6969 return true;
6970}
6971
6972static bool isVTBLMask(ArrayRef<int> M, EVT VT) {
6973 // We can handle <8 x i8> vector shuffles. If the index in the mask is out of
6974 // range, then 0 is placed into the resulting vector. So pretty much any mask
6975 // of 8 elements can work here.
6976 return VT == MVT::v8i8 && M.size() == 8;
6977}
6978
6979static unsigned SelectPairHalf(unsigned Elements, ArrayRef<int> Mask,
6980 unsigned Index) {
6981 if (Mask.size() == Elements * 2)
6982 return Index / Elements;
6983 return Mask[Index] == 0 ? 0 : 1;
6984}
6985
6986// Checks whether the shuffle mask represents a vector transpose (VTRN) by
6987// checking that pairs of elements in the shuffle mask represent the same index
6988// in each vector, incrementing the expected index by 2 at each step.
6989// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 2, 6]
6990// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,c,g}
6991// v2={e,f,g,h}
6992// WhichResult gives the offset for each element in the mask based on which
6993// of the two results it belongs to.
6994//
6995// The transpose can be represented either as:
6996// result1 = shufflevector v1, v2, result1_shuffle_mask
6997// result2 = shufflevector v1, v2, result2_shuffle_mask
6998// where v1/v2 and the shuffle masks have the same number of elements
6999// (here WhichResult (see below) indicates which result is being checked)
7000//
7001// or as:
7002// results = shufflevector v1, v2, shuffle_mask
7003// where both results are returned in one vector and the shuffle mask has twice
7004// as many elements as v1/v2 (here WhichResult will always be 0 if true) here we
7005// want to check the low half and high half of the shuffle mask as if it were
7006// the other case
7007static bool isVTRNMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7008 unsigned EltSz = VT.getScalarSizeInBits();
7009 if (EltSz == 64)
7010 return false;
7011
7012 unsigned NumElts = VT.getVectorNumElements();
7013 if (M.size() != NumElts && M.size() != NumElts*2)
7014 return false;
7015
7016 // If the mask is twice as long as the input vector then we need to check the
7017 // upper and lower parts of the mask with a matching value for WhichResult
7018 // FIXME: A mask with only even values will be rejected in case the first
7019 // element is undefined, e.g. [-1, 4, 2, 6] will be rejected, because only
7020 // M[0] is used to determine WhichResult
7021 for (unsigned i = 0; i < M.size(); i += NumElts) {
7022 WhichResult = SelectPairHalf(NumElts, M, i);
7023 for (unsigned j = 0; j < NumElts; j += 2) {
7024 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7025 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + NumElts + WhichResult))
7026 return false;
7027 }
7028 }
7029
7030 if (M.size() == NumElts*2)
7031 WhichResult = 0;
7032
7033 return true;
7034}
7035
7036/// isVTRN_v_undef_Mask - Special case of isVTRNMask for canonical form of
7037/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7038/// Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
7039static bool isVTRN_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7040 unsigned EltSz = VT.getScalarSizeInBits();
7041 if (EltSz == 64)
7042 return false;
7043
7044 unsigned NumElts = VT.getVectorNumElements();
7045 if (M.size() != NumElts && M.size() != NumElts*2)
7046 return false;
7047
7048 for (unsigned i = 0; i < M.size(); i += NumElts) {
7049 WhichResult = SelectPairHalf(NumElts, M, i);
7050 for (unsigned j = 0; j < NumElts; j += 2) {
7051 if ((M[i+j] >= 0 && (unsigned) M[i+j] != j + WhichResult) ||
7052 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != j + WhichResult))
7053 return false;
7054 }
7055 }
7056
7057 if (M.size() == NumElts*2)
7058 WhichResult = 0;
7059
7060 return true;
7061}
7062
7063// Checks whether the shuffle mask represents a vector unzip (VUZP) by checking
7064// that the mask elements are either all even and in steps of size 2 or all odd
7065// and in steps of size 2.
7066// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 2, 4, 6]
7067// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,c,e,g}
7068// v2={e,f,g,h}
7069// Requires similar checks to that of isVTRNMask with
7070// respect the how results are returned.
7071static bool isVUZPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7072 unsigned EltSz = VT.getScalarSizeInBits();
7073 if (EltSz == 64)
7074 return false;
7075
7076 unsigned NumElts = VT.getVectorNumElements();
7077 if (M.size() != NumElts && M.size() != NumElts*2)
7078 return false;
7079
7080 for (unsigned i = 0; i < M.size(); i += NumElts) {
7081 WhichResult = SelectPairHalf(NumElts, M, i);
7082 for (unsigned j = 0; j < NumElts; ++j) {
7083 if (M[i+j] >= 0 && (unsigned) M[i+j] != 2 * j + WhichResult)
7084 return false;
7085 }
7086 }
7087
7088 if (M.size() == NumElts*2)
7089 WhichResult = 0;
7090
7091 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7092 if (VT.is64BitVector() && EltSz == 32)
7093 return false;
7094
7095 return true;
7096}
7097
7098/// isVUZP_v_undef_Mask - Special case of isVUZPMask for canonical form of
7099/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7100/// Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
7101static bool isVUZP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7102 unsigned EltSz = VT.getScalarSizeInBits();
7103 if (EltSz == 64)
7104 return false;
7105
7106 unsigned NumElts = VT.getVectorNumElements();
7107 if (M.size() != NumElts && M.size() != NumElts*2)
7108 return false;
7109
7110 unsigned Half = NumElts / 2;
7111 for (unsigned i = 0; i < M.size(); i += NumElts) {
7112 WhichResult = SelectPairHalf(NumElts, M, i);
7113 for (unsigned j = 0; j < NumElts; j += Half) {
7114 unsigned Idx = WhichResult;
7115 for (unsigned k = 0; k < Half; ++k) {
7116 int MIdx = M[i + j + k];
7117 if (MIdx >= 0 && (unsigned) MIdx != Idx)
7118 return false;
7119 Idx += 2;
7120 }
7121 }
7122 }
7123
7124 if (M.size() == NumElts*2)
7125 WhichResult = 0;
7126
7127 // VUZP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7128 if (VT.is64BitVector() && EltSz == 32)
7129 return false;
7130
7131 return true;
7132}
7133
7134// Checks whether the shuffle mask represents a vector zip (VZIP) by checking
7135// that pairs of elements of the shufflemask represent the same index in each
7136// vector incrementing sequentially through the vectors.
7137// e.g. For v1,v2 of type v4i32 a valid shuffle mask is: [0, 4, 1, 5]
7138// v1={a,b,c,d} => x=shufflevector v1, v2 shufflemask => x={a,e,b,f}
7139// v2={e,f,g,h}
7140// Requires similar checks to that of isVTRNMask with respect the how results
7141// are returned.
7142static bool isVZIPMask(ArrayRef<int> M, EVT VT, unsigned &WhichResult) {
7143 unsigned EltSz = VT.getScalarSizeInBits();
7144 if (EltSz == 64)
7145 return false;
7146
7147 unsigned NumElts = VT.getVectorNumElements();
7148 if (M.size() != NumElts && M.size() != NumElts*2)
7149 return false;
7150
7151 for (unsigned i = 0; i < M.size(); i += NumElts) {
7152 WhichResult = SelectPairHalf(NumElts, M, i);
7153 unsigned Idx = WhichResult * NumElts / 2;
7154 for (unsigned j = 0; j < NumElts; j += 2) {
7155 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7156 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx + NumElts))
7157 return false;
7158 Idx += 1;
7159 }
7160 }
7161
7162 if (M.size() == NumElts*2)
7163 WhichResult = 0;
7164
7165 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7166 if (VT.is64BitVector() && EltSz == 32)
7167 return false;
7168
7169 return true;
7170}
7171
7172/// isVZIP_v_undef_Mask - Special case of isVZIPMask for canonical form of
7173/// "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
7174/// Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
7175static bool isVZIP_v_undef_Mask(ArrayRef<int> M, EVT VT, unsigned &WhichResult){
7176 unsigned EltSz = VT.getScalarSizeInBits();
7177 if (EltSz == 64)
7178 return false;
7179
7180 unsigned NumElts = VT.getVectorNumElements();
7181 if (M.size() != NumElts && M.size() != NumElts*2)
7182 return false;
7183
7184 for (unsigned i = 0; i < M.size(); i += NumElts) {
7185 WhichResult = SelectPairHalf(NumElts, M, i);
7186 unsigned Idx = WhichResult * NumElts / 2;
7187 for (unsigned j = 0; j < NumElts; j += 2) {
7188 if ((M[i+j] >= 0 && (unsigned) M[i+j] != Idx) ||
7189 (M[i+j+1] >= 0 && (unsigned) M[i+j+1] != Idx))
7190 return false;
7191 Idx += 1;
7192 }
7193 }
7194
7195 if (M.size() == NumElts*2)
7196 WhichResult = 0;
7197
7198 // VZIP.32 for 64-bit vectors is a pseudo-instruction alias for VTRN.32.
7199 if (VT.is64BitVector() && EltSz == 32)
7200 return false;
7201
7202 return true;
7203}
7204
7205/// Check if \p ShuffleMask is a NEON two-result shuffle (VZIP, VUZP, VTRN),
7206/// and return the corresponding ARMISD opcode if it is, or 0 if it isn't.
7207static unsigned isNEONTwoResultShuffleMask(ArrayRef<int> ShuffleMask, EVT VT,
7208 unsigned &WhichResult,
7209 bool &isV_UNDEF) {
7210 isV_UNDEF = false;
7211 if (isVTRNMask(ShuffleMask, VT, WhichResult))
7212 return ARMISD::VTRN;
7213 if (isVUZPMask(ShuffleMask, VT, WhichResult))
7214 return ARMISD::VUZP;
7215 if (isVZIPMask(ShuffleMask, VT, WhichResult))
7216 return ARMISD::VZIP;
7217
7218 isV_UNDEF = true;
7219 if (isVTRN_v_undef_Mask(ShuffleMask, VT, WhichResult))
7220 return ARMISD::VTRN;
7221 if (isVUZP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7222 return ARMISD::VUZP;
7223 if (isVZIP_v_undef_Mask(ShuffleMask, VT, WhichResult))
7224 return ARMISD::VZIP;
7225
7226 return 0;
7227}
7228
7229/// \return true if this is a reverse operation on an vector.
7230static bool isReverseMask(ArrayRef<int> M, EVT VT) {
7231 unsigned NumElts = VT.getVectorNumElements();
7232 // Make sure the mask has the right size.
7233 if (NumElts != M.size())
7234 return false;
7235
7236 // Look for <15, ..., 3, -1, 1, 0>.
7237 for (unsigned i = 0; i != NumElts; ++i)
7238 if (M[i] >= 0 && M[i] != (int) (NumElts - 1 - i))
7239 return false;
7240
7241 return true;
7242}
7243
7244static bool isVMOVNMask(ArrayRef<int> M, EVT VT, bool Top, bool SingleSource) {
7245 unsigned NumElts = VT.getVectorNumElements();
7246 // Make sure the mask has the right size.
7247 if (NumElts != M.size() || (VT != MVT::v8i16 && VT != MVT::v16i8))
7248 return false;
7249
7250 // If Top
7251 // Look for <0, N, 2, N+2, 4, N+4, ..>.
7252 // This inserts Input2 into Input1
7253 // else if not Top
7254 // Look for <0, N+1, 2, N+3, 4, N+5, ..>
7255 // This inserts Input1 into Input2
7256 unsigned Offset = Top ? 0 : 1;
7257 unsigned N = SingleSource ? 0 : NumElts;
7258 for (unsigned i = 0; i < NumElts; i += 2) {
7259 if (M[i] >= 0 && M[i] != (int)i)
7260 return false;
7261 if (M[i + 1] >= 0 && M[i + 1] != (int)(N + i + Offset))
7262 return false;
7263 }
7264
7265 return true;
7266}
7267
7268// Reconstruct an MVE VCVT from a BuildVector of scalar fptrunc, all extracted
7269// from a pair of inputs. For example:
7270// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7271// FP_ROUND(EXTRACT_ELT(Y, 0),
7272// FP_ROUND(EXTRACT_ELT(X, 1),
7273// FP_ROUND(EXTRACT_ELT(Y, 1), ...)
7274static SDValue LowerBuildVectorOfFPTrunc(SDValue BV, SelectionDAG &DAG,
7275 const ARMSubtarget *ST) {
7276 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!")(static_cast <bool> (BV.getOpcode() == ISD::BUILD_VECTOR
&& "Unknown opcode!") ? void (0) : __assert_fail ("BV.getOpcode() == ISD::BUILD_VECTOR && \"Unknown opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7276, __extension__ __PRETTY_FUNCTION__))
;
7277 if (!ST->hasMVEFloatOps())
7278 return SDValue();
7279
7280 SDLoc dl(BV);
7281 EVT VT = BV.getValueType();
7282 if (VT != MVT::v8f16)
7283 return SDValue();
7284
7285 // We are looking for a buildvector of fptrunc elements, where all the
7286 // elements are interleavingly extracted from two sources. Check the first two
7287 // items are valid enough and extract some info from them (they are checked
7288 // properly in the loop below).
7289 if (BV.getOperand(0).getOpcode() != ISD::FP_ROUND ||
7290 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7291 BV.getOperand(0).getOperand(0).getConstantOperandVal(1) != 0)
7292 return SDValue();
7293 if (BV.getOperand(1).getOpcode() != ISD::FP_ROUND ||
7294 BV.getOperand(1).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7295 BV.getOperand(1).getOperand(0).getConstantOperandVal(1) != 0)
7296 return SDValue();
7297 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7298 SDValue Op1 = BV.getOperand(1).getOperand(0).getOperand(0);
7299 if (Op0.getValueType() != MVT::v4f32 || Op1.getValueType() != MVT::v4f32)
7300 return SDValue();
7301
7302 // Check all the values in the BuildVector line up with our expectations.
7303 for (unsigned i = 1; i < 4; i++) {
7304 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7305 return Trunc.getOpcode() == ISD::FP_ROUND &&
7306 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7307 Trunc.getOperand(0).getOperand(0) == Op &&
7308 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7309 };
7310 if (!Check(BV.getOperand(i * 2 + 0), Op0, i))
7311 return SDValue();
7312 if (!Check(BV.getOperand(i * 2 + 1), Op1, i))
7313 return SDValue();
7314 }
7315
7316 SDValue N1 = DAG.getNode(ARMISD::VCVTN, dl, VT, DAG.getUNDEF(VT), Op0,
7317 DAG.getConstant(0, dl, MVT::i32));
7318 return DAG.getNode(ARMISD::VCVTN, dl, VT, N1, Op1,
7319 DAG.getConstant(1, dl, MVT::i32));
7320}
7321
7322// Reconstruct an MVE VCVT from a BuildVector of scalar fpext, all extracted
7323// from a single input on alternating lanes. For example:
7324// BUILDVECTOR(FP_ROUND(EXTRACT_ELT(X, 0),
7325// FP_ROUND(EXTRACT_ELT(X, 2),
7326// FP_ROUND(EXTRACT_ELT(X, 4), ...)
7327static SDValue LowerBuildVectorOfFPExt(SDValue BV, SelectionDAG &DAG,
7328 const ARMSubtarget *ST) {
7329 assert(BV.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!")(static_cast <bool> (BV.getOpcode() == ISD::BUILD_VECTOR
&& "Unknown opcode!") ? void (0) : __assert_fail ("BV.getOpcode() == ISD::BUILD_VECTOR && \"Unknown opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7329, __extension__ __PRETTY_FUNCTION__))
;
7330 if (!ST->hasMVEFloatOps())
7331 return SDValue();
7332
7333 SDLoc dl(BV);
7334 EVT VT = BV.getValueType();
7335 if (VT != MVT::v4f32)
7336 return SDValue();
7337
7338 // We are looking for a buildvector of fptext elements, where all the
7339 // elements are alternating lanes from a single source. For example <0,2,4,6>
7340 // or <1,3,5,7>. Check the first two items are valid enough and extract some
7341 // info from them (they are checked properly in the loop below).
7342 if (BV.getOperand(0).getOpcode() != ISD::FP_EXTEND ||
7343 BV.getOperand(0).getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
7344 return SDValue();
7345 SDValue Op0 = BV.getOperand(0).getOperand(0).getOperand(0);
7346 int Offset = BV.getOperand(0).getOperand(0).getConstantOperandVal(1);
7347 if (Op0.getValueType() != MVT::v8f16 || (Offset != 0 && Offset != 1))
7348 return SDValue();
7349
7350 // Check all the values in the BuildVector line up with our expectations.
7351 for (unsigned i = 1; i < 4; i++) {
7352 auto Check = [](SDValue Trunc, SDValue Op, unsigned Idx) {
7353 return Trunc.getOpcode() == ISD::FP_EXTEND &&
7354 Trunc.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7355 Trunc.getOperand(0).getOperand(0) == Op &&
7356 Trunc.getOperand(0).getConstantOperandVal(1) == Idx;
7357 };
7358 if (!Check(BV.getOperand(i), Op0, 2 * i + Offset))
7359 return SDValue();
7360 }
7361
7362 return DAG.getNode(ARMISD::VCVTL, dl, VT, Op0,
7363 DAG.getConstant(Offset, dl, MVT::i32));
7364}
7365
7366// If N is an integer constant that can be moved into a register in one
7367// instruction, return an SDValue of such a constant (will become a MOV
7368// instruction). Otherwise return null.
7369static SDValue IsSingleInstrConstant(SDValue N, SelectionDAG &DAG,
7370 const ARMSubtarget *ST, const SDLoc &dl) {
7371 uint64_t Val;
7372 if (!isa<ConstantSDNode>(N))
7373 return SDValue();
7374 Val = cast<ConstantSDNode>(N)->getZExtValue();
7375
7376 if (ST->isThumb1Only()) {
7377 if (Val <= 255 || ~Val <= 255)
7378 return DAG.getConstant(Val, dl, MVT::i32);
7379 } else {
7380 if (ARM_AM::getSOImmVal(Val) != -1 || ARM_AM::getSOImmVal(~Val) != -1)
7381 return DAG.getConstant(Val, dl, MVT::i32);
7382 }
7383 return SDValue();
7384}
7385
7386static SDValue LowerBUILD_VECTOR_i1(SDValue Op, SelectionDAG &DAG,
7387 const ARMSubtarget *ST) {
7388 SDLoc dl(Op);
7389 EVT VT = Op.getValueType();
7390
7391 assert(ST->hasMVEIntegerOps() && "LowerBUILD_VECTOR_i1 called without MVE!")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"LowerBUILD_VECTOR_i1 called without MVE!") ? void (0) : __assert_fail
("ST->hasMVEIntegerOps() && \"LowerBUILD_VECTOR_i1 called without MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7391, __extension__ __PRETTY_FUNCTION__))
;
7392
7393 unsigned NumElts = VT.getVectorNumElements();
7394 unsigned BoolMask;
7395 unsigned BitsPerBool;
7396 if (NumElts == 4) {
7397 BitsPerBool = 4;
7398 BoolMask = 0xf;
7399 } else if (NumElts == 8) {
7400 BitsPerBool = 2;
7401 BoolMask = 0x3;
7402 } else if (NumElts == 16) {
7403 BitsPerBool = 1;
7404 BoolMask = 0x1;
7405 } else
7406 return SDValue();
7407
7408 // If this is a single value copied into all lanes (a splat), we can just sign
7409 // extend that single value
7410 SDValue FirstOp = Op.getOperand(0);
7411 if (!isa<ConstantSDNode>(FirstOp) &&
7412 std::all_of(std::next(Op->op_begin()), Op->op_end(),
7413 [&FirstOp](SDUse &U) {
7414 return U.get().isUndef() || U.get() == FirstOp;
7415 })) {
7416 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32, FirstOp,
7417 DAG.getValueType(MVT::i1));
7418 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), Ext);
7419 }
7420
7421 // First create base with bits set where known
7422 unsigned Bits32 = 0;
7423 for (unsigned i = 0; i < NumElts; ++i) {
7424 SDValue V = Op.getOperand(i);
7425 if (!isa<ConstantSDNode>(V) && !V.isUndef())
7426 continue;
7427 bool BitSet = V.isUndef() ? false : cast<ConstantSDNode>(V)->getZExtValue();
7428 if (BitSet)
7429 Bits32 |= BoolMask << (i * BitsPerBool);
7430 }
7431
7432 // Add in unknown nodes
7433 SDValue Base = DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
7434 DAG.getConstant(Bits32, dl, MVT::i32));
7435 for (unsigned i = 0; i < NumElts; ++i) {
7436 SDValue V = Op.getOperand(i);
7437 if (isa<ConstantSDNode>(V) || V.isUndef())
7438 continue;
7439 Base = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Base, V,
7440 DAG.getConstant(i, dl, MVT::i32));
7441 }
7442
7443 return Base;
7444}
7445
7446static SDValue LowerBUILD_VECTORToVIDUP(SDValue Op, SelectionDAG &DAG,
7447 const ARMSubtarget *ST) {
7448 if (!ST->hasMVEIntegerOps())
7449 return SDValue();
7450
7451 // We are looking for a buildvector where each element is Op[0] + i*N
7452 EVT VT = Op.getValueType();
7453 SDValue Op0 = Op.getOperand(0);
7454 unsigned NumElts = VT.getVectorNumElements();
7455
7456 // Get the increment value from operand 1
7457 SDValue Op1 = Op.getOperand(1);
7458 if (Op1.getOpcode() != ISD::ADD || Op1.getOperand(0) != Op0 ||
7459 !isa<ConstantSDNode>(Op1.getOperand(1)))
7460 return SDValue();
7461 unsigned N = Op1.getConstantOperandVal(1);
7462 if (N != 1 && N != 2 && N != 4 && N != 8)
7463 return SDValue();
7464
7465 // Check that each other operand matches
7466 for (unsigned I = 2; I < NumElts; I++) {
7467 SDValue OpI = Op.getOperand(I);
7468 if (OpI.getOpcode() != ISD::ADD || OpI.getOperand(0) != Op0 ||
7469 !isa<ConstantSDNode>(OpI.getOperand(1)) ||
7470 OpI.getConstantOperandVal(1) != I * N)
7471 return SDValue();
7472 }
7473
7474 SDLoc DL(Op);
7475 return DAG.getNode(ARMISD::VIDUP, DL, DAG.getVTList(VT, MVT::i32), Op0,
7476 DAG.getConstant(N, DL, MVT::i32));
7477}
7478
7479// If this is a case we can't handle, return null and let the default
7480// expansion code take care of it.
7481SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
7482 const ARMSubtarget *ST) const {
7483 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Op.getNode());
7484 SDLoc dl(Op);
7485 EVT VT = Op.getValueType();
7486
7487 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
7488 return LowerBUILD_VECTOR_i1(Op, DAG, ST);
7489
7490 if (SDValue R = LowerBUILD_VECTORToVIDUP(Op, DAG, ST))
7491 return R;
7492
7493 APInt SplatBits, SplatUndef;
7494 unsigned SplatBitSize;
7495 bool HasAnyUndefs;
7496 if (BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
7497 if (SplatUndef.isAllOnesValue())
7498 return DAG.getUNDEF(VT);
7499
7500 if ((ST->hasNEON() && SplatBitSize <= 64) ||
7501 (ST->hasMVEIntegerOps() && SplatBitSize <= 64)) {
7502 // Check if an immediate VMOV works.
7503 EVT VmovVT;
7504 SDValue Val =
7505 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
7506 SplatBitSize, DAG, dl, VmovVT, VT, VMOVModImm);
7507
7508 if (Val.getNode()) {
7509 SDValue Vmov = DAG.getNode(ARMISD::VMOVIMM, dl, VmovVT, Val);
7510 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
7511 }
7512
7513 // Try an immediate VMVN.
7514 uint64_t NegatedImm = (~SplatBits).getZExtValue();
7515 Val = isVMOVModifiedImm(
7516 NegatedImm, SplatUndef.getZExtValue(), SplatBitSize, DAG, dl, VmovVT,
7517 VT, ST->hasMVEIntegerOps() ? MVEVMVNModImm : VMVNModImm);
7518 if (Val.getNode()) {
7519 SDValue Vmov = DAG.getNode(ARMISD::VMVNIMM, dl, VmovVT, Val);
7520 return DAG.getNode(ISD::BITCAST, dl, VT, Vmov);
7521 }
7522
7523 // Use vmov.f32 to materialize other v2f32 and v4f32 splats.
7524 if ((VT == MVT::v2f32 || VT == MVT::v4f32) && SplatBitSize == 32) {
7525 int ImmVal = ARM_AM::getFP32Imm(SplatBits);
7526 if (ImmVal != -1) {
7527 SDValue Val = DAG.getTargetConstant(ImmVal, dl, MVT::i32);
7528 return DAG.getNode(ARMISD::VMOVFPIMM, dl, VT, Val);
7529 }
7530 }
7531 }
7532 }
7533
7534 // Scan through the operands to see if only one value is used.
7535 //
7536 // As an optimisation, even if more than one value is used it may be more
7537 // profitable to splat with one value then change some lanes.
7538 //
7539 // Heuristically we decide to do this if the vector has a "dominant" value,
7540 // defined as splatted to more than half of the lanes.
7541 unsigned NumElts = VT.getVectorNumElements();
7542 bool isOnlyLowElement = true;
7543 bool usesOnlyOneValue = true;
7544 bool hasDominantValue = false;
7545 bool isConstant = true;
7546
7547 // Map of the number of times a particular SDValue appears in the
7548 // element list.
7549 DenseMap<SDValue, unsigned> ValueCounts;
7550 SDValue Value;
7551 for (unsigned i = 0; i < NumElts; ++i) {
7552 SDValue V = Op.getOperand(i);
7553 if (V.isUndef())
7554 continue;
7555 if (i > 0)
7556 isOnlyLowElement = false;
7557 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
7558 isConstant = false;
7559
7560 ValueCounts.insert(std::make_pair(V, 0));
7561 unsigned &Count = ValueCounts[V];
7562
7563 // Is this value dominant? (takes up more than half of the lanes)
7564 if (++Count > (NumElts / 2)) {
7565 hasDominantValue = true;
7566 Value = V;
7567 }
7568 }
7569 if (ValueCounts.size() != 1)
7570 usesOnlyOneValue = false;
7571 if (!Value.getNode() && !ValueCounts.empty())
7572 Value = ValueCounts.begin()->first;
7573
7574 if (ValueCounts.empty())
7575 return DAG.getUNDEF(VT);
7576
7577 // Loads are better lowered with insert_vector_elt/ARMISD::BUILD_VECTOR.
7578 // Keep going if we are hitting this case.
7579 if (isOnlyLowElement && !ISD::isNormalLoad(Value.getNode()))
7580 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value);
7581
7582 unsigned EltSize = VT.getScalarSizeInBits();
7583
7584 // Use VDUP for non-constant splats. For f32 constant splats, reduce to
7585 // i32 and try again.
7586 if (hasDominantValue && EltSize <= 32) {
7587 if (!isConstant) {
7588 SDValue N;
7589
7590 // If we are VDUPing a value that comes directly from a vector, that will
7591 // cause an unnecessary move to and from a GPR, where instead we could
7592 // just use VDUPLANE. We can only do this if the lane being extracted
7593 // is at a constant index, as the VDUP from lane instructions only have
7594 // constant-index forms.
7595 ConstantSDNode *constIndex;
7596 if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7597 (constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1)))) {
7598 // We need to create a new undef vector to use for the VDUPLANE if the
7599 // size of the vector from which we get the value is different than the
7600 // size of the vector that we need to create. We will insert the element
7601 // such that the register coalescer will remove unnecessary copies.
7602 if (VT != Value->getOperand(0).getValueType()) {
7603 unsigned index = constIndex->getAPIntValue().getLimitedValue() %
7604 VT.getVectorNumElements();
7605 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7606 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
7607 Value, DAG.getConstant(index, dl, MVT::i32)),
7608 DAG.getConstant(index, dl, MVT::i32));
7609 } else
7610 N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
7611 Value->getOperand(0), Value->getOperand(1));
7612 } else
7613 N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
7614
7615 if (!usesOnlyOneValue) {
7616 // The dominant value was splatted as 'N', but we now have to insert
7617 // all differing elements.
7618 for (unsigned I = 0; I < NumElts; ++I) {
7619 if (Op.getOperand(I) == Value)
7620 continue;
7621 SmallVector<SDValue, 3> Ops;
7622 Ops.push_back(N);
7623 Ops.push_back(Op.getOperand(I));
7624 Ops.push_back(DAG.getConstant(I, dl, MVT::i32));
7625 N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ops);
7626 }
7627 }
7628 return N;
7629 }
7630 if (VT.getVectorElementType().isFloatingPoint()) {
7631 SmallVector<SDValue, 8> Ops;
7632 MVT FVT = VT.getVectorElementType().getSimpleVT();
7633 assert(FVT == MVT::f32 || FVT == MVT::f16)(static_cast <bool> (FVT == MVT::f32 || FVT == MVT::f16
) ? void (0) : __assert_fail ("FVT == MVT::f32 || FVT == MVT::f16"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7633, __extension__ __PRETTY_FUNCTION__))
;
7634 MVT IVT = (FVT == MVT::f32) ? MVT::i32 : MVT::i16;
7635 for (unsigned i = 0; i < NumElts; ++i)
7636 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, IVT,
7637 Op.getOperand(i)));
7638 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), IVT, NumElts);
7639 SDValue Val = DAG.getBuildVector(VecVT, dl, Ops);
7640 Val = LowerBUILD_VECTOR(Val, DAG, ST);
7641 if (Val.getNode())
7642 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
7643 }
7644 if (usesOnlyOneValue) {
7645 SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
7646 if (isConstant && Val.getNode())
7647 return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
7648 }
7649 }
7650
7651 // If all elements are constants and the case above didn't get hit, fall back
7652 // to the default expansion, which will generate a load from the constant
7653 // pool.
7654 if (isConstant)
7655 return SDValue();
7656
7657 // Reconstruct the BUILDVECTOR to one of the legal shuffles (such as vext and
7658 // vmovn). Empirical tests suggest this is rarely worth it for vectors of
7659 // length <= 2.
7660 if (NumElts >= 4)
7661 if (SDValue shuffle = ReconstructShuffle(Op, DAG))
7662 return shuffle;
7663
7664 // Attempt to turn a buildvector of scalar fptrunc's or fpext's back into
7665 // VCVT's
7666 if (SDValue VCVT = LowerBuildVectorOfFPTrunc(Op, DAG, Subtarget))
7667 return VCVT;
7668 if (SDValue VCVT = LowerBuildVectorOfFPExt(Op, DAG, Subtarget))
7669 return VCVT;
7670
7671 if (ST->hasNEON() && VT.is128BitVector() && VT != MVT::v2f64 && VT != MVT::v4f32) {
7672 // If we haven't found an efficient lowering, try splitting a 128-bit vector
7673 // into two 64-bit vectors; we might discover a better way to lower it.
7674 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElts);
7675 EVT ExtVT = VT.getVectorElementType();
7676 EVT HVT = EVT::getVectorVT(*DAG.getContext(), ExtVT, NumElts / 2);
7677 SDValue Lower =
7678 DAG.getBuildVector(HVT, dl, makeArrayRef(&Ops[0], NumElts / 2));
7679 if (Lower.getOpcode() == ISD::BUILD_VECTOR)
7680 Lower = LowerBUILD_VECTOR(Lower, DAG, ST);
7681 SDValue Upper = DAG.getBuildVector(
7682 HVT, dl, makeArrayRef(&Ops[NumElts / 2], NumElts / 2));
7683 if (Upper.getOpcode() == ISD::BUILD_VECTOR)
7684 Upper = LowerBUILD_VECTOR(Upper, DAG, ST);
7685 if (Lower && Upper)
7686 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lower, Upper);
7687 }
7688
7689 // Vectors with 32- or 64-bit elements can be built by directly assigning
7690 // the subregisters. Lower it to an ARMISD::BUILD_VECTOR so the operands
7691 // will be legalized.
7692 if (EltSize >= 32) {
7693 // Do the expansion with floating-point types, since that is what the VFP
7694 // registers are defined to use, and since i64 is not legal.
7695 EVT EltVT = EVT::getFloatingPointVT(EltSize);
7696 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
7697 SmallVector<SDValue, 8> Ops;
7698 for (unsigned i = 0; i < NumElts; ++i)
7699 Ops.push_back(DAG.getNode(ISD::BITCAST, dl, EltVT, Op.getOperand(i)));
7700 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
7701 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
7702 }
7703
7704 // If all else fails, just use a sequence of INSERT_VECTOR_ELT when we
7705 // know the default expansion would otherwise fall back on something even
7706 // worse. For a vector with one or two non-undef values, that's
7707 // scalar_to_vector for the elements followed by a shuffle (provided the
7708 // shuffle is valid for the target) and materialization element by element
7709 // on the stack followed by a load for everything else.
7710 if (!isConstant && !usesOnlyOneValue) {
7711 SDValue Vec = DAG.getUNDEF(VT);
7712 for (unsigned i = 0 ; i < NumElts; ++i) {
7713 SDValue V = Op.getOperand(i);
7714 if (V.isUndef())
7715 continue;
7716 SDValue LaneIdx = DAG.getConstant(i, dl, MVT::i32);
7717 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Vec, V, LaneIdx);
7718 }
7719 return Vec;
7720 }
7721
7722 return SDValue();
7723}
7724
7725// Gather data to see if the operation can be modelled as a
7726// shuffle in combination with VEXTs.
7727SDValue ARMTargetLowering::ReconstructShuffle(SDValue Op,
7728 SelectionDAG &DAG) const {
7729 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unknown opcode!")(static_cast <bool> (Op.getOpcode() == ISD::BUILD_VECTOR
&& "Unknown opcode!") ? void (0) : __assert_fail ("Op.getOpcode() == ISD::BUILD_VECTOR && \"Unknown opcode!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7729, __extension__ __PRETTY_FUNCTION__))
;
7730 SDLoc dl(Op);
7731 EVT VT = Op.getValueType();
7732 unsigned NumElts = VT.getVectorNumElements();
7733
7734 struct ShuffleSourceInfo {
7735 SDValue Vec;
7736 unsigned MinElt = std::numeric_limits<unsigned>::max();
7737 unsigned MaxElt = 0;
7738
7739 // We may insert some combination of BITCASTs and VEXT nodes to force Vec to
7740 // be compatible with the shuffle we intend to construct. As a result
7741 // ShuffleVec will be some sliding window into the original Vec.
7742 SDValue ShuffleVec;
7743
7744 // Code should guarantee that element i in Vec starts at element "WindowBase
7745 // + i * WindowScale in ShuffleVec".
7746 int WindowBase = 0;
7747 int WindowScale = 1;
7748
7749 ShuffleSourceInfo(SDValue Vec) : Vec(Vec), ShuffleVec(Vec) {}
7750
7751 bool operator ==(SDValue OtherVec) { return Vec == OtherVec; }
7752 };
7753
7754 // First gather all vectors used as an immediate source for this BUILD_VECTOR
7755 // node.
7756 SmallVector<ShuffleSourceInfo, 2> Sources;
7757 for (unsigned i = 0; i < NumElts; ++i) {
7758 SDValue V = Op.getOperand(i);
7759 if (V.isUndef())
7760 continue;
7761 else if (V.getOpcode() != ISD::EXTRACT_VECTOR_ELT) {
7762 // A shuffle can only come from building a vector from various
7763 // elements of other vectors.
7764 return SDValue();
7765 } else if (!isa<ConstantSDNode>(V.getOperand(1))) {
7766 // Furthermore, shuffles require a constant mask, whereas extractelts
7767 // accept variable indices.
7768 return SDValue();
7769 }
7770
7771 // Add this element source to the list if it's not already there.
7772 SDValue SourceVec = V.getOperand(0);
7773 auto Source = llvm::find(Sources, SourceVec);
7774 if (Source == Sources.end())
7775 Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec));
7776
7777 // Update the minimum and maximum lane number seen.
7778 unsigned EltNo = cast<ConstantSDNode>(V.getOperand(1))->getZExtValue();
7779 Source->MinElt = std::min(Source->MinElt, EltNo);
7780 Source->MaxElt = std::max(Source->MaxElt, EltNo);
7781 }
7782
7783 // Currently only do something sane when at most two source vectors
7784 // are involved.
7785 if (Sources.size() > 2)
7786 return SDValue();
7787
7788 // Find out the smallest element size among result and two sources, and use
7789 // it as element size to build the shuffle_vector.
7790 EVT SmallestEltTy = VT.getVectorElementType();
7791 for (auto &Source : Sources) {
7792 EVT SrcEltTy = Source.Vec.getValueType().getVectorElementType();
7793 if (SrcEltTy.bitsLT(SmallestEltTy))
7794 SmallestEltTy = SrcEltTy;
7795 }
7796 unsigned ResMultiplier =
7797 VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits();
7798 NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits();
7799 EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts);
7800
7801 // If the source vector is too wide or too narrow, we may nevertheless be able
7802 // to construct a compatible shuffle either by concatenating it with UNDEF or
7803 // extracting a suitable range of elements.
7804 for (auto &Src : Sources) {
7805 EVT SrcVT = Src.ShuffleVec.getValueType();
7806
7807 uint64_t SrcVTSize = SrcVT.getFixedSizeInBits();
7808 uint64_t VTSize = VT.getFixedSizeInBits();
7809 if (SrcVTSize == VTSize)
7810 continue;
7811
7812 // This stage of the search produces a source with the same element type as
7813 // the original, but with a total width matching the BUILD_VECTOR output.
7814 EVT EltVT = SrcVT.getVectorElementType();
7815 unsigned NumSrcElts = VTSize / EltVT.getFixedSizeInBits();
7816 EVT DestVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumSrcElts);
7817
7818 if (SrcVTSize < VTSize) {
7819 if (2 * SrcVTSize != VTSize)
7820 return SDValue();
7821 // We can pad out the smaller vector for free, so if it's part of a
7822 // shuffle...
7823 Src.ShuffleVec =
7824 DAG.getNode(ISD::CONCAT_VECTORS, dl, DestVT, Src.ShuffleVec,
7825 DAG.getUNDEF(Src.ShuffleVec.getValueType()));
7826 continue;
7827 }
7828
7829 if (SrcVTSize != 2 * VTSize)
7830 return SDValue();
7831
7832 if (Src.MaxElt - Src.MinElt >= NumSrcElts) {
7833 // Span too large for a VEXT to cope
7834 return SDValue();
7835 }
7836
7837 if (Src.MinElt >= NumSrcElts) {
7838 // The extraction can just take the second half
7839 Src.ShuffleVec =
7840 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7841 DAG.getConstant(NumSrcElts, dl, MVT::i32));
7842 Src.WindowBase = -NumSrcElts;
7843 } else if (Src.MaxElt < NumSrcElts) {
7844 // The extraction can just take the first half
7845 Src.ShuffleVec =
7846 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7847 DAG.getConstant(0, dl, MVT::i32));
7848 } else {
7849 // An actual VEXT is needed
7850 SDValue VEXTSrc1 =
7851 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7852 DAG.getConstant(0, dl, MVT::i32));
7853 SDValue VEXTSrc2 =
7854 DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT, Src.ShuffleVec,
7855 DAG.getConstant(NumSrcElts, dl, MVT::i32));
7856
7857 Src.ShuffleVec = DAG.getNode(ARMISD::VEXT, dl, DestVT, VEXTSrc1,
7858 VEXTSrc2,
7859 DAG.getConstant(Src.MinElt, dl, MVT::i32));
7860 Src.WindowBase = -Src.MinElt;
7861 }
7862 }
7863
7864 // Another possible incompatibility occurs from the vector element types. We
7865 // can fix this by bitcasting the source vectors to the same type we intend
7866 // for the shuffle.
7867 for (auto &Src : Sources) {
7868 EVT SrcEltTy = Src.ShuffleVec.getValueType().getVectorElementType();
7869 if (SrcEltTy == SmallestEltTy)
7870 continue;
7871 assert(ShuffleVT.getVectorElementType() == SmallestEltTy)(static_cast <bool> (ShuffleVT.getVectorElementType() ==
SmallestEltTy) ? void (0) : __assert_fail ("ShuffleVT.getVectorElementType() == SmallestEltTy"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7871, __extension__ __PRETTY_FUNCTION__))
;
7872 Src.ShuffleVec = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, ShuffleVT, Src.ShuffleVec);
7873 Src.WindowScale = SrcEltTy.getSizeInBits() / SmallestEltTy.getSizeInBits();
7874 Src.WindowBase *= Src.WindowScale;
7875 }
7876
7877 // Final sanity check before we try to actually produce a shuffle.
7878 LLVM_DEBUG(for (auto Srcdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { for (auto Src : Sources) (static_cast <bool
> (Src.ShuffleVec.getValueType() == ShuffleVT) ? void (0) :
__assert_fail ("Src.ShuffleVec.getValueType() == ShuffleVT",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7880, __extension__ __PRETTY_FUNCTION__));; } } while (false
)
7879 : Sources)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { for (auto Src : Sources) (static_cast <bool
> (Src.ShuffleVec.getValueType() == ShuffleVT) ? void (0) :
__assert_fail ("Src.ShuffleVec.getValueType() == ShuffleVT",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7880, __extension__ __PRETTY_FUNCTION__));; } } while (false
)
7880 assert(Src.ShuffleVec.getValueType() == ShuffleVT);)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { for (auto Src : Sources) (static_cast <bool
> (Src.ShuffleVec.getValueType() == ShuffleVT) ? void (0) :
__assert_fail ("Src.ShuffleVec.getValueType() == ShuffleVT",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7880, __extension__ __PRETTY_FUNCTION__));; } } while (false
)
;
7881
7882 // The stars all align, our next step is to produce the mask for the shuffle.
7883 SmallVector<int, 8> Mask(ShuffleVT.getVectorNumElements(), -1);
7884 int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits();
7885 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
7886 SDValue Entry = Op.getOperand(i);
7887 if (Entry.isUndef())
7888 continue;
7889
7890 auto Src = llvm::find(Sources, Entry.getOperand(0));
7891 int EltNo = cast<ConstantSDNode>(Entry.getOperand(1))->getSExtValue();
7892
7893 // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit
7894 // trunc. So only std::min(SrcBits, DestBits) actually get defined in this
7895 // segment.
7896 EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType();
7897 int BitsDefined = std::min(OrigEltTy.getScalarSizeInBits(),
7898 VT.getScalarSizeInBits());
7899 int LanesDefined = BitsDefined / BitsPerShuffleLane;
7900
7901 // This source is expected to fill ResMultiplier lanes of the final shuffle,
7902 // starting at the appropriate offset.
7903 int *LaneMask = &Mask[i * ResMultiplier];
7904
7905 int ExtractBase = EltNo * Src->WindowScale + Src->WindowBase;
7906 ExtractBase += NumElts * (Src - Sources.begin());
7907 for (int j = 0; j < LanesDefined; ++j)
7908 LaneMask[j] = ExtractBase + j;
7909 }
7910
7911
7912 // We can't handle more than two sources. This should have already
7913 // been checked before this point.
7914 assert(Sources.size() <= 2 && "Too many sources!")(static_cast <bool> (Sources.size() <= 2 && "Too many sources!"
) ? void (0) : __assert_fail ("Sources.size() <= 2 && \"Too many sources!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 7914, __extension__ __PRETTY_FUNCTION__))
;
7915
7916 SDValue ShuffleOps[] = { DAG.getUNDEF(ShuffleVT), DAG.getUNDEF(ShuffleVT) };
7917 for (unsigned i = 0; i < Sources.size(); ++i)
7918 ShuffleOps[i] = Sources[i].ShuffleVec;
7919
7920 SDValue Shuffle = buildLegalVectorShuffle(ShuffleVT, dl, ShuffleOps[0],
7921 ShuffleOps[1], Mask, DAG);
7922 if (!Shuffle)
7923 return SDValue();
7924 return DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Shuffle);
7925}
7926
7927enum ShuffleOpCodes {
7928 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
7929 OP_VREV,
7930 OP_VDUP0,
7931 OP_VDUP1,
7932 OP_VDUP2,
7933 OP_VDUP3,
7934 OP_VEXT1,
7935 OP_VEXT2,
7936 OP_VEXT3,
7937 OP_VUZPL, // VUZP, left result
7938 OP_VUZPR, // VUZP, right result
7939 OP_VZIPL, // VZIP, left result
7940 OP_VZIPR, // VZIP, right result
7941 OP_VTRNL, // VTRN, left result
7942 OP_VTRNR // VTRN, right result
7943};
7944
7945static bool isLegalMVEShuffleOp(unsigned PFEntry) {
7946 unsigned OpNum = (PFEntry >> 26) & 0x0F;
7947 switch (OpNum) {
7948 case OP_COPY:
7949 case OP_VREV:
7950 case OP_VDUP0:
7951 case OP_VDUP1:
7952 case OP_VDUP2:
7953 case OP_VDUP3:
7954 return true;
7955 }
7956 return false;
7957}
7958
7959/// isShuffleMaskLegal - Targets can use this to indicate that they only
7960/// support *some* VECTOR_SHUFFLE operations, those with specific masks.
7961/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
7962/// are assumed to be legal.
7963bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7964 if (VT.getVectorNumElements() == 4 &&
7965 (VT.is128BitVector() || VT.is64BitVector())) {
7966 unsigned PFIndexes[4];
7967 for (unsigned i = 0; i != 4; ++i) {
7968 if (M[i] < 0)
7969 PFIndexes[i] = 8;
7970 else
7971 PFIndexes[i] = M[i];
7972 }
7973
7974 // Compute the index in the perfect shuffle table.
7975 unsigned PFTableIndex =
7976 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
7977 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
7978 unsigned Cost = (PFEntry >> 30);
7979
7980 if (Cost <= 4 && (Subtarget->hasNEON() || isLegalMVEShuffleOp(PFEntry)))
7981 return true;
7982 }
7983
7984 bool ReverseVEXT, isV_UNDEF;
7985 unsigned Imm, WhichResult;
7986
7987 unsigned EltSize = VT.getScalarSizeInBits();
7988 if (EltSize >= 32 ||
7989 ShuffleVectorSDNode::isSplatMask(&M[0], VT) ||
7990 ShuffleVectorInst::isIdentityMask(M) ||
7991 isVREVMask(M, VT, 64) ||
7992 isVREVMask(M, VT, 32) ||
7993 isVREVMask(M, VT, 16))
7994 return true;
7995 else if (Subtarget->hasNEON() &&
7996 (isVEXTMask(M, VT, ReverseVEXT, Imm) ||
7997 isVTBLMask(M, VT) ||
7998 isNEONTwoResultShuffleMask(M, VT, WhichResult, isV_UNDEF)))
7999 return true;
8000 else if (Subtarget->hasNEON() && (VT == MVT::v8i16 || VT == MVT::v16i8) &&
8001 isReverseMask(M, VT))
8002 return true;
8003 else if (Subtarget->hasMVEIntegerOps() &&
8004 (isVMOVNMask(M, VT, true, false) ||
8005 isVMOVNMask(M, VT, false, false) || isVMOVNMask(M, VT, true, true)))
8006 return true;
8007 else
8008 return false;
8009}
8010
8011/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
8012/// the specified operations to build the shuffle.
8013static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
8014 SDValue RHS, SelectionDAG &DAG,
8015 const SDLoc &dl) {
8016 unsigned OpNum = (PFEntry >> 26) & 0x0F;
8017 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8018 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8019
8020 if (OpNum == OP_COPY) {
8021 if (LHSID == (1*9+2)*9+3) return LHS;
8022 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!")(static_cast <bool> (LHSID == ((4*9+5)*9+6)*9+7 &&
"Illegal OP_COPY!") ? void (0) : __assert_fail ("LHSID == ((4*9+5)*9+6)*9+7 && \"Illegal OP_COPY!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8022, __extension__ __PRETTY_FUNCTION__))
;
8023 return RHS;
8024 }
8025
8026 SDValue OpLHS, OpRHS;
8027 OpLHS = GeneratePerfectShuffle(PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
8028 OpRHS = GeneratePerfectShuffle(PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
8029 EVT VT = OpLHS.getValueType();
8030
8031 switch (OpNum) {
8032 default: llvm_unreachable("Unknown shuffle opcode!")::llvm::llvm_unreachable_internal("Unknown shuffle opcode!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8032)
;
8033 case OP_VREV:
8034 // VREV divides the vector in half and swaps within the half.
8035 if (VT.getVectorElementType() == MVT::i32 ||
8036 VT.getVectorElementType() == MVT::f32)
8037 return DAG.getNode(ARMISD::VREV64, dl, VT, OpLHS);
8038 // vrev <4 x i16> -> VREV32
8039 if (VT.getVectorElementType() == MVT::i16 ||
8040 VT.getVectorElementType() == MVT::f16)
8041 return DAG.getNode(ARMISD::VREV32, dl, VT, OpLHS);
8042 // vrev <4 x i8> -> VREV16
8043 assert(VT.getVectorElementType() == MVT::i8)(static_cast <bool> (VT.getVectorElementType() == MVT::
i8) ? void (0) : __assert_fail ("VT.getVectorElementType() == MVT::i8"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8043, __extension__ __PRETTY_FUNCTION__))
;
8044 return DAG.getNode(ARMISD::VREV16, dl, VT, OpLHS);
8045 case OP_VDUP0:
8046 case OP_VDUP1:
8047 case OP_VDUP2:
8048 case OP_VDUP3:
8049 return DAG.getNode(ARMISD::VDUPLANE, dl, VT,
8050 OpLHS, DAG.getConstant(OpNum-OP_VDUP0, dl, MVT::i32));
8051 case OP_VEXT1:
8052 case OP_VEXT2:
8053 case OP_VEXT3:
8054 return DAG.getNode(ARMISD::VEXT, dl, VT,
8055 OpLHS, OpRHS,
8056 DAG.getConstant(OpNum - OP_VEXT1 + 1, dl, MVT::i32));
8057 case OP_VUZPL:
8058 case OP_VUZPR:
8059 return DAG.getNode(ARMISD::VUZP, dl, DAG.getVTList(VT, VT),
8060 OpLHS, OpRHS).getValue(OpNum-OP_VUZPL);
8061 case OP_VZIPL:
8062 case OP_VZIPR:
8063 return DAG.getNode(ARMISD::VZIP, dl, DAG.getVTList(VT, VT),
8064 OpLHS, OpRHS).getValue(OpNum-OP_VZIPL);
8065 case OP_VTRNL:
8066 case OP_VTRNR:
8067 return DAG.getNode(ARMISD::VTRN, dl, DAG.getVTList(VT, VT),
8068 OpLHS, OpRHS).getValue(OpNum-OP_VTRNL);
8069 }
8070}
8071
8072static SDValue LowerVECTOR_SHUFFLEv8i8(SDValue Op,
8073 ArrayRef<int> ShuffleMask,
8074 SelectionDAG &DAG) {
8075 // Check to see if we can use the VTBL instruction.
8076 SDValue V1 = Op.getOperand(0);
8077 SDValue V2 = Op.getOperand(1);
8078 SDLoc DL(Op);
8079
8080 SmallVector<SDValue, 8> VTBLMask;
8081 for (ArrayRef<int>::iterator
8082 I = ShuffleMask.begin(), E = ShuffleMask.end(); I != E; ++I)
8083 VTBLMask.push_back(DAG.getConstant(*I, DL, MVT::i32));
8084
8085 if (V2.getNode()->isUndef())
8086 return DAG.getNode(ARMISD::VTBL1, DL, MVT::v8i8, V1,
8087 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8088
8089 return DAG.getNode(ARMISD::VTBL2, DL, MVT::v8i8, V1, V2,
8090 DAG.getBuildVector(MVT::v8i8, DL, VTBLMask));
8091}
8092
8093static SDValue LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(SDValue Op,
8094 SelectionDAG &DAG) {
8095 SDLoc DL(Op);
8096 SDValue OpLHS = Op.getOperand(0);
8097 EVT VT = OpLHS.getValueType();
8098
8099 assert((VT == MVT::v8i16 || VT == MVT::v16i8) &&(static_cast <bool> ((VT == MVT::v8i16 || VT == MVT::v16i8
) && "Expect an v8i16/v16i8 type") ? void (0) : __assert_fail
("(VT == MVT::v8i16 || VT == MVT::v16i8) && \"Expect an v8i16/v16i8 type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8100, __extension__ __PRETTY_FUNCTION__))
8100 "Expect an v8i16/v16i8 type")(static_cast <bool> ((VT == MVT::v8i16 || VT == MVT::v16i8
) && "Expect an v8i16/v16i8 type") ? void (0) : __assert_fail
("(VT == MVT::v8i16 || VT == MVT::v16i8) && \"Expect an v8i16/v16i8 type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8100, __extension__ __PRETTY_FUNCTION__))
;
8101 OpLHS = DAG.getNode(ARMISD::VREV64, DL, VT, OpLHS);
8102 // For a v16i8 type: After the VREV, we have got <8, ...15, 8, ..., 0>. Now,
8103 // extract the first 8 bytes into the top double word and the last 8 bytes
8104 // into the bottom double word. The v8i16 case is similar.
8105 unsigned ExtractNum = (VT == MVT::v16i8) ? 8 : 4;
8106 return DAG.getNode(ARMISD::VEXT, DL, VT, OpLHS, OpLHS,
8107 DAG.getConstant(ExtractNum, DL, MVT::i32));
8108}
8109
8110static EVT getVectorTyFromPredicateVector(EVT VT) {
8111 switch (VT.getSimpleVT().SimpleTy) {
8112 case MVT::v4i1:
8113 return MVT::v4i32;
8114 case MVT::v8i1:
8115 return MVT::v8i16;
8116 case MVT::v16i1:
8117 return MVT::v16i8;
8118 default:
8119 llvm_unreachable("Unexpected vector predicate type")::llvm::llvm_unreachable_internal("Unexpected vector predicate type"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8119)
;
8120 }
8121}
8122
8123static SDValue PromoteMVEPredVector(SDLoc dl, SDValue Pred, EVT VT,
8124 SelectionDAG &DAG) {
8125 // Converting from boolean predicates to integers involves creating a vector
8126 // of all ones or all zeroes and selecting the lanes based upon the real
8127 // predicate.
8128 SDValue AllOnes =
8129 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0xff), dl, MVT::i32);
8130 AllOnes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllOnes);
8131
8132 SDValue AllZeroes =
8133 DAG.getTargetConstant(ARM_AM::createVMOVModImm(0xe, 0x0), dl, MVT::i32);
8134 AllZeroes = DAG.getNode(ARMISD::VMOVIMM, dl, MVT::v16i8, AllZeroes);
8135
8136 // Get full vector type from predicate type
8137 EVT NewVT = getVectorTyFromPredicateVector(VT);
8138
8139 SDValue RecastV1;
8140 // If the real predicate is an v8i1 or v4i1 (not v16i1) then we need to recast
8141 // this to a v16i1. This cannot be done with an ordinary bitcast because the
8142 // sizes are not the same. We have to use a MVE specific PREDICATE_CAST node,
8143 // since we know in hardware the sizes are really the same.
8144 if (VT != MVT::v16i1)
8145 RecastV1 = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Pred);
8146 else
8147 RecastV1 = Pred;
8148
8149 // Select either all ones or zeroes depending upon the real predicate bits.
8150 SDValue PredAsVector =
8151 DAG.getNode(ISD::VSELECT, dl, MVT::v16i8, RecastV1, AllOnes, AllZeroes);
8152
8153 // Recast our new predicate-as-integer v16i8 vector into something
8154 // appropriate for the shuffle, i.e. v4i32 for a real v4i1 predicate.
8155 return DAG.getNode(ISD::BITCAST, dl, NewVT, PredAsVector);
8156}
8157
8158static SDValue LowerVECTOR_SHUFFLE_i1(SDValue Op, SelectionDAG &DAG,
8159 const ARMSubtarget *ST) {
8160 EVT VT = Op.getValueType();
8161 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
8162 ArrayRef<int> ShuffleMask = SVN->getMask();
8163
8164 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"No support for vector shuffle of boolean predicates") ? void
(0) : __assert_fail ("ST->hasMVEIntegerOps() && \"No support for vector shuffle of boolean predicates\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8165, __extension__ __PRETTY_FUNCTION__))
8165 "No support for vector shuffle of boolean predicates")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"No support for vector shuffle of boolean predicates") ? void
(0) : __assert_fail ("ST->hasMVEIntegerOps() && \"No support for vector shuffle of boolean predicates\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8165, __extension__ __PRETTY_FUNCTION__))
;
8166
8167 SDValue V1 = Op.getOperand(0);
8168 SDLoc dl(Op);
8169 if (isReverseMask(ShuffleMask, VT)) {
8170 SDValue cast = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, V1);
8171 SDValue rbit = DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, cast);
8172 SDValue srl = DAG.getNode(ISD::SRL, dl, MVT::i32, rbit,
8173 DAG.getConstant(16, dl, MVT::i32));
8174 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, srl);
8175 }
8176
8177 // Until we can come up with optimised cases for every single vector
8178 // shuffle in existence we have chosen the least painful strategy. This is
8179 // to essentially promote the boolean predicate to a 8-bit integer, where
8180 // each predicate represents a byte. Then we fall back on a normal integer
8181 // vector shuffle and convert the result back into a predicate vector. In
8182 // many cases the generated code might be even better than scalar code
8183 // operating on bits. Just imagine trying to shuffle 8 arbitrary 2-bit
8184 // fields in a register into 8 other arbitrary 2-bit fields!
8185 SDValue PredAsVector = PromoteMVEPredVector(dl, V1, VT, DAG);
8186 EVT NewVT = PredAsVector.getValueType();
8187
8188 // Do the shuffle!
8189 SDValue Shuffled = DAG.getVectorShuffle(NewVT, dl, PredAsVector,
8190 DAG.getUNDEF(NewVT), ShuffleMask);
8191
8192 // Now return the result of comparing the shuffled vector with zero,
8193 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8194 return DAG.getNode(ARMISD::VCMPZ, dl, VT, Shuffled,
8195 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8196}
8197
8198static SDValue LowerVECTOR_SHUFFLEUsingMovs(SDValue Op,
8199 ArrayRef<int> ShuffleMask,
8200 SelectionDAG &DAG) {
8201 // Attempt to lower the vector shuffle using as many whole register movs as
8202 // possible. This is useful for types smaller than 32bits, which would
8203 // often otherwise become a series for grp movs.
8204 SDLoc dl(Op);
8205 EVT VT = Op.getValueType();
8206 if (VT.getScalarSizeInBits() >= 32)
8207 return SDValue();
8208
8209 assert((VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) &&(static_cast <bool> ((VT == MVT::v8i16 || VT == MVT::v8f16
|| VT == MVT::v16i8) && "Unexpected vector type") ? void
(0) : __assert_fail ("(VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8210, __extension__ __PRETTY_FUNCTION__))
8210 "Unexpected vector type")(static_cast <bool> ((VT == MVT::v8i16 || VT == MVT::v8f16
|| VT == MVT::v16i8) && "Unexpected vector type") ? void
(0) : __assert_fail ("(VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i8) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8210, __extension__ __PRETTY_FUNCTION__))
;
8211 int NumElts = VT.getVectorNumElements();
8212 int QuarterSize = NumElts / 4;
8213 // The four final parts of the vector, as i32's
8214 SDValue Parts[4];
8215
8216 // Look for full lane vmovs like <0,1,2,3> or <u,5,6,7> etc, (but not
8217 // <u,u,u,u>), returning the vmov lane index
8218 auto getMovIdx = [](ArrayRef<int> ShuffleMask, int Start, int Length) {
8219 // Detect which mov lane this would be from the first non-undef element.
8220 int MovIdx = -1;
8221 for (int i = 0; i < Length; i++) {
8222 if (ShuffleMask[Start + i] >= 0) {
8223 if (ShuffleMask[Start + i] % Length != i)
8224 return -1;
8225 MovIdx = ShuffleMask[Start + i] / Length;
8226 break;
8227 }
8228 }
8229 // If all items are undef, leave this for other combines
8230 if (MovIdx == -1)
8231 return -1;
8232 // Check the remaining values are the correct part of the same mov
8233 for (int i = 1; i < Length; i++) {
8234 if (ShuffleMask[Start + i] >= 0 &&
8235 (ShuffleMask[Start + i] / Length != MovIdx ||
8236 ShuffleMask[Start + i] % Length != i))
8237 return -1;
8238 }
8239 return MovIdx;
8240 };
8241
8242 for (int Part = 0; Part < 4; ++Part) {
8243 // Does this part look like a mov
8244 int Elt = getMovIdx(ShuffleMask, Part * QuarterSize, QuarterSize);
8245 if (Elt != -1) {
8246 SDValue Input = Op->getOperand(0);
8247 if (Elt >= 4) {
8248 Input = Op->getOperand(1);
8249 Elt -= 4;
8250 }
8251 SDValue BitCast = DAG.getBitcast(MVT::v4f32, Input);
8252 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32, BitCast,
8253 DAG.getConstant(Elt, dl, MVT::i32));
8254 }
8255 }
8256
8257 // Nothing interesting found, just return
8258 if (!Parts[0] && !Parts[1] && !Parts[2] && !Parts[3])
8259 return SDValue();
8260
8261 // The other parts need to be built with the old shuffle vector, cast to a
8262 // v4i32 and extract_vector_elts
8263 if (!Parts[0] || !Parts[1] || !Parts[2] || !Parts[3]) {
8264 SmallVector<int, 16> NewShuffleMask;
8265 for (int Part = 0; Part < 4; ++Part)
8266 for (int i = 0; i < QuarterSize; i++)
8267 NewShuffleMask.push_back(
8268 Parts[Part] ? -1 : ShuffleMask[Part * QuarterSize + i]);
8269 SDValue NewShuffle = DAG.getVectorShuffle(
8270 VT, dl, Op->getOperand(0), Op->getOperand(1), NewShuffleMask);
8271 SDValue BitCast = DAG.getBitcast(MVT::v4f32, NewShuffle);
8272
8273 for (int Part = 0; Part < 4; ++Part)
8274 if (!Parts[Part])
8275 Parts[Part] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f32,
8276 BitCast, DAG.getConstant(Part, dl, MVT::i32));
8277 }
8278 // Build a vector out of the various parts and bitcast it back to the original
8279 // type.
8280 SDValue NewVec = DAG.getNode(ARMISD::BUILD_VECTOR, dl, MVT::v4f32, Parts);
8281 return DAG.getBitcast(VT, NewVec);
8282}
8283
8284static SDValue LowerVECTOR_SHUFFLEUsingOneOff(SDValue Op,
8285 ArrayRef<int> ShuffleMask,
8286 SelectionDAG &DAG) {
8287 SDValue V1 = Op.getOperand(0);
8288 SDValue V2 = Op.getOperand(1);
8289 EVT VT = Op.getValueType();
8290 unsigned NumElts = VT.getVectorNumElements();
8291
8292 // An One-Off Identity mask is one that is mostly an identity mask from as
8293 // single source but contains a single element out-of-place, either from a
8294 // different vector or from another position in the same vector. As opposed to
8295 // lowering this via a ARMISD::BUILD_VECTOR we can generate an extract/insert
8296 // pair directly.
8297 auto isOneOffIdentityMask = [](ArrayRef<int> Mask, EVT VT, int BaseOffset,
8298 int &OffElement) {
8299 OffElement = -1;
8300 int NonUndef = 0;
8301 for (int i = 0, NumMaskElts = Mask.size(); i < NumMaskElts; ++i) {
8302 if (Mask[i] == -1)
8303 continue;
8304 NonUndef++;
8305 if (Mask[i] != i + BaseOffset) {
8306 if (OffElement == -1)
8307 OffElement = i;
8308 else
8309 return false;
8310 }
8311 }
8312 return NonUndef > 2 && OffElement != -1;
8313 };
8314 int OffElement;
8315 SDValue VInput;
8316 if (isOneOffIdentityMask(ShuffleMask, VT, 0, OffElement))
8317 VInput = V1;
8318 else if (isOneOffIdentityMask(ShuffleMask, VT, NumElts, OffElement))
8319 VInput = V2;
8320 else
8321 return SDValue();
8322
8323 SDLoc dl(Op);
8324 EVT SVT = VT.getScalarType() == MVT::i8 || VT.getScalarType() == MVT::i16
8325 ? MVT::i32
8326 : VT.getScalarType();
8327 SDValue Elt = DAG.getNode(
8328 ISD::EXTRACT_VECTOR_ELT, dl, SVT,
8329 ShuffleMask[OffElement] < (int)NumElts ? V1 : V2,
8330 DAG.getVectorIdxConstant(ShuffleMask[OffElement] % NumElts, dl));
8331 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, VInput, Elt,
8332 DAG.getVectorIdxConstant(OffElement % NumElts, dl));
8333}
8334
8335static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
8336 const ARMSubtarget *ST) {
8337 SDValue V1 = Op.getOperand(0);
8338 SDValue V2 = Op.getOperand(1);
8339 SDLoc dl(Op);
8340 EVT VT = Op.getValueType();
8341 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
8342 unsigned EltSize = VT.getScalarSizeInBits();
8343
8344 if (ST->hasMVEIntegerOps() && EltSize == 1)
8345 return LowerVECTOR_SHUFFLE_i1(Op, DAG, ST);
8346
8347 // Convert shuffles that are directly supported on NEON to target-specific
8348 // DAG nodes, instead of keeping them as shuffles and matching them again
8349 // during code selection. This is more efficient and avoids the possibility
8350 // of inconsistencies between legalization and selection.
8351 // FIXME: floating-point vectors should be canonicalized to integer vectors
8352 // of the same time so that they get CSEd properly.
8353 ArrayRef<int> ShuffleMask = SVN->getMask();
8354
8355 if (EltSize <= 32) {
8356 if (SVN->isSplat()) {
8357 int Lane = SVN->getSplatIndex();
8358 // If this is undef splat, generate it via "just" vdup, if possible.
8359 if (Lane == -1) Lane = 0;
8360
8361 // Test if V1 is a SCALAR_TO_VECTOR.
8362 if (Lane == 0 && V1.getOpcode() == ISD::SCALAR_TO_VECTOR) {
8363 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8364 }
8365 // Test if V1 is a BUILD_VECTOR which is equivalent to a SCALAR_TO_VECTOR
8366 // (and probably will turn into a SCALAR_TO_VECTOR once legalization
8367 // reaches it).
8368 if (Lane == 0 && V1.getOpcode() == ISD::BUILD_VECTOR &&
8369 !isa<ConstantSDNode>(V1.getOperand(0))) {
8370 bool IsScalarToVector = true;
8371 for (unsigned i = 1, e = V1.getNumOperands(); i != e; ++i)
8372 if (!V1.getOperand(i).isUndef()) {
8373 IsScalarToVector = false;
8374 break;
8375 }
8376 if (IsScalarToVector)
8377 return DAG.getNode(ARMISD::VDUP, dl, VT, V1.getOperand(0));
8378 }
8379 return DAG.getNode(ARMISD::VDUPLANE, dl, VT, V1,
8380 DAG.getConstant(Lane, dl, MVT::i32));
8381 }
8382
8383 bool ReverseVEXT = false;
8384 unsigned Imm = 0;
8385 if (ST->hasNEON() && isVEXTMask(ShuffleMask, VT, ReverseVEXT, Imm)) {
8386 if (ReverseVEXT)
8387 std::swap(V1, V2);
8388 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V2,
8389 DAG.getConstant(Imm, dl, MVT::i32));
8390 }
8391
8392 if (isVREVMask(ShuffleMask, VT, 64))
8393 return DAG.getNode(ARMISD::VREV64, dl, VT, V1);
8394 if (isVREVMask(ShuffleMask, VT, 32))
8395 return DAG.getNode(ARMISD::VREV32, dl, VT, V1);
8396 if (isVREVMask(ShuffleMask, VT, 16))
8397 return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
8398
8399 if (ST->hasNEON() && V2->isUndef() && isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
8400 return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
8401 DAG.getConstant(Imm, dl, MVT::i32));
8402 }
8403
8404 // Check for Neon shuffles that modify both input vectors in place.
8405 // If both results are used, i.e., if there are two shuffles with the same
8406 // source operands and with masks corresponding to both results of one of
8407 // these operations, DAG memoization will ensure that a single node is
8408 // used for both shuffles.
8409 unsigned WhichResult = 0;
8410 bool isV_UNDEF = false;
8411 if (ST->hasNEON()) {
8412 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8413 ShuffleMask, VT, WhichResult, isV_UNDEF)) {
8414 if (isV_UNDEF)
8415 V2 = V1;
8416 return DAG.getNode(ShuffleOpc, dl, DAG.getVTList(VT, VT), V1, V2)
8417 .getValue(WhichResult);
8418 }
8419 }
8420 if (ST->hasMVEIntegerOps()) {
8421 if (isVMOVNMask(ShuffleMask, VT, false, false))
8422 return DAG.getNode(ARMISD::VMOVN, dl, VT, V2, V1,
8423 DAG.getConstant(0, dl, MVT::i32));
8424 if (isVMOVNMask(ShuffleMask, VT, true, false))
8425 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V2,
8426 DAG.getConstant(1, dl, MVT::i32));
8427 if (isVMOVNMask(ShuffleMask, VT, true, true))
8428 return DAG.getNode(ARMISD::VMOVN, dl, VT, V1, V1,
8429 DAG.getConstant(1, dl, MVT::i32));
8430 }
8431
8432 // Also check for these shuffles through CONCAT_VECTORS: we canonicalize
8433 // shuffles that produce a result larger than their operands with:
8434 // shuffle(concat(v1, undef), concat(v2, undef))
8435 // ->
8436 // shuffle(concat(v1, v2), undef)
8437 // because we can access quad vectors (see PerformVECTOR_SHUFFLECombine).
8438 //
8439 // This is useful in the general case, but there are special cases where
8440 // native shuffles produce larger results: the two-result ops.
8441 //
8442 // Look through the concat when lowering them:
8443 // shuffle(concat(v1, v2), undef)
8444 // ->
8445 // concat(VZIP(v1, v2):0, :1)
8446 //
8447 if (ST->hasNEON() && V1->getOpcode() == ISD::CONCAT_VECTORS && V2->isUndef()) {
8448 SDValue SubV1 = V1->getOperand(0);
8449 SDValue SubV2 = V1->getOperand(1);
8450 EVT SubVT = SubV1.getValueType();
8451
8452 // We expect these to have been canonicalized to -1.
8453 assert(llvm::all_of(ShuffleMask, [&](int i) {(static_cast <bool> (llvm::all_of(ShuffleMask, [&](
int i) { return i < (int)VT.getVectorNumElements(); }) &&
"Unexpected shuffle index into UNDEF operand!") ? void (0) :
__assert_fail ("llvm::all_of(ShuffleMask, [&](int i) { return i < (int)VT.getVectorNumElements(); }) && \"Unexpected shuffle index into UNDEF operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8455, __extension__ __PRETTY_FUNCTION__))
8454 return i < (int)VT.getVectorNumElements();(static_cast <bool> (llvm::all_of(ShuffleMask, [&](
int i) { return i < (int)VT.getVectorNumElements(); }) &&
"Unexpected shuffle index into UNDEF operand!") ? void (0) :
__assert_fail ("llvm::all_of(ShuffleMask, [&](int i) { return i < (int)VT.getVectorNumElements(); }) && \"Unexpected shuffle index into UNDEF operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8455, __extension__ __PRETTY_FUNCTION__))
8455 }) && "Unexpected shuffle index into UNDEF operand!")(static_cast <bool> (llvm::all_of(ShuffleMask, [&](
int i) { return i < (int)VT.getVectorNumElements(); }) &&
"Unexpected shuffle index into UNDEF operand!") ? void (0) :
__assert_fail ("llvm::all_of(ShuffleMask, [&](int i) { return i < (int)VT.getVectorNumElements(); }) && \"Unexpected shuffle index into UNDEF operand!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8455, __extension__ __PRETTY_FUNCTION__))
;
8456
8457 if (unsigned ShuffleOpc = isNEONTwoResultShuffleMask(
8458 ShuffleMask, SubVT, WhichResult, isV_UNDEF)) {
8459 if (isV_UNDEF)
8460 SubV2 = SubV1;
8461 assert((WhichResult == 0) &&(static_cast <bool> ((WhichResult == 0) && "In-place shuffle of concat can only have one result!"
) ? void (0) : __assert_fail ("(WhichResult == 0) && \"In-place shuffle of concat can only have one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8462, __extension__ __PRETTY_FUNCTION__))
8462 "In-place shuffle of concat can only have one result!")(static_cast <bool> ((WhichResult == 0) && "In-place shuffle of concat can only have one result!"
) ? void (0) : __assert_fail ("(WhichResult == 0) && \"In-place shuffle of concat can only have one result!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8462, __extension__ __PRETTY_FUNCTION__))
;
8463 SDValue Res = DAG.getNode(ShuffleOpc, dl, DAG.getVTList(SubVT, SubVT),
8464 SubV1, SubV2);
8465 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Res.getValue(0),
8466 Res.getValue(1));
8467 }
8468 }
8469 }
8470
8471 if (ST->hasMVEIntegerOps() && EltSize <= 32)
8472 if (SDValue V = LowerVECTOR_SHUFFLEUsingOneOff(Op, ShuffleMask, DAG))
8473 return V;
8474
8475 // If the shuffle is not directly supported and it has 4 elements, use
8476 // the PerfectShuffle-generated table to synthesize it from other shuffles.
8477 unsigned NumElts = VT.getVectorNumElements();
8478 if (NumElts == 4) {
8479 unsigned PFIndexes[4];
8480 for (unsigned i = 0; i != 4; ++i) {
8481 if (ShuffleMask[i] < 0)
8482 PFIndexes[i] = 8;
8483 else
8484 PFIndexes[i] = ShuffleMask[i];
8485 }
8486
8487 // Compute the index in the perfect shuffle table.
8488 unsigned PFTableIndex =
8489 PFIndexes[0]*9*9*9+PFIndexes[1]*9*9+PFIndexes[2]*9+PFIndexes[3];
8490 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
8491 unsigned Cost = (PFEntry >> 30);
8492
8493 if (Cost <= 4) {
8494 if (ST->hasNEON())
8495 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8496 else if (isLegalMVEShuffleOp(PFEntry)) {
8497 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
8498 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
8499 unsigned PFEntryLHS = PerfectShuffleTable[LHSID];
8500 unsigned PFEntryRHS = PerfectShuffleTable[RHSID];
8501 if (isLegalMVEShuffleOp(PFEntryLHS) && isLegalMVEShuffleOp(PFEntryRHS))
8502 return GeneratePerfectShuffle(PFEntry, V1, V2, DAG, dl);
8503 }
8504 }
8505 }
8506
8507 // Implement shuffles with 32- or 64-bit elements as ARMISD::BUILD_VECTORs.
8508 if (EltSize >= 32) {
8509 // Do the expansion with floating-point types, since that is what the VFP
8510 // registers are defined to use, and since i64 is not legal.
8511 EVT EltVT = EVT::getFloatingPointVT(EltSize);
8512 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
8513 V1 = DAG.getNode(ISD::BITCAST, dl, VecVT, V1);
8514 V2 = DAG.getNode(ISD::BITCAST, dl, VecVT, V2);
8515 SmallVector<SDValue, 8> Ops;
8516 for (unsigned i = 0; i < NumElts; ++i) {
8517 if (ShuffleMask[i] < 0)
8518 Ops.push_back(DAG.getUNDEF(EltVT));
8519 else
8520 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
8521 ShuffleMask[i] < (int)NumElts ? V1 : V2,
8522 DAG.getConstant(ShuffleMask[i] & (NumElts-1),
8523 dl, MVT::i32)));
8524 }
8525 SDValue Val = DAG.getNode(ARMISD::BUILD_VECTOR, dl, VecVT, Ops);
8526 return DAG.getNode(ISD::BITCAST, dl, VT, Val);
8527 }
8528
8529 if (ST->hasNEON() && (VT == MVT::v8i16 || VT == MVT::v16i8) && isReverseMask(ShuffleMask, VT))
8530 return LowerReverse_VECTOR_SHUFFLEv16i8_v8i16(Op, DAG);
8531
8532 if (ST->hasNEON() && VT == MVT::v8i8)
8533 if (SDValue NewOp = LowerVECTOR_SHUFFLEv8i8(Op, ShuffleMask, DAG))
8534 return NewOp;
8535
8536 if (ST->hasMVEIntegerOps())
8537 if (SDValue NewOp = LowerVECTOR_SHUFFLEUsingMovs(Op, ShuffleMask, DAG))
8538 return NewOp;
8539
8540 return SDValue();
8541}
8542
8543static SDValue LowerINSERT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8544 const ARMSubtarget *ST) {
8545 EVT VecVT = Op.getOperand(0).getValueType();
8546 SDLoc dl(Op);
8547
8548 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"LowerINSERT_VECTOR_ELT_i1 called without MVE!") ? void (0) :
__assert_fail ("ST->hasMVEIntegerOps() && \"LowerINSERT_VECTOR_ELT_i1 called without MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8549, __extension__ __PRETTY_FUNCTION__))
8549 "LowerINSERT_VECTOR_ELT_i1 called without MVE!")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"LowerINSERT_VECTOR_ELT_i1 called without MVE!") ? void (0) :
__assert_fail ("ST->hasMVEIntegerOps() && \"LowerINSERT_VECTOR_ELT_i1 called without MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8549, __extension__ __PRETTY_FUNCTION__))
;
8550
8551 SDValue Conv =
8552 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
8553 unsigned Lane = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
8554 unsigned LaneWidth =
8555 getVectorTyFromPredicateVector(VecVT).getScalarSizeInBits() / 8;
8556 unsigned Mask = ((1 << LaneWidth) - 1) << Lane * LaneWidth;
8557 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::i32,
8558 Op.getOperand(1), DAG.getValueType(MVT::i1));
8559 SDValue BFI = DAG.getNode(ARMISD::BFI, dl, MVT::i32, Conv, Ext,
8560 DAG.getConstant(~Mask, dl, MVT::i32));
8561 return DAG.getNode(ARMISD::PREDICATE_CAST, dl, Op.getValueType(), BFI);
8562}
8563
8564SDValue ARMTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
8565 SelectionDAG &DAG) const {
8566 // INSERT_VECTOR_ELT is legal only for immediate indexes.
8567 SDValue Lane = Op.getOperand(2);
8568 if (!isa<ConstantSDNode>(Lane))
8569 return SDValue();
8570
8571 SDValue Elt = Op.getOperand(1);
8572 EVT EltVT = Elt.getValueType();
8573
8574 if (Subtarget->hasMVEIntegerOps() &&
8575 Op.getValueType().getScalarSizeInBits() == 1)
8576 return LowerINSERT_VECTOR_ELT_i1(Op, DAG, Subtarget);
8577
8578 if (getTypeAction(*DAG.getContext(), EltVT) ==
8579 TargetLowering::TypePromoteFloat) {
8580 // INSERT_VECTOR_ELT doesn't want f16 operands promoting to f32,
8581 // but the type system will try to do that if we don't intervene.
8582 // Reinterpret any such vector-element insertion as one with the
8583 // corresponding integer types.
8584
8585 SDLoc dl(Op);
8586
8587 EVT IEltVT = MVT::getIntegerVT(EltVT.getScalarSizeInBits());
8588 assert(getTypeAction(*DAG.getContext(), IEltVT) !=(static_cast <bool> (getTypeAction(*DAG.getContext(), IEltVT
) != TargetLowering::TypePromoteFloat) ? void (0) : __assert_fail
("getTypeAction(*DAG.getContext(), IEltVT) != TargetLowering::TypePromoteFloat"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8589, __extension__ __PRETTY_FUNCTION__))
8589 TargetLowering::TypePromoteFloat)(static_cast <bool> (getTypeAction(*DAG.getContext(), IEltVT
) != TargetLowering::TypePromoteFloat) ? void (0) : __assert_fail
("getTypeAction(*DAG.getContext(), IEltVT) != TargetLowering::TypePromoteFloat"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8589, __extension__ __PRETTY_FUNCTION__))
;
8590
8591 SDValue VecIn = Op.getOperand(0);
8592 EVT VecVT = VecIn.getValueType();
8593 EVT IVecVT = EVT::getVectorVT(*DAG.getContext(), IEltVT,
8594 VecVT.getVectorNumElements());
8595
8596 SDValue IElt = DAG.getNode(ISD::BITCAST, dl, IEltVT, Elt);
8597 SDValue IVecIn = DAG.getNode(ISD::BITCAST, dl, IVecVT, VecIn);
8598 SDValue IVecOut = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, IVecVT,
8599 IVecIn, IElt, Lane);
8600 return DAG.getNode(ISD::BITCAST, dl, VecVT, IVecOut);
8601 }
8602
8603 return Op;
8604}
8605
8606static SDValue LowerEXTRACT_VECTOR_ELT_i1(SDValue Op, SelectionDAG &DAG,
8607 const ARMSubtarget *ST) {
8608 EVT VecVT = Op.getOperand(0).getValueType();
8609 SDLoc dl(Op);
8610
8611 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"LowerINSERT_VECTOR_ELT_i1 called without MVE!") ? void (0) :
__assert_fail ("ST->hasMVEIntegerOps() && \"LowerINSERT_VECTOR_ELT_i1 called without MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8612, __extension__ __PRETTY_FUNCTION__))
8612 "LowerINSERT_VECTOR_ELT_i1 called without MVE!")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"LowerINSERT_VECTOR_ELT_i1 called without MVE!") ? void (0) :
__assert_fail ("ST->hasMVEIntegerOps() && \"LowerINSERT_VECTOR_ELT_i1 called without MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8612, __extension__ __PRETTY_FUNCTION__))
;
8613
8614 SDValue Conv =
8615 DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Op->getOperand(0));
8616 unsigned Lane = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
8617 unsigned LaneWidth =
8618 getVectorTyFromPredicateVector(VecVT).getScalarSizeInBits() / 8;
8619 SDValue Shift = DAG.getNode(ISD::SRL, dl, MVT::i32, Conv,
8620 DAG.getConstant(Lane * LaneWidth, dl, MVT::i32));
8621 return Shift;
8622}
8623
8624static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG,
8625 const ARMSubtarget *ST) {
8626 // EXTRACT_VECTOR_ELT is legal only for immediate indexes.
8627 SDValue Lane = Op.getOperand(1);
8628 if (!isa<ConstantSDNode>(Lane))
8629 return SDValue();
8630
8631 SDValue Vec = Op.getOperand(0);
8632 EVT VT = Vec.getValueType();
8633
8634 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8635 return LowerEXTRACT_VECTOR_ELT_i1(Op, DAG, ST);
8636
8637 if (Op.getValueType() == MVT::i32 && Vec.getScalarValueSizeInBits() < 32) {
8638 SDLoc dl(Op);
8639 return DAG.getNode(ARMISD::VGETLANEu, dl, MVT::i32, Vec, Lane);
8640 }
8641
8642 return Op;
8643}
8644
8645static SDValue LowerCONCAT_VECTORS_i1(SDValue Op, SelectionDAG &DAG,
8646 const ARMSubtarget *ST) {
8647 SDValue V1 = Op.getOperand(0);
8648 SDValue V2 = Op.getOperand(1);
8649 SDLoc dl(Op);
8650 EVT VT = Op.getValueType();
8651 EVT Op1VT = V1.getValueType();
8652 EVT Op2VT = V2.getValueType();
8653 unsigned NumElts = VT.getVectorNumElements();
8654
8655 assert(Op1VT == Op2VT && "Operand types don't match!")(static_cast <bool> (Op1VT == Op2VT && "Operand types don't match!"
) ? void (0) : __assert_fail ("Op1VT == Op2VT && \"Operand types don't match!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8655, __extension__ __PRETTY_FUNCTION__))
;
8656 assert(VT.getScalarSizeInBits() == 1 &&(static_cast <bool> (VT.getScalarSizeInBits() == 1 &&
"Unexpected custom CONCAT_VECTORS lowering") ? void (0) : __assert_fail
("VT.getScalarSizeInBits() == 1 && \"Unexpected custom CONCAT_VECTORS lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8657, __extension__ __PRETTY_FUNCTION__))
8657 "Unexpected custom CONCAT_VECTORS lowering")(static_cast <bool> (VT.getScalarSizeInBits() == 1 &&
"Unexpected custom CONCAT_VECTORS lowering") ? void (0) : __assert_fail
("VT.getScalarSizeInBits() == 1 && \"Unexpected custom CONCAT_VECTORS lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8657, __extension__ __PRETTY_FUNCTION__))
;
8658 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"CONCAT_VECTORS lowering only supported for MVE") ? void (0)
: __assert_fail ("ST->hasMVEIntegerOps() && \"CONCAT_VECTORS lowering only supported for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8659, __extension__ __PRETTY_FUNCTION__))
8659 "CONCAT_VECTORS lowering only supported for MVE")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"CONCAT_VECTORS lowering only supported for MVE") ? void (0)
: __assert_fail ("ST->hasMVEIntegerOps() && \"CONCAT_VECTORS lowering only supported for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8659, __extension__ __PRETTY_FUNCTION__))
;
8660
8661 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8662 SDValue NewV2 = PromoteMVEPredVector(dl, V2, Op2VT, DAG);
8663
8664 // We now have Op1 + Op2 promoted to vectors of integers, where v8i1 gets
8665 // promoted to v8i16, etc.
8666
8667 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8668
8669 // Extract the vector elements from Op1 and Op2 one by one and truncate them
8670 // to be the right size for the destination. For example, if Op1 is v4i1 then
8671 // the promoted vector is v4i32. The result of concatentation gives a v8i1,
8672 // which when promoted is v8i16. That means each i32 element from Op1 needs
8673 // truncating to i16 and inserting in the result.
8674 EVT ConcatVT = MVT::getVectorVT(ElType, NumElts);
8675 SDValue ConVec = DAG.getNode(ISD::UNDEF, dl, ConcatVT);
8676 auto ExractInto = [&DAG, &dl](SDValue NewV, SDValue ConVec, unsigned &j) {
8677 EVT NewVT = NewV.getValueType();
8678 EVT ConcatVT = ConVec.getValueType();
8679 for (unsigned i = 0, e = NewVT.getVectorNumElements(); i < e; i++, j++) {
8680 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV,
8681 DAG.getIntPtrConstant(i, dl));
8682 ConVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ConcatVT, ConVec, Elt,
8683 DAG.getConstant(j, dl, MVT::i32));
8684 }
8685 return ConVec;
8686 };
8687 unsigned j = 0;
8688 ConVec = ExractInto(NewV1, ConVec, j);
8689 ConVec = ExractInto(NewV2, ConVec, j);
8690
8691 // Now return the result of comparing the subvector with zero,
8692 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8693 return DAG.getNode(ARMISD::VCMPZ, dl, VT, ConVec,
8694 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8695}
8696
8697static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
8698 const ARMSubtarget *ST) {
8699 EVT VT = Op->getValueType(0);
8700 if (ST->hasMVEIntegerOps() && VT.getScalarSizeInBits() == 1)
8701 return LowerCONCAT_VECTORS_i1(Op, DAG, ST);
8702
8703 // The only time a CONCAT_VECTORS operation can have legal types is when
8704 // two 64-bit vectors are concatenated to a 128-bit vector.
8705 assert(Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 &&(static_cast <bool> (Op.getValueType().is128BitVector()
&& Op.getNumOperands() == 2 && "unexpected CONCAT_VECTORS"
) ? void (0) : __assert_fail ("Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 && \"unexpected CONCAT_VECTORS\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8706, __extension__ __PRETTY_FUNCTION__))
8706 "unexpected CONCAT_VECTORS")(static_cast <bool> (Op.getValueType().is128BitVector()
&& Op.getNumOperands() == 2 && "unexpected CONCAT_VECTORS"
) ? void (0) : __assert_fail ("Op.getValueType().is128BitVector() && Op.getNumOperands() == 2 && \"unexpected CONCAT_VECTORS\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8706, __extension__ __PRETTY_FUNCTION__))
;
8707 SDLoc dl(Op);
8708 SDValue Val = DAG.getUNDEF(MVT::v2f64);
8709 SDValue Op0 = Op.getOperand(0);
8710 SDValue Op1 = Op.getOperand(1);
8711 if (!Op0.isUndef())
8712 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
8713 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op0),
8714 DAG.getIntPtrConstant(0, dl));
8715 if (!Op1.isUndef())
8716 Val = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v2f64, Val,
8717 DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op1),
8718 DAG.getIntPtrConstant(1, dl));
8719 return DAG.getNode(ISD::BITCAST, dl, Op.getValueType(), Val);
8720}
8721
8722static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG,
8723 const ARMSubtarget *ST) {
8724 SDValue V1 = Op.getOperand(0);
8725 SDValue V2 = Op.getOperand(1);
8726 SDLoc dl(Op);
8727 EVT VT = Op.getValueType();
8728 EVT Op1VT = V1.getValueType();
8729 unsigned NumElts = VT.getVectorNumElements();
8730 unsigned Index = cast<ConstantSDNode>(V2)->getZExtValue();
8731
8732 assert(VT.getScalarSizeInBits() == 1 &&(static_cast <bool> (VT.getScalarSizeInBits() == 1 &&
"Unexpected custom EXTRACT_SUBVECTOR lowering") ? void (0) :
__assert_fail ("VT.getScalarSizeInBits() == 1 && \"Unexpected custom EXTRACT_SUBVECTOR lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8733, __extension__ __PRETTY_FUNCTION__))
8733 "Unexpected custom EXTRACT_SUBVECTOR lowering")(static_cast <bool> (VT.getScalarSizeInBits() == 1 &&
"Unexpected custom EXTRACT_SUBVECTOR lowering") ? void (0) :
__assert_fail ("VT.getScalarSizeInBits() == 1 && \"Unexpected custom EXTRACT_SUBVECTOR lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8733, __extension__ __PRETTY_FUNCTION__))
;
8734 assert(ST->hasMVEIntegerOps() &&(static_cast <bool> (ST->hasMVEIntegerOps() &&
"EXTRACT_SUBVECTOR lowering only supported for MVE") ? void (
0) : __assert_fail ("ST->hasMVEIntegerOps() && \"EXTRACT_SUBVECTOR lowering only supported for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8735, __extension__ __PRETTY_FUNCTION__))
8735 "EXTRACT_SUBVECTOR lowering only supported for MVE")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"EXTRACT_SUBVECTOR lowering only supported for MVE") ? void (
0) : __assert_fail ("ST->hasMVEIntegerOps() && \"EXTRACT_SUBVECTOR lowering only supported for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8735, __extension__ __PRETTY_FUNCTION__))
;
8736
8737 SDValue NewV1 = PromoteMVEPredVector(dl, V1, Op1VT, DAG);
8738
8739 // We now have Op1 promoted to a vector of integers, where v8i1 gets
8740 // promoted to v8i16, etc.
8741
8742 MVT ElType = getVectorTyFromPredicateVector(VT).getScalarType().getSimpleVT();
8743
8744 EVT SubVT = MVT::getVectorVT(ElType, NumElts);
8745 SDValue SubVec = DAG.getNode(ISD::UNDEF, dl, SubVT);
8746 for (unsigned i = Index, j = 0; i < (Index + NumElts); i++, j++) {
8747 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, NewV1,
8748 DAG.getIntPtrConstant(i, dl));
8749 SubVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, SubVT, SubVec, Elt,
8750 DAG.getConstant(j, dl, MVT::i32));
8751 }
8752
8753 // Now return the result of comparing the subvector with zero,
8754 // which will generate a real predicate, i.e. v4i1, v8i1 or v16i1.
8755 return DAG.getNode(ARMISD::VCMPZ, dl, VT, SubVec,
8756 DAG.getConstant(ARMCC::NE, dl, MVT::i32));
8757}
8758
8759// Turn a truncate into a predicate (an i1 vector) into icmp(and(x, 1), 0).
8760static SDValue LowerTruncatei1(SDValue N, SelectionDAG &DAG,
8761 const ARMSubtarget *ST) {
8762 assert(ST->hasMVEIntegerOps() && "Expected MVE!")(static_cast <bool> (ST->hasMVEIntegerOps() &&
"Expected MVE!") ? void (0) : __assert_fail ("ST->hasMVEIntegerOps() && \"Expected MVE!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8762, __extension__ __PRETTY_FUNCTION__))
;
8763 EVT VT = N.getValueType();
8764 assert((VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) &&(static_cast <bool> ((VT == MVT::v16i1 || VT == MVT::v8i1
|| VT == MVT::v4i1) && "Expected a vector i1 type!")
? void (0) : __assert_fail ("(VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) && \"Expected a vector i1 type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8765, __extension__ __PRETTY_FUNCTION__))
8765 "Expected a vector i1 type!")(static_cast <bool> ((VT == MVT::v16i1 || VT == MVT::v8i1
|| VT == MVT::v4i1) && "Expected a vector i1 type!")
? void (0) : __assert_fail ("(VT == MVT::v16i1 || VT == MVT::v8i1 || VT == MVT::v4i1) && \"Expected a vector i1 type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8765, __extension__ __PRETTY_FUNCTION__))
;
8766 SDValue Op = N.getOperand(0);
8767 EVT FromVT = Op.getValueType();
8768 SDLoc DL(N);
8769
8770 SDValue And =
8771 DAG.getNode(ISD::AND, DL, FromVT, Op, DAG.getConstant(1, DL, FromVT));
8772 return DAG.getNode(ISD::SETCC, DL, VT, And, DAG.getConstant(0, DL, FromVT),
8773 DAG.getCondCode(ISD::SETNE));
8774}
8775
8776/// isExtendedBUILD_VECTOR - Check if N is a constant BUILD_VECTOR where each
8777/// element has been zero/sign-extended, depending on the isSigned parameter,
8778/// from an integer type half its size.
8779static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG,
8780 bool isSigned) {
8781 // A v2i64 BUILD_VECTOR will have been legalized to a BITCAST from v4i32.
8782 EVT VT = N->getValueType(0);
8783 if (VT == MVT::v2i64 && N->getOpcode() == ISD::BITCAST) {
8784 SDNode *BVN = N->getOperand(0).getNode();
8785 if (BVN->getValueType(0) != MVT::v4i32 ||
8786 BVN->getOpcode() != ISD::BUILD_VECTOR)
8787 return false;
8788 unsigned LoElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
8789 unsigned HiElt = 1 - LoElt;
8790 ConstantSDNode *Lo0 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt));
8791 ConstantSDNode *Hi0 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt));
8792 ConstantSDNode *Lo1 = dyn_cast<ConstantSDNode>(BVN->getOperand(LoElt+2));
8793 ConstantSDNode *Hi1 = dyn_cast<ConstantSDNode>(BVN->getOperand(HiElt+2));
8794 if (!Lo0 || !Hi0 || !Lo1 || !Hi1)
8795 return false;
8796 if (isSigned) {
8797 if (Hi0->getSExtValue() == Lo0->getSExtValue() >> 32 &&
8798 Hi1->getSExtValue() == Lo1->getSExtValue() >> 32)
8799 return true;
8800 } else {
8801 if (Hi0->isNullValue() && Hi1->isNullValue())
8802 return true;
8803 }
8804 return false;
8805 }
8806
8807 if (N->getOpcode() != ISD::BUILD_VECTOR)
8808 return false;
8809
8810 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
8811 SDNode *Elt = N->getOperand(i).getNode();
8812 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Elt)) {
8813 unsigned EltSize = VT.getScalarSizeInBits();
8814 unsigned HalfSize = EltSize / 2;
8815 if (isSigned) {
8816 if (!isIntN(HalfSize, C->getSExtValue()))
8817 return false;
8818 } else {
8819 if (!isUIntN(HalfSize, C->getZExtValue()))
8820 return false;
8821 }
8822 continue;
8823 }
8824 return false;
8825 }
8826
8827 return true;
8828}
8829
8830/// isSignExtended - Check if a node is a vector value that is sign-extended
8831/// or a constant BUILD_VECTOR with sign-extended elements.
8832static bool isSignExtended(SDNode *N, SelectionDAG &DAG) {
8833 if (N->getOpcode() == ISD::SIGN_EXTEND || ISD::isSEXTLoad(N))
8834 return true;
8835 if (isExtendedBUILD_VECTOR(N, DAG, true))
8836 return true;
8837 return false;
8838}
8839
8840/// isZeroExtended - Check if a node is a vector value that is zero-extended (or
8841/// any-extended) or a constant BUILD_VECTOR with zero-extended elements.
8842static bool isZeroExtended(SDNode *N, SelectionDAG &DAG) {
8843 if (N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND ||
8844 ISD::isZEXTLoad(N))
8845 return true;
8846 if (isExtendedBUILD_VECTOR(N, DAG, false))
8847 return true;
8848 return false;
8849}
8850
8851static EVT getExtensionTo64Bits(const EVT &OrigVT) {
8852 if (OrigVT.getSizeInBits() >= 64)
8853 return OrigVT;
8854
8855 assert(OrigVT.isSimple() && "Expecting a simple value type")(static_cast <bool> (OrigVT.isSimple() && "Expecting a simple value type"
) ? void (0) : __assert_fail ("OrigVT.isSimple() && \"Expecting a simple value type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8855, __extension__ __PRETTY_FUNCTION__))
;
8856
8857 MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
8858 switch (OrigSimpleTy) {
8859 default: llvm_unreachable("Unexpected Vector Type")::llvm::llvm_unreachable_internal("Unexpected Vector Type", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8859)
;
8860 case MVT::v2i8:
8861 case MVT::v2i16:
8862 return MVT::v2i32;
8863 case MVT::v4i8:
8864 return MVT::v4i16;
8865 }
8866}
8867
8868/// AddRequiredExtensionForVMULL - Add a sign/zero extension to extend the total
8869/// value size to 64 bits. We need a 64-bit D register as an operand to VMULL.
8870/// We insert the required extension here to get the vector to fill a D register.
8871static SDValue AddRequiredExtensionForVMULL(SDValue N, SelectionDAG &DAG,
8872 const EVT &OrigTy,
8873 const EVT &ExtTy,
8874 unsigned ExtOpcode) {
8875 // The vector originally had a size of OrigTy. It was then extended to ExtTy.
8876 // We expect the ExtTy to be 128-bits total. If the OrigTy is less than
8877 // 64-bits we need to insert a new extension so that it will be 64-bits.
8878 assert(ExtTy.is128BitVector() && "Unexpected extension size")(static_cast <bool> (ExtTy.is128BitVector() && "Unexpected extension size"
) ? void (0) : __assert_fail ("ExtTy.is128BitVector() && \"Unexpected extension size\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8878, __extension__ __PRETTY_FUNCTION__))
;
8879 if (OrigTy.getSizeInBits() >= 64)
8880 return N;
8881
8882 // Must extend size to at least 64 bits to be used as an operand for VMULL.
8883 EVT NewVT = getExtensionTo64Bits(OrigTy);
8884
8885 return DAG.getNode(ExtOpcode, SDLoc(N), NewVT, N);
8886}
8887
8888/// SkipLoadExtensionForVMULL - return a load of the original vector size that
8889/// does not do any sign/zero extension. If the original vector is less
8890/// than 64 bits, an appropriate extension will be added after the load to
8891/// reach a total size of 64 bits. We have to add the extension separately
8892/// because ARM does not have a sign/zero extending load for vectors.
8893static SDValue SkipLoadExtensionForVMULL(LoadSDNode *LD, SelectionDAG& DAG) {
8894 EVT ExtendedTy = getExtensionTo64Bits(LD->getMemoryVT());
8895
8896 // The load already has the right type.
8897 if (ExtendedTy == LD->getMemoryVT())
8898 return DAG.getLoad(LD->getMemoryVT(), SDLoc(LD), LD->getChain(),
8899 LD->getBasePtr(), LD->getPointerInfo(),
8900 LD->getAlignment(), LD->getMemOperand()->getFlags());
8901
8902 // We need to create a zextload/sextload. We cannot just create a load
8903 // followed by a zext/zext node because LowerMUL is also run during normal
8904 // operation legalization where we can't create illegal types.
8905 return DAG.getExtLoad(LD->getExtensionType(), SDLoc(LD), ExtendedTy,
8906 LD->getChain(), LD->getBasePtr(), LD->getPointerInfo(),
8907 LD->getMemoryVT(), LD->getAlignment(),
8908 LD->getMemOperand()->getFlags());
8909}
8910
8911/// SkipExtensionForVMULL - For a node that is a SIGN_EXTEND, ZERO_EXTEND,
8912/// ANY_EXTEND, extending load, or BUILD_VECTOR with extended elements, return
8913/// the unextended value. The unextended vector should be 64 bits so that it can
8914/// be used as an operand to a VMULL instruction. If the original vector size
8915/// before extension is less than 64 bits we add a an extension to resize
8916/// the vector to 64 bits.
8917static SDValue SkipExtensionForVMULL(SDNode *N, SelectionDAG &DAG) {
8918 if (N->getOpcode() == ISD::SIGN_EXTEND ||
8919 N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::ANY_EXTEND)
8920 return AddRequiredExtensionForVMULL(N->getOperand(0), DAG,
8921 N->getOperand(0)->getValueType(0),
8922 N->getValueType(0),
8923 N->getOpcode());
8924
8925 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
8926 assert((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) &&(static_cast <bool> ((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad
(LD)) && "Expected extending load") ? void (0) : __assert_fail
("(ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) && \"Expected extending load\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8927, __extension__ __PRETTY_FUNCTION__))
8927 "Expected extending load")(static_cast <bool> ((ISD::isSEXTLoad(LD) || ISD::isZEXTLoad
(LD)) && "Expected extending load") ? void (0) : __assert_fail
("(ISD::isSEXTLoad(LD) || ISD::isZEXTLoad(LD)) && \"Expected extending load\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8927, __extension__ __PRETTY_FUNCTION__))
;
8928
8929 SDValue newLoad = SkipLoadExtensionForVMULL(LD, DAG);
8930 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), newLoad.getValue(1));
8931 unsigned Opcode = ISD::isSEXTLoad(LD) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
8932 SDValue extLoad =
8933 DAG.getNode(Opcode, SDLoc(newLoad), LD->getValueType(0), newLoad);
8934 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 0), extLoad);
8935
8936 return newLoad;
8937 }
8938
8939 // Otherwise, the value must be a BUILD_VECTOR. For v2i64, it will
8940 // have been legalized as a BITCAST from v4i32.
8941 if (N->getOpcode() == ISD::BITCAST) {
8942 SDNode *BVN = N->getOperand(0).getNode();
8943 assert(BVN->getOpcode() == ISD::BUILD_VECTOR &&(static_cast <bool> (BVN->getOpcode() == ISD::BUILD_VECTOR
&& BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"
) ? void (0) : __assert_fail ("BVN->getOpcode() == ISD::BUILD_VECTOR && BVN->getValueType(0) == MVT::v4i32 && \"expected v4i32 BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8944, __extension__ __PRETTY_FUNCTION__))
8944 BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR")(static_cast <bool> (BVN->getOpcode() == ISD::BUILD_VECTOR
&& BVN->getValueType(0) == MVT::v4i32 && "expected v4i32 BUILD_VECTOR"
) ? void (0) : __assert_fail ("BVN->getOpcode() == ISD::BUILD_VECTOR && BVN->getValueType(0) == MVT::v4i32 && \"expected v4i32 BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8944, __extension__ __PRETTY_FUNCTION__))
;
8945 unsigned LowElt = DAG.getDataLayout().isBigEndian() ? 1 : 0;
8946 return DAG.getBuildVector(
8947 MVT::v2i32, SDLoc(N),
8948 {BVN->getOperand(LowElt), BVN->getOperand(LowElt + 2)});
8949 }
8950 // Construct a new BUILD_VECTOR with elements truncated to half the size.
8951 assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR")(static_cast <bool> (N->getOpcode() == ISD::BUILD_VECTOR
&& "expected BUILD_VECTOR") ? void (0) : __assert_fail
("N->getOpcode() == ISD::BUILD_VECTOR && \"expected BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8951, __extension__ __PRETTY_FUNCTION__))
;
8952 EVT VT = N->getValueType(0);
8953 unsigned EltSize = VT.getScalarSizeInBits() / 2;
8954 unsigned NumElts = VT.getVectorNumElements();
8955 MVT TruncVT = MVT::getIntegerVT(EltSize);
8956 SmallVector<SDValue, 8> Ops;
8957 SDLoc dl(N);
8958 for (unsigned i = 0; i != NumElts; ++i) {
8959 ConstantSDNode *C = cast<ConstantSDNode>(N->getOperand(i));
8960 const APInt &CInt = C->getAPIntValue();
8961 // Element types smaller than 32 bits are not legal, so use i32 elements.
8962 // The values are implicitly truncated so sext vs. zext doesn't matter.
8963 Ops.push_back(DAG.getConstant(CInt.zextOrTrunc(32), dl, MVT::i32));
8964 }
8965 return DAG.getBuildVector(MVT::getVectorVT(TruncVT, NumElts), dl, Ops);
8966}
8967
8968static bool isAddSubSExt(SDNode *N, SelectionDAG &DAG) {
8969 unsigned Opcode = N->getOpcode();
8970 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
8971 SDNode *N0 = N->getOperand(0).getNode();
8972 SDNode *N1 = N->getOperand(1).getNode();
8973 return N0->hasOneUse() && N1->hasOneUse() &&
8974 isSignExtended(N0, DAG) && isSignExtended(N1, DAG);
8975 }
8976 return false;
8977}
8978
8979static bool isAddSubZExt(SDNode *N, SelectionDAG &DAG) {
8980 unsigned Opcode = N->getOpcode();
8981 if (Opcode == ISD::ADD || Opcode == ISD::SUB) {
8982 SDNode *N0 = N->getOperand(0).getNode();
8983 SDNode *N1 = N->getOperand(1).getNode();
8984 return N0->hasOneUse() && N1->hasOneUse() &&
8985 isZeroExtended(N0, DAG) && isZeroExtended(N1, DAG);
8986 }
8987 return false;
8988}
8989
8990static SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) {
8991 // Multiplications are only custom-lowered for 128-bit vectors so that
8992 // VMULL can be detected. Otherwise v2i64 multiplications are not legal.
8993 EVT VT = Op.getValueType();
8994 assert(VT.is128BitVector() && VT.isInteger() &&(static_cast <bool> (VT.is128BitVector() && VT.
isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? void (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__))
8995 "unexpected type for custom-lowering ISD::MUL")(static_cast <bool> (VT.is128BitVector() && VT.
isInteger() && "unexpected type for custom-lowering ISD::MUL"
) ? void (0) : __assert_fail ("VT.is128BitVector() && VT.isInteger() && \"unexpected type for custom-lowering ISD::MUL\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__))
;
8996 SDNode *N0 = Op.getOperand(0).getNode();
8997 SDNode *N1 = Op.getOperand(1).getNode();
8998 unsigned NewOpc = 0;
8999 bool isMLA = false;
9000 bool isN0SExt = isSignExtended(N0, DAG);
9001 bool isN1SExt = isSignExtended(N1, DAG);
9002 if (isN0SExt && isN1SExt)
9003 NewOpc = ARMISD::VMULLs;
9004 else {
9005 bool isN0ZExt = isZeroExtended(N0, DAG);
9006 bool isN1ZExt = isZeroExtended(N1, DAG);
9007 if (isN0ZExt && isN1ZExt)
9008 NewOpc = ARMISD::VMULLu;
9009 else if (isN1SExt || isN1ZExt) {
9010 // Look for (s/zext A + s/zext B) * (s/zext C). We want to turn these
9011 // into (s/zext A * s/zext C) + (s/zext B * s/zext C)
9012 if (isN1SExt && isAddSubSExt(N0, DAG)) {
9013 NewOpc = ARMISD::VMULLs;
9014 isMLA = true;
9015 } else if (isN1ZExt && isAddSubZExt(N0, DAG)) {
9016 NewOpc = ARMISD::VMULLu;
9017 isMLA = true;
9018 } else if (isN0ZExt && isAddSubZExt(N1, DAG)) {
9019 std::swap(N0, N1);
9020 NewOpc = ARMISD::VMULLu;
9021 isMLA = true;
9022 }
9023 }
9024
9025 if (!NewOpc) {
9026 if (VT == MVT::v2i64)
9027 // Fall through to expand this. It is not legal.
9028 return SDValue();
9029 else
9030 // Other vector multiplications are legal.
9031 return Op;
9032 }
9033 }
9034
9035 // Legalize to a VMULL instruction.
9036 SDLoc DL(Op);
9037 SDValue Op0;
9038 SDValue Op1 = SkipExtensionForVMULL(N1, DAG);
9039 if (!isMLA) {
9040 Op0 = SkipExtensionForVMULL(N0, DAG);
9041 assert(Op0.getValueType().is64BitVector() &&(static_cast <bool> (Op0.getValueType().is64BitVector()
&& Op1.getValueType().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? void (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9043, __extension__ __PRETTY_FUNCTION__))
9042 Op1.getValueType().is64BitVector() &&(static_cast <bool> (Op0.getValueType().is64BitVector()
&& Op1.getValueType().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? void (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9043, __extension__ __PRETTY_FUNCTION__))
9043 "unexpected types for extended operands to VMULL")(static_cast <bool> (Op0.getValueType().is64BitVector()
&& Op1.getValueType().is64BitVector() && "unexpected types for extended operands to VMULL"
) ? void (0) : __assert_fail ("Op0.getValueType().is64BitVector() && Op1.getValueType().is64BitVector() && \"unexpected types for extended operands to VMULL\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9043, __extension__ __PRETTY_FUNCTION__))
;
9044 return DAG.getNode(NewOpc, DL, VT, Op0, Op1);
9045 }
9046
9047 // Optimizing (zext A + zext B) * C, to (VMULL A, C) + (VMULL B, C) during
9048 // isel lowering to take advantage of no-stall back to back vmul + vmla.
9049 // vmull q0, d4, d6
9050 // vmlal q0, d5, d6
9051 // is faster than
9052 // vaddl q0, d4, d5
9053 // vmovl q1, d6
9054 // vmul q0, q0, q1
9055 SDValue N00 = SkipExtensionForVMULL(N0->getOperand(0).getNode(), DAG);
9056 SDValue N01 = SkipExtensionForVMULL(N0->getOperand(1).getNode(), DAG);
9057 EVT Op1VT = Op1.getValueType();
9058 return DAG.getNode(N0->getOpcode(), DL, VT,
9059 DAG.getNode(NewOpc, DL, VT,
9060 DAG.getNode(ISD::BITCAST, DL, Op1VT, N00), Op1),
9061 DAG.getNode(NewOpc, DL, VT,
9062 DAG.getNode(ISD::BITCAST, DL, Op1VT, N01), Op1));
9063}
9064
9065static SDValue LowerSDIV_v4i8(SDValue X, SDValue Y, const SDLoc &dl,
9066 SelectionDAG &DAG) {
9067 // TODO: Should this propagate fast-math-flags?
9068
9069 // Convert to float
9070 // float4 xf = vcvt_f32_s32(vmovl_s16(a.lo));
9071 // float4 yf = vcvt_f32_s32(vmovl_s16(b.lo));
9072 X = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, X);
9073 Y = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Y);
9074 X = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, X);
9075 Y = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, Y);
9076 // Get reciprocal estimate.
9077 // float4 recip = vrecpeq_f32(yf);
9078 Y = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9079 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9080 Y);
9081 // Because char has a smaller range than uchar, we can actually get away
9082 // without any newton steps. This requires that we use a weird bias
9083 // of 0xb000, however (again, this has been exhaustively tested).
9084 // float4 result = as_float4(as_int4(xf*recip) + 0xb000);
9085 X = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, X, Y);
9086 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, X);
9087 Y = DAG.getConstant(0xb000, dl, MVT::v4i32);
9088 X = DAG.getNode(ISD::ADD, dl, MVT::v4i32, X, Y);
9089 X = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, X);
9090 // Convert back to short.
9091 X = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, X);
9092 X = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, X);
9093 return X;
9094}
9095
9096static SDValue LowerSDIV_v4i16(SDValue N0, SDValue N1, const SDLoc &dl,
9097 SelectionDAG &DAG) {
9098 // TODO: Should this propagate fast-math-flags?
9099
9100 SDValue N2;
9101 // Convert to float.
9102 // float4 yf = vcvt_f32_s32(vmovl_s16(y));
9103 // float4 xf = vcvt_f32_s32(vmovl_s16(x));
9104 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N0);
9105 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, N1);
9106 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9107 N1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9108
9109 // Use reciprocal estimate and one refinement step.
9110 // float4 recip = vrecpeq_f32(yf);
9111 // recip *= vrecpsq_f32(yf, recip);
9112 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9113 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9114 N1);
9115 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9116 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9117 N1, N2);
9118 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9119 // Because short has a smaller range than ushort, we can actually get away
9120 // with only a single newton step. This requires that we use a weird bias
9121 // of 89, however (again, this has been exhaustively tested).
9122 // float4 result = as_float4(as_int4(xf*recip) + 0x89);
9123 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9124 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9125 N1 = DAG.getConstant(0x89, dl, MVT::v4i32);
9126 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9127 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9128 // Convert back to integer and return.
9129 // return vmovn_s32(vcvt_s32_f32(result));
9130 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9131 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9132 return N0;
9133}
9134
9135static SDValue LowerSDIV(SDValue Op, SelectionDAG &DAG,
9136 const ARMSubtarget *ST) {
9137 EVT VT = Op.getValueType();
9138 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&(static_cast <bool> ((VT == MVT::v4i16 || VT == MVT::v8i8
) && "unexpected type for custom-lowering ISD::SDIV")
? void (0) : __assert_fail ("(VT == MVT::v4i16 || VT == MVT::v8i8) && \"unexpected type for custom-lowering ISD::SDIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9139, __extension__ __PRETTY_FUNCTION__))
9139 "unexpected type for custom-lowering ISD::SDIV")(static_cast <bool> ((VT == MVT::v4i16 || VT == MVT::v8i8
) && "unexpected type for custom-lowering ISD::SDIV")
? void (0) : __assert_fail ("(VT == MVT::v4i16 || VT == MVT::v8i8) && \"unexpected type for custom-lowering ISD::SDIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9139, __extension__ __PRETTY_FUNCTION__))
;
9140
9141 SDLoc dl(Op);
9142 SDValue N0 = Op.getOperand(0);
9143 SDValue N1 = Op.getOperand(1);
9144 SDValue N2, N3;
9145
9146 if (VT == MVT::v8i8) {
9147 N0 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N0);
9148 N1 = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v8i16, N1);
9149
9150 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9151 DAG.getIntPtrConstant(4, dl));
9152 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9153 DAG.getIntPtrConstant(4, dl));
9154 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9155 DAG.getIntPtrConstant(0, dl));
9156 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9157 DAG.getIntPtrConstant(0, dl));
9158
9159 N0 = LowerSDIV_v4i8(N0, N1, dl, DAG); // v4i16
9160 N2 = LowerSDIV_v4i8(N2, N3, dl, DAG); // v4i16
9161
9162 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9163 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9164
9165 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v8i8, N0);
9166 return N0;
9167 }
9168 return LowerSDIV_v4i16(N0, N1, dl, DAG);
9169}
9170
9171static SDValue LowerUDIV(SDValue Op, SelectionDAG &DAG,
9172 const ARMSubtarget *ST) {
9173 // TODO: Should this propagate fast-math-flags?
9174 EVT VT = Op.getValueType();
9175 assert((VT == MVT::v4i16 || VT == MVT::v8i8) &&(static_cast <bool> ((VT == MVT::v4i16 || VT == MVT::v8i8
) && "unexpected type for custom-lowering ISD::UDIV")
? void (0) : __assert_fail ("(VT == MVT::v4i16 || VT == MVT::v8i8) && \"unexpected type for custom-lowering ISD::UDIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9176, __extension__ __PRETTY_FUNCTION__))
9176 "unexpected type for custom-lowering ISD::UDIV")(static_cast <bool> ((VT == MVT::v4i16 || VT == MVT::v8i8
) && "unexpected type for custom-lowering ISD::UDIV")
? void (0) : __assert_fail ("(VT == MVT::v4i16 || VT == MVT::v8i8) && \"unexpected type for custom-lowering ISD::UDIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9176, __extension__ __PRETTY_FUNCTION__))
;
9177
9178 SDLoc dl(Op);
9179 SDValue N0 = Op.getOperand(0);
9180 SDValue N1 = Op.getOperand(1);
9181 SDValue N2, N3;
9182
9183 if (VT == MVT::v8i8) {
9184 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N0);
9185 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v8i16, N1);
9186
9187 N2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9188 DAG.getIntPtrConstant(4, dl));
9189 N3 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9190 DAG.getIntPtrConstant(4, dl));
9191 N0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N0,
9192 DAG.getIntPtrConstant(0, dl));
9193 N1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v4i16, N1,
9194 DAG.getIntPtrConstant(0, dl));
9195
9196 N0 = LowerSDIV_v4i16(N0, N1, dl, DAG); // v4i16
9197 N2 = LowerSDIV_v4i16(N2, N3, dl, DAG); // v4i16
9198
9199 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i16, N0, N2);
9200 N0 = LowerCONCAT_VECTORS(N0, DAG, ST);
9201
9202 N0 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v8i8,
9203 DAG.getConstant(Intrinsic::arm_neon_vqmovnsu, dl,
9204 MVT::i32),
9205 N0);
9206 return N0;
9207 }
9208
9209 // v4i16 sdiv ... Convert to float.
9210 // float4 yf = vcvt_f32_s32(vmovl_u16(y));
9211 // float4 xf = vcvt_f32_s32(vmovl_u16(x));
9212 N0 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N0);
9213 N1 = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v4i32, N1);
9214 N0 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N0);
9215 SDValue BN1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::v4f32, N1);
9216
9217 // Use reciprocal estimate and two refinement steps.
9218 // float4 recip = vrecpeq_f32(yf);
9219 // recip *= vrecpsq_f32(yf, recip);
9220 // recip *= vrecpsq_f32(yf, recip);
9221 N2 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9222 DAG.getConstant(Intrinsic::arm_neon_vrecpe, dl, MVT::i32),
9223 BN1);
9224 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9225 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9226 BN1, N2);
9227 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9228 N1 = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, MVT::v4f32,
9229 DAG.getConstant(Intrinsic::arm_neon_vrecps, dl, MVT::i32),
9230 BN1, N2);
9231 N2 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N1, N2);
9232 // Simply multiplying by the reciprocal estimate can leave us a few ulps
9233 // too low, so we add 2 ulps (exhaustive testing shows that this is enough,
9234 // and that it will never cause us to return an answer too large).
9235 // float4 result = as_float4(as_int4(xf*recip) + 2);
9236 N0 = DAG.getNode(ISD::FMUL, dl, MVT::v4f32, N0, N2);
9237 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, N0);
9238 N1 = DAG.getConstant(2, dl, MVT::v4i32);
9239 N0 = DAG.getNode(ISD::ADD, dl, MVT::v4i32, N0, N1);
9240 N0 = DAG.getNode(ISD::BITCAST, dl, MVT::v4f32, N0);
9241 // Convert back to integer and return.
9242 // return vmovn_u32(vcvt_s32_f32(result));
9243 N0 = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::v4i32, N0);
9244 N0 = DAG.getNode(ISD::TRUNCATE, dl, MVT::v4i16, N0);
9245 return N0;
9246}
9247
9248static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
9249 SDNode *N = Op.getNode();
9250 EVT VT = N->getValueType(0);
9251 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
9252
9253 SDValue Carry = Op.getOperand(2);
9254
9255 SDLoc DL(Op);
9256
9257 SDValue Result;
9258 if (Op.getOpcode() == ISD::ADDCARRY) {
9259 // This converts the boolean value carry into the carry flag.
9260 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9261
9262 // Do the addition proper using the carry flag we wanted.
9263 Result = DAG.getNode(ARMISD::ADDE, DL, VTs, Op.getOperand(0),
9264 Op.getOperand(1), Carry);
9265
9266 // Now convert the carry flag into a boolean value.
9267 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9268 } else {
9269 // ARMISD::SUBE expects a carry not a borrow like ISD::SUBCARRY so we
9270 // have to invert the carry first.
9271 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9272 DAG.getConstant(1, DL, MVT::i32), Carry);
9273 // This converts the boolean value carry into the carry flag.
9274 Carry = ConvertBooleanCarryToCarryFlag(Carry, DAG);
9275
9276 // Do the subtraction proper using the carry flag we wanted.
9277 Result = DAG.getNode(ARMISD::SUBE, DL, VTs, Op.getOperand(0),
9278 Op.getOperand(1), Carry);
9279
9280 // Now convert the carry flag into a boolean value.
9281 Carry = ConvertCarryFlagToBooleanCarry(Result.getValue(1), VT, DAG);
9282 // But the carry returned by ARMISD::SUBE is not a borrow as expected
9283 // by ISD::SUBCARRY, so compute 1 - C.
9284 Carry = DAG.getNode(ISD::SUB, DL, MVT::i32,
9285 DAG.getConstant(1, DL, MVT::i32), Carry);
9286 }
9287
9288 // Return both values.
9289 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Result, Carry);
9290}
9291
9292SDValue ARMTargetLowering::LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const {
9293 assert(Subtarget->isTargetDarwin())(static_cast <bool> (Subtarget->isTargetDarwin()) ? void
(0) : __assert_fail ("Subtarget->isTargetDarwin()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9293, __extension__ __PRETTY_FUNCTION__))
;
9294
9295 // For iOS, we want to call an alternative entry point: __sincos_stret,
9296 // return values are passed via sret.
9297 SDLoc dl(Op);
9298 SDValue Arg = Op.getOperand(0);
9299 EVT ArgVT = Arg.getValueType();
9300 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
9301 auto PtrVT = getPointerTy(DAG.getDataLayout());
9302
9303 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
9304 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9305
9306 // Pair of floats / doubles used to pass the result.
9307 Type *RetTy = StructType::get(ArgTy, ArgTy);
9308 auto &DL = DAG.getDataLayout();
9309
9310 ArgListTy Args;
9311 bool ShouldUseSRet = Subtarget->isAPCS_ABI();
9312 SDValue SRet;
9313 if (ShouldUseSRet) {
9314 // Create stack object for sret.
9315 const uint64_t ByteSize = DL.getTypeAllocSize(RetTy);
9316 const Align StackAlign = DL.getPrefTypeAlign(RetTy);
9317 int FrameIdx = MFI.CreateStackObject(ByteSize, StackAlign, false);
9318 SRet = DAG.getFrameIndex(FrameIdx, TLI.getPointerTy(DL));
9319
9320 ArgListEntry Entry;
9321 Entry.Node = SRet;
9322 Entry.Ty = RetTy->getPointerTo();
9323 Entry.IsSExt = false;
9324 Entry.IsZExt = false;
9325 Entry.IsSRet = true;
9326 Args.push_back(Entry);
9327 RetTy = Type::getVoidTy(*DAG.getContext());
9328 }
9329
9330 ArgListEntry Entry;
9331 Entry.Node = Arg;
9332 Entry.Ty = ArgTy;
9333 Entry.IsSExt = false;
9334 Entry.IsZExt = false;
9335 Args.push_back(Entry);
9336
9337 RTLIB::Libcall LC =
9338 (ArgVT == MVT::f64) ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
9339 const char *LibcallName = getLibcallName(LC);
9340 CallingConv::ID CC = getLibcallCallingConv(LC);
9341 SDValue Callee = DAG.getExternalSymbol(LibcallName, getPointerTy(DL));
9342
9343 TargetLowering::CallLoweringInfo CLI(DAG);
9344 CLI.setDebugLoc(dl)
9345 .setChain(DAG.getEntryNode())
9346 .setCallee(CC, RetTy, Callee, std::move(Args))
9347 .setDiscardResult(ShouldUseSRet);
9348 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
9349
9350 if (!ShouldUseSRet)
9351 return CallResult.first;
9352
9353 SDValue LoadSin =
9354 DAG.getLoad(ArgVT, dl, CallResult.second, SRet, MachinePointerInfo());
9355
9356 // Address of cos field.
9357 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, SRet,
9358 DAG.getIntPtrConstant(ArgVT.getStoreSize(), dl));
9359 SDValue LoadCos =
9360 DAG.getLoad(ArgVT, dl, LoadSin.getValue(1), Add, MachinePointerInfo());
9361
9362 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
9363 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys,
9364 LoadSin.getValue(0), LoadCos.getValue(0));
9365}
9366
9367SDValue ARMTargetLowering::LowerWindowsDIVLibCall(SDValue Op, SelectionDAG &DAG,
9368 bool Signed,
9369 SDValue &Chain) const {
9370 EVT VT = Op.getValueType();
9371 assert((VT == MVT::i32 || VT == MVT::i64) &&(static_cast <bool> ((VT == MVT::i32 || VT == MVT::i64)
&& "unexpected type for custom lowering DIV") ? void
(0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9372, __extension__ __PRETTY_FUNCTION__))
9372 "unexpected type for custom lowering DIV")(static_cast <bool> ((VT == MVT::i32 || VT == MVT::i64)
&& "unexpected type for custom lowering DIV") ? void
(0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9372, __extension__ __PRETTY_FUNCTION__))
;
9373 SDLoc dl(Op);
9374
9375 const auto &DL = DAG.getDataLayout();
9376 const auto &TLI = DAG.getTargetLoweringInfo();
9377
9378 const char *Name = nullptr;
9379 if (Signed)
9380 Name = (VT == MVT::i32) ? "__rt_sdiv" : "__rt_sdiv64";
9381 else
9382 Name = (VT == MVT::i32) ? "__rt_udiv" : "__rt_udiv64";
9383
9384 SDValue ES = DAG.getExternalSymbol(Name, TLI.getPointerTy(DL));
9385
9386 ARMTargetLowering::ArgListTy Args;
9387
9388 for (auto AI : {1, 0}) {
9389 ArgListEntry Arg;
9390 Arg.Node = Op.getOperand(AI);
9391 Arg.Ty = Arg.Node.getValueType().getTypeForEVT(*DAG.getContext());
9392 Args.push_back(Arg);
9393 }
9394
9395 CallLoweringInfo CLI(DAG);
9396 CLI.setDebugLoc(dl)
9397 .setChain(Chain)
9398 .setCallee(CallingConv::ARM_AAPCS_VFP, VT.getTypeForEVT(*DAG.getContext()),
9399 ES, std::move(Args));
9400
9401 return LowerCallTo(CLI).first;
9402}
9403
9404// This is a code size optimisation: return the original SDIV node to
9405// DAGCombiner when we don't want to expand SDIV into a sequence of
9406// instructions, and an empty node otherwise which will cause the
9407// SDIV to be expanded in DAGCombine.
9408SDValue
9409ARMTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
9410 SelectionDAG &DAG,
9411 SmallVectorImpl<SDNode *> &Created) const {
9412 // TODO: Support SREM
9413 if (N->getOpcode() != ISD::SDIV)
9414 return SDValue();
9415
9416 const auto &ST = static_cast<const ARMSubtarget&>(DAG.getSubtarget());
9417 const bool MinSize = ST.hasMinSize();
9418 const bool HasDivide = ST.isThumb() ? ST.hasDivideInThumbMode()
9419 : ST.hasDivideInARMMode();
9420
9421 // Don't touch vector types; rewriting this may lead to scalarizing
9422 // the int divs.
9423 if (N->getOperand(0).getValueType().isVector())
9424 return SDValue();
9425
9426 // Bail if MinSize is not set, and also for both ARM and Thumb mode we need
9427 // hwdiv support for this to be really profitable.
9428 if (!(MinSize && HasDivide))
9429 return SDValue();
9430
9431 // ARM mode is a bit simpler than Thumb: we can handle large power
9432 // of 2 immediates with 1 mov instruction; no further checks required,
9433 // just return the sdiv node.
9434 if (!ST.isThumb())
9435 return SDValue(N, 0);
9436
9437 // In Thumb mode, immediates larger than 128 need a wide 4-byte MOV,
9438 // and thus lose the code size benefits of a MOVS that requires only 2.
9439 // TargetTransformInfo and 'getIntImmCodeSizeCost' could be helpful here,
9440 // but as it's doing exactly this, it's not worth the trouble to get TTI.
9441 if (Divisor.sgt(128))
9442 return SDValue();
9443
9444 return SDValue(N, 0);
9445}
9446
9447SDValue ARMTargetLowering::LowerDIV_Windows(SDValue Op, SelectionDAG &DAG,
9448 bool Signed) const {
9449 assert(Op.getValueType() == MVT::i32 &&(static_cast <bool> (Op.getValueType() == MVT::i32 &&
"unexpected type for custom lowering DIV") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i32 && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9450, __extension__ __PRETTY_FUNCTION__))
9450 "unexpected type for custom lowering DIV")(static_cast <bool> (Op.getValueType() == MVT::i32 &&
"unexpected type for custom lowering DIV") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i32 && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9450, __extension__ __PRETTY_FUNCTION__))
;
9451 SDLoc dl(Op);
9452
9453 SDValue DBZCHK = DAG.getNode(ARMISD::WIN__DBZCHK, dl, MVT::Other,
9454 DAG.getEntryNode(), Op.getOperand(1));
9455
9456 return LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
9457}
9458
9459static SDValue WinDBZCheckDenominator(SelectionDAG &DAG, SDNode *N, SDValue InChain) {
9460 SDLoc DL(N);
9461 SDValue Op = N->getOperand(1);
9462 if (N->getValueType(0) == MVT::i32)
9463 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain, Op);
9464 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
9465 DAG.getConstant(0, DL, MVT::i32));
9466 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op,
9467 DAG.getConstant(1, DL, MVT::i32));
9468 return DAG.getNode(ARMISD::WIN__DBZCHK, DL, MVT::Other, InChain,
9469 DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi));
9470}
9471
9472void ARMTargetLowering::ExpandDIV_Windows(
9473 SDValue Op, SelectionDAG &DAG, bool Signed,
9474 SmallVectorImpl<SDValue> &Results) const {
9475 const auto &DL = DAG.getDataLayout();
9476 const auto &TLI = DAG.getTargetLoweringInfo();
9477
9478 assert(Op.getValueType() == MVT::i64 &&(static_cast <bool> (Op.getValueType() == MVT::i64 &&
"unexpected type for custom lowering DIV") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i64 && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9479, __extension__ __PRETTY_FUNCTION__))
9479 "unexpected type for custom lowering DIV")(static_cast <bool> (Op.getValueType() == MVT::i64 &&
"unexpected type for custom lowering DIV") ? void (0) : __assert_fail
("Op.getValueType() == MVT::i64 && \"unexpected type for custom lowering DIV\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9479, __extension__ __PRETTY_FUNCTION__))
;
9480 SDLoc dl(Op);
9481
9482 SDValue DBZCHK = WinDBZCheckDenominator(DAG, Op.getNode(), DAG.getEntryNode());
9483
9484 SDValue Result = LowerWindowsDIVLibCall(Op, DAG, Signed, DBZCHK);
9485
9486 SDValue Lower = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Result);
9487 SDValue Upper = DAG.getNode(ISD::SRL, dl, MVT::i64, Result,
9488 DAG.getConstant(32, dl, TLI.getPointerTy(DL)));
9489 Upper = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Upper);
9490
9491 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lower, Upper));
9492}
9493
9494static SDValue LowerPredicateLoad(SDValue Op, SelectionDAG &DAG) {
9495 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
9496 EVT MemVT = LD->getMemoryVT();
9497 assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) &&(static_cast <bool> ((MemVT == MVT::v4i1 || MemVT == MVT
::v8i1 || MemVT == MVT::v16i1) && "Expected a predicate type!"
) ? void (0) : __assert_fail ("(MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && \"Expected a predicate type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9498, __extension__ __PRETTY_FUNCTION__))
9498 "Expected a predicate type!")(static_cast <bool> ((MemVT == MVT::v4i1 || MemVT == MVT
::v8i1 || MemVT == MVT::v16i1) && "Expected a predicate type!"
) ? void (0) : __assert_fail ("(MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && \"Expected a predicate type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9498, __extension__ __PRETTY_FUNCTION__))
;
9499 assert(MemVT == Op.getValueType())(static_cast <bool> (MemVT == Op.getValueType()) ? void
(0) : __assert_fail ("MemVT == Op.getValueType()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9499, __extension__ __PRETTY_FUNCTION__))
;
9500 assert(LD->getExtensionType() == ISD::NON_EXTLOAD &&(static_cast <bool> (LD->getExtensionType() == ISD::
NON_EXTLOAD && "Expected a non-extending load") ? void
(0) : __assert_fail ("LD->getExtensionType() == ISD::NON_EXTLOAD && \"Expected a non-extending load\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9501, __extension__ __PRETTY_FUNCTION__))
9501 "Expected a non-extending load")(static_cast <bool> (LD->getExtensionType() == ISD::
NON_EXTLOAD && "Expected a non-extending load") ? void
(0) : __assert_fail ("LD->getExtensionType() == ISD::NON_EXTLOAD && \"Expected a non-extending load\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9501, __extension__ __PRETTY_FUNCTION__))
;
9502 assert(LD->isUnindexed() && "Expected a unindexed load")(static_cast <bool> (LD->isUnindexed() && "Expected a unindexed load"
) ? void (0) : __assert_fail ("LD->isUnindexed() && \"Expected a unindexed load\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9502, __extension__ __PRETTY_FUNCTION__))
;
9503
9504 // The basic MVE VLDR on a v4i1/v8i1 actually loads the entire 16bit
9505 // predicate, with the "v4i1" bits spread out over the 16 bits loaded. We
9506 // need to make sure that 8/4 bits are actually loaded into the correct
9507 // place, which means loading the value and then shuffling the values into
9508 // the bottom bits of the predicate.
9509 // Equally, VLDR for an v16i1 will actually load 32bits (so will be incorrect
9510 // for BE).
9511 // Speaking of BE, apparently the rest of llvm will assume a reverse order to
9512 // a natural VMSR(load), so needs to be reversed.
9513
9514 SDLoc dl(Op);
9515 SDValue Load = DAG.getExtLoad(
9516 ISD::EXTLOAD, dl, MVT::i32, LD->getChain(), LD->getBasePtr(),
9517 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
9518 LD->getMemOperand());
9519 SDValue Val = Load;
9520 if (DAG.getDataLayout().isBigEndian())
9521 Val = DAG.getNode(ISD::SRL, dl, MVT::i32,
9522 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, Load),
9523 DAG.getConstant(32 - MemVT.getSizeInBits(), dl, MVT::i32));
9524 SDValue Pred = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::v16i1, Val);
9525 if (MemVT != MVT::v16i1)
9526 Pred = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MemVT, Pred,
9527 DAG.getConstant(0, dl, MVT::i32));
9528 return DAG.getMergeValues({Pred, Load.getValue(1)}, dl);
9529}
9530
9531void ARMTargetLowering::LowerLOAD(SDNode *N, SmallVectorImpl<SDValue> &Results,
9532 SelectionDAG &DAG) const {
9533 LoadSDNode *LD = cast<LoadSDNode>(N);
9534 EVT MemVT = LD->getMemoryVT();
9535 assert(LD->isUnindexed() && "Loads should be unindexed at this point.")(static_cast <bool> (LD->isUnindexed() && "Loads should be unindexed at this point."
) ? void (0) : __assert_fail ("LD->isUnindexed() && \"Loads should be unindexed at this point.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9535, __extension__ __PRETTY_FUNCTION__))
;
9536
9537 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9538 !Subtarget->isThumb1Only() && LD->isVolatile()) {
9539 SDLoc dl(N);
9540 SDValue Result = DAG.getMemIntrinsicNode(
9541 ARMISD::LDRD, dl, DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
9542 {LD->getChain(), LD->getBasePtr()}, MemVT, LD->getMemOperand());
9543 SDValue Lo = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 0 : 1);
9544 SDValue Hi = Result.getValue(DAG.getDataLayout().isLittleEndian() ? 1 : 0);
9545 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
9546 Results.append({Pair, Result.getValue(2)});
9547 }
9548}
9549
9550static SDValue LowerPredicateStore(SDValue Op, SelectionDAG &DAG) {
9551 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
9552 EVT MemVT = ST->getMemoryVT();
9553 assert((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) &&(static_cast <bool> ((MemVT == MVT::v4i1 || MemVT == MVT
::v8i1 || MemVT == MVT::v16i1) && "Expected a predicate type!"
) ? void (0) : __assert_fail ("(MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && \"Expected a predicate type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9554, __extension__ __PRETTY_FUNCTION__))
9554 "Expected a predicate type!")(static_cast <bool> ((MemVT == MVT::v4i1 || MemVT == MVT
::v8i1 || MemVT == MVT::v16i1) && "Expected a predicate type!"
) ? void (0) : __assert_fail ("(MemVT == MVT::v4i1 || MemVT == MVT::v8i1 || MemVT == MVT::v16i1) && \"Expected a predicate type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9554, __extension__ __PRETTY_FUNCTION__))
;
9555 assert(MemVT == ST->getValue().getValueType())(static_cast <bool> (MemVT == ST->getValue().getValueType
()) ? void (0) : __assert_fail ("MemVT == ST->getValue().getValueType()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9555, __extension__ __PRETTY_FUNCTION__))
;
9556 assert(!ST->isTruncatingStore() && "Expected a non-extending store")(static_cast <bool> (!ST->isTruncatingStore() &&
"Expected a non-extending store") ? void (0) : __assert_fail
("!ST->isTruncatingStore() && \"Expected a non-extending store\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9556, __extension__ __PRETTY_FUNCTION__))
;
9557 assert(ST->isUnindexed() && "Expected a unindexed store")(static_cast <bool> (ST->isUnindexed() && "Expected a unindexed store"
) ? void (0) : __assert_fail ("ST->isUnindexed() && \"Expected a unindexed store\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9557, __extension__ __PRETTY_FUNCTION__))
;
9558
9559 // Only store the v4i1 or v8i1 worth of bits, via a buildvector with top bits
9560 // unset and a scalar store.
9561 SDLoc dl(Op);
9562 SDValue Build = ST->getValue();
9563 if (MemVT != MVT::v16i1) {
9564 SmallVector<SDValue, 16> Ops;
9565 for (unsigned I = 0; I < MemVT.getVectorNumElements(); I++) {
9566 unsigned Elt = DAG.getDataLayout().isBigEndian()
9567 ? MemVT.getVectorNumElements() - I - 1
9568 : I;
9569 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Build,
9570 DAG.getConstant(Elt, dl, MVT::i32)));
9571 }
9572 for (unsigned I = MemVT.getVectorNumElements(); I < 16; I++)
9573 Ops.push_back(DAG.getUNDEF(MVT::i32));
9574 Build = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v16i1, Ops);
9575 }
9576 SDValue GRP = DAG.getNode(ARMISD::PREDICATE_CAST, dl, MVT::i32, Build);
9577 if (MemVT == MVT::v16i1 && DAG.getDataLayout().isBigEndian())
9578 GRP = DAG.getNode(ISD::SRL, dl, MVT::i32,
9579 DAG.getNode(ISD::BITREVERSE, dl, MVT::i32, GRP),
9580 DAG.getConstant(16, dl, MVT::i32));
9581 return DAG.getTruncStore(
9582 ST->getChain(), dl, GRP, ST->getBasePtr(),
9583 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
9584 ST->getMemOperand());
9585}
9586
9587static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG,
9588 const ARMSubtarget *Subtarget) {
9589 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
9590 EVT MemVT = ST->getMemoryVT();
9591 assert(ST->isUnindexed() && "Stores should be unindexed at this point.")(static_cast <bool> (ST->isUnindexed() && "Stores should be unindexed at this point."
) ? void (0) : __assert_fail ("ST->isUnindexed() && \"Stores should be unindexed at this point.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9591, __extension__ __PRETTY_FUNCTION__))
;
9592
9593 if (MemVT == MVT::i64 && Subtarget->hasV5TEOps() &&
9594 !Subtarget->isThumb1Only() && ST->isVolatile()) {
9595 SDNode *N = Op.getNode();
9596 SDLoc dl(N);
9597
9598 SDValue Lo = DAG.getNode(
9599 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
9600 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 0 : 1, dl,
9601 MVT::i32));
9602 SDValue Hi = DAG.getNode(
9603 ISD::EXTRACT_ELEMENT, dl, MVT::i32, ST->getValue(),
9604 DAG.getTargetConstant(DAG.getDataLayout().isLittleEndian() ? 1 : 0, dl,
9605 MVT::i32));
9606
9607 return DAG.getMemIntrinsicNode(ARMISD::STRD, dl, DAG.getVTList(MVT::Other),
9608 {ST->getChain(), Lo, Hi, ST->getBasePtr()},
9609 MemVT, ST->getMemOperand());
9610 } else if (Subtarget->hasMVEIntegerOps() &&
9611 ((MemVT == MVT::v4i1 || MemVT == MVT::v8i1 ||
9612 MemVT == MVT::v16i1))) {
9613 return LowerPredicateStore(Op, DAG);
9614 }
9615
9616 return SDValue();
9617}
9618
9619static bool isZeroVector(SDValue N) {
9620 return (ISD::isBuildVectorAllZeros(N.getNode()) ||
9621 (N->getOpcode() == ARMISD::VMOVIMM &&
9622 isNullConstant(N->getOperand(0))));
9623}
9624
9625static SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) {
9626 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
9627 MVT VT = Op.getSimpleValueType();
9628 SDValue Mask = N->getMask();
9629 SDValue PassThru = N->getPassThru();
9630 SDLoc dl(Op);
9631
9632 if (isZeroVector(PassThru))
9633 return Op;
9634
9635 // MVE Masked loads use zero as the passthru value. Here we convert undef to
9636 // zero too, and other values are lowered to a select.
9637 SDValue ZeroVec = DAG.getNode(ARMISD::VMOVIMM, dl, VT,
9638 DAG.getTargetConstant(0, dl, MVT::i32));
9639 SDValue NewLoad = DAG.getMaskedLoad(
9640 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask, ZeroVec,
9641 N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
9642 N->getExtensionType(), N->isExpandingLoad());
9643 SDValue Combo = NewLoad;
9644 bool PassThruIsCastZero = (PassThru.getOpcode() == ISD::BITCAST ||
9645 PassThru.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
9646 isZeroVector(PassThru->getOperand(0));
9647 if (!PassThru.isUndef() && !PassThruIsCastZero)
9648 Combo = DAG.getNode(ISD::VSELECT, dl, VT, Mask, NewLoad, PassThru);
9649 return DAG.getMergeValues({Combo, NewLoad.getValue(1)}, dl);
9650}
9651
9652static SDValue LowerVecReduce(SDValue Op, SelectionDAG &DAG,
9653 const ARMSubtarget *ST) {
9654 if (!ST->hasMVEIntegerOps())
9655 return SDValue();
9656
9657 SDLoc dl(Op);
9658 unsigned BaseOpcode = 0;
9659 switch (Op->getOpcode()) {
9660 default: llvm_unreachable("Expected VECREDUCE opcode")::llvm::llvm_unreachable_internal("Expected VECREDUCE opcode"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9660)
;
9661 case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break;
9662 case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break;
9663 case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break;
9664 case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break;
9665 case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break;
9666 case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break;
9667 case ISD::VECREDUCE_FMAX: BaseOpcode = ISD::FMAXNUM; break;
9668 case ISD::VECREDUCE_FMIN: BaseOpcode = ISD::FMINNUM; break;
9669 }
9670
9671 SDValue Op0 = Op->getOperand(0);
9672 EVT VT = Op0.getValueType();
9673 EVT EltVT = VT.getVectorElementType();
9674 unsigned NumElts = VT.getVectorNumElements();
9675 unsigned NumActiveLanes = NumElts;
9676
9677 assert((NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 ||(static_cast <bool> ((NumActiveLanes == 16 || NumActiveLanes
== 8 || NumActiveLanes == 4 || NumActiveLanes == 2) &&
"Only expected a power 2 vector size") ? void (0) : __assert_fail
("(NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 || NumActiveLanes == 2) && \"Only expected a power 2 vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9679, __extension__ __PRETTY_FUNCTION__))
9678 NumActiveLanes == 2) &&(static_cast <bool> ((NumActiveLanes == 16 || NumActiveLanes
== 8 || NumActiveLanes == 4 || NumActiveLanes == 2) &&
"Only expected a power 2 vector size") ? void (0) : __assert_fail
("(NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 || NumActiveLanes == 2) && \"Only expected a power 2 vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9679, __extension__ __PRETTY_FUNCTION__))
9679 "Only expected a power 2 vector size")(static_cast <bool> ((NumActiveLanes == 16 || NumActiveLanes
== 8 || NumActiveLanes == 4 || NumActiveLanes == 2) &&
"Only expected a power 2 vector size") ? void (0) : __assert_fail
("(NumActiveLanes == 16 || NumActiveLanes == 8 || NumActiveLanes == 4 || NumActiveLanes == 2) && \"Only expected a power 2 vector size\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9679, __extension__ __PRETTY_FUNCTION__))
;
9680
9681 // Use Mul(X, Rev(X)) until 4 items remain. Going down to 4 vector elements
9682 // allows us to easily extract vector elements from the lanes.
9683 while (NumActiveLanes > 4) {
9684 unsigned RevOpcode = NumActiveLanes == 16 ? ARMISD::VREV16 : ARMISD::VREV32;
9685 SDValue Rev = DAG.getNode(RevOpcode, dl, VT, Op0);
9686 Op0 = DAG.getNode(BaseOpcode, dl, VT, Op0, Rev);
9687 NumActiveLanes /= 2;
9688 }
9689
9690 SDValue Res;
9691 if (NumActiveLanes == 4) {
9692 // The remaining 4 elements are summed sequentially
9693 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9694 DAG.getConstant(0 * NumElts / 4, dl, MVT::i32));
9695 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9696 DAG.getConstant(1 * NumElts / 4, dl, MVT::i32));
9697 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9698 DAG.getConstant(2 * NumElts / 4, dl, MVT::i32));
9699 SDValue Ext3 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9700 DAG.getConstant(3 * NumElts / 4, dl, MVT::i32));
9701 SDValue Res0 = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
9702 SDValue Res1 = DAG.getNode(BaseOpcode, dl, EltVT, Ext2, Ext3, Op->getFlags());
9703 Res = DAG.getNode(BaseOpcode, dl, EltVT, Res0, Res1, Op->getFlags());
9704 } else {
9705 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9706 DAG.getConstant(0, dl, MVT::i32));
9707 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
9708 DAG.getConstant(1, dl, MVT::i32));
9709 Res = DAG.getNode(BaseOpcode, dl, EltVT, Ext0, Ext1, Op->getFlags());
9710 }
9711
9712 // Result type may be wider than element type.
9713 if (EltVT != Op->getValueType(0))
9714 Res = DAG.getNode(ISD::ANY_EXTEND, dl, Op->getValueType(0), Res);
9715 return Res;
9716}
9717
9718static SDValue LowerVecReduceF(SDValue Op, SelectionDAG &DAG,
9719 const ARMSubtarget *ST) {
9720 if (!ST->hasMVEFloatOps())
9721 return SDValue();
9722 return LowerVecReduce(Op, DAG, ST);
9723}
9724
9725static SDValue LowerAtomicLoadStore(SDValue Op, SelectionDAG &DAG) {
9726 if (isStrongerThanMonotonic(cast<AtomicSDNode>(Op)->getOrdering()))
9727 // Acquire/Release load/store is not legal for targets without a dmb or
9728 // equivalent available.
9729 return SDValue();
9730
9731 // Monotonic load/store is legal for all targets.
9732 return Op;
9733}
9734
9735static void ReplaceREADCYCLECOUNTER(SDNode *N,
9736 SmallVectorImpl<SDValue> &Results,
9737 SelectionDAG &DAG,
9738 const ARMSubtarget *Subtarget) {
9739 SDLoc DL(N);
9740 // Under Power Management extensions, the cycle-count is:
9741 // mrc p15, #0, <Rt>, c9, c13, #0
9742 SDValue Ops[] = { N->getOperand(0), // Chain
9743 DAG.getTargetConstant(Intrinsic::arm_mrc, DL, MVT::i32),
9744 DAG.getTargetConstant(15, DL, MVT::i32),
9745 DAG.getTargetConstant(0, DL, MVT::i32),
9746 DAG.getTargetConstant(9, DL, MVT::i32),
9747 DAG.getTargetConstant(13, DL, MVT::i32),
9748 DAG.getTargetConstant(0, DL, MVT::i32)
9749 };
9750
9751 SDValue Cycles32 = DAG.getNode(ISD::INTRINSIC_W_CHAIN, DL,
9752 DAG.getVTList(MVT::i32, MVT::Other), Ops);
9753 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Cycles32,
9754 DAG.getConstant(0, DL, MVT::i32)));
9755 Results.push_back(Cycles32.getValue(1));
9756}
9757
9758static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V) {
9759 SDLoc dl(V.getNode());
9760 SDValue VLo = DAG.getAnyExtOrTrunc(V, dl, MVT::i32);
9761 SDValue VHi = DAG.getAnyExtOrTrunc(
9762 DAG.getNode(ISD::SRL, dl, MVT::i64, V, DAG.getConstant(32, dl, MVT::i32)),
9763 dl, MVT::i32);
9764 bool isBigEndian = DAG.getDataLayout().isBigEndian();
9765 if (isBigEndian)
9766 std::swap (VLo, VHi);
9767 SDValue RegClass =
9768 DAG.getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
9769 SDValue SubReg0 = DAG.getTargetConstant(ARM::gsub_0, dl, MVT::i32);
9770 SDValue SubReg1 = DAG.getTargetConstant(ARM::gsub_1, dl, MVT::i32);
9771 const SDValue Ops[] = { RegClass, VLo, SubReg0, VHi, SubReg1 };
9772 return SDValue(
9773 DAG.getMachineNode(TargetOpcode::REG_SEQUENCE, dl, MVT::Untyped, Ops), 0);
9774}
9775
9776static void ReplaceCMP_SWAP_64Results(SDNode *N,
9777 SmallVectorImpl<SDValue> & Results,
9778 SelectionDAG &DAG) {
9779 assert(N->getValueType(0) == MVT::i64 &&(static_cast <bool> (N->getValueType(0) == MVT::i64 &&
"AtomicCmpSwap on types less than 64 should be legal") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"AtomicCmpSwap on types less than 64 should be legal\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9780, __extension__ __PRETTY_FUNCTION__))
9780 "AtomicCmpSwap on types less than 64 should be legal")(static_cast <bool> (N->getValueType(0) == MVT::i64 &&
"AtomicCmpSwap on types less than 64 should be legal") ? void
(0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"AtomicCmpSwap on types less than 64 should be legal\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9780, __extension__ __PRETTY_FUNCTION__))
;
9781 SDValue Ops[] = {N->getOperand(1),
9782 createGPRPairNode(DAG, N->getOperand(2)),
9783 createGPRPairNode(DAG, N->getOperand(3)),
9784 N->getOperand(0)};
9785 SDNode *CmpSwap = DAG.getMachineNode(
9786 ARM::CMP_SWAP_64, SDLoc(N),
9787 DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops);
9788
9789 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
9790 DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
9791
9792 bool isBigEndian = DAG.getDataLayout().isBigEndian();
9793
9794 SDValue Lo =
9795 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_1 : ARM::gsub_0,
9796 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
9797 SDValue Hi =
9798 DAG.getTargetExtractSubreg(isBigEndian ? ARM::gsub_0 : ARM::gsub_1,
9799 SDLoc(N), MVT::i32, SDValue(CmpSwap, 0));
9800 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), MVT::i64, Lo, Hi));
9801 Results.push_back(SDValue(CmpSwap, 2));
9802}
9803
9804SDValue ARMTargetLowering::LowerFSETCC(SDValue Op, SelectionDAG &DAG) const {
9805 SDLoc dl(Op);
9806 EVT VT = Op.getValueType();
9807 SDValue Chain = Op.getOperand(0);
9808 SDValue LHS = Op.getOperand(1);
9809 SDValue RHS = Op.getOperand(2);
9810 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(3))->get();
9811 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
9812
9813 // If we don't have instructions of this float type then soften to a libcall
9814 // and use SETCC instead.
9815 if (isUnsupportedFloatingType(LHS.getValueType())) {
9816 DAG.getTargetLoweringInfo().softenSetCCOperands(
9817 DAG, LHS.getValueType(), LHS, RHS, CC, dl, LHS, RHS, Chain, IsSignaling);
9818 if (!RHS.getNode()) {
9819 RHS = DAG.getConstant(0, dl, LHS.getValueType());
9820 CC = ISD::SETNE;
9821 }
9822 SDValue Result = DAG.getNode(ISD::SETCC, dl, VT, LHS, RHS,
9823 DAG.getCondCode(CC));
9824 return DAG.getMergeValues({Result, Chain}, dl);
9825 }
9826
9827 ARMCC::CondCodes CondCode, CondCode2;
9828 FPCCToARMCC(CC, CondCode, CondCode2);
9829
9830 // FIXME: Chain is not handled correctly here. Currently the FPSCR is implicit
9831 // in CMPFP and CMPFPE, but instead it should be made explicit by these
9832 // instructions using a chain instead of glue. This would also fix the problem
9833 // here (and also in LowerSELECT_CC) where we generate two comparisons when
9834 // CondCode2 != AL.
9835 SDValue True = DAG.getConstant(1, dl, VT);
9836 SDValue False = DAG.getConstant(0, dl, VT);
9837 SDValue ARMcc = DAG.getConstant(CondCode, dl, MVT::i32);
9838 SDValue CCR = DAG.getRegister(ARM::CPSR, MVT::i32);
9839 SDValue Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
9840 SDValue Result = getCMOV(dl, VT, False, True, ARMcc, CCR, Cmp, DAG);
9841 if (CondCode2 != ARMCC::AL) {
9842 ARMcc = DAG.getConstant(CondCode2, dl, MVT::i32);
9843 Cmp = getVFPCmp(LHS, RHS, DAG, dl, IsSignaling);
9844 Result = getCMOV(dl, VT, Result, True, ARMcc, CCR, Cmp, DAG);
9845 }
9846 return DAG.getMergeValues({Result, Chain}, dl);
9847}
9848
9849SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
9850 LLVM_DEBUG(dbgs() << "Lowering node: "; Op.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { dbgs() << "Lowering node: "; Op.dump();
} } while (false)
;
9851 switch (Op.getOpcode()) {
9852 default: llvm_unreachable("Don't know how to custom lower this!")::llvm::llvm_unreachable_internal("Don't know how to custom lower this!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9852)
;
9853 case ISD::WRITE_REGISTER: return LowerWRITE_REGISTER(Op, DAG);
9854 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
9855 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
9856 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
9857 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
9858 case ISD::SELECT: return LowerSELECT(Op, DAG);
9859 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
9860 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
9861 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
9862 case ISD::BR_JT: return LowerBR_JT(Op, DAG);
9863 case ISD::VASTART: return LowerVASTART(Op, DAG);
9864 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, DAG, Subtarget);
9865 case ISD::PREFETCH: return LowerPREFETCH(Op, DAG, Subtarget);
9866 case ISD::SINT_TO_FP:
9867 case ISD::UINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
9868 case ISD::STRICT_FP_TO_SINT:
9869 case ISD::STRICT_FP_TO_UINT:
9870 case ISD::FP_TO_SINT:
9871 case ISD::FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
9872 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
9873 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
9874 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
9875 case ISD::EH_SJLJ_SETJMP: return LowerEH_SJLJ_SETJMP(Op, DAG);
9876 case ISD::EH_SJLJ_LONGJMP: return LowerEH_SJLJ_LONGJMP(Op, DAG);
9877 case ISD::EH_SJLJ_SETUP_DISPATCH: return LowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
9878 case ISD::INTRINSIC_VOID: return LowerINTRINSIC_VOID(Op, DAG, Subtarget);
9879 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG,
9880 Subtarget);
9881 case ISD::BITCAST: return ExpandBITCAST(Op.getNode(), DAG, Subtarget);
9882 case ISD::SHL:
9883 case ISD::SRL:
9884 case ISD::SRA: return LowerShift(Op.getNode(), DAG, Subtarget);
9885 case ISD::SREM: return LowerREM(Op.getNode(), DAG);
9886 case ISD::UREM: return LowerREM(Op.getNode(), DAG);
9887 case ISD::SHL_PARTS: return LowerShiftLeftParts(Op, DAG);
9888 case ISD::SRL_PARTS:
9889 case ISD::SRA_PARTS: return LowerShiftRightParts(Op, DAG);
9890 case ISD::CTTZ:
9891 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op.getNode(), DAG, Subtarget);
9892 case ISD::CTPOP: return LowerCTPOP(Op.getNode(), DAG, Subtarget);
9893 case ISD::SETCC: return LowerVSETCC(Op, DAG, Subtarget);
9894 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
9895 case ISD::ConstantFP: return LowerConstantFP(Op, DAG, Subtarget);
9896 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG, Subtarget);
9897 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
9898 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op, DAG, Subtarget);
9899 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
9900 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG, Subtarget);
9901 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG, Subtarget);
9902 case ISD::TRUNCATE: return LowerTruncatei1(Op, DAG, Subtarget);
9903 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
9904 case ISD::SET_ROUNDING: return LowerSET_ROUNDING(Op, DAG);
9905 case ISD::MUL: return LowerMUL(Op, DAG);
9906 case ISD::SDIV:
9907 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector())
9908 return LowerDIV_Windows(Op, DAG, /* Signed */ true);
9909 return LowerSDIV(Op, DAG, Subtarget);
9910 case ISD::UDIV:
9911 if (Subtarget->isTargetWindows() && !Op.getValueType().isVector())
9912 return LowerDIV_Windows(Op, DAG, /* Signed */ false);
9913 return LowerUDIV(Op, DAG, Subtarget);
9914 case ISD::ADDCARRY:
9915 case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG);
9916 case ISD::SADDO:
9917 case ISD::SSUBO:
9918 return LowerSignedALUO(Op, DAG);
9919 case ISD::UADDO:
9920 case ISD::USUBO:
9921 return LowerUnsignedALUO(Op, DAG);
9922 case ISD::SADDSAT:
9923 case ISD::SSUBSAT:
9924 return LowerSADDSUBSAT(Op, DAG, Subtarget);
9925 case ISD::LOAD:
9926 return LowerPredicateLoad(Op, DAG);
9927 case ISD::STORE:
9928 return LowerSTORE(Op, DAG, Subtarget);
9929 case ISD::MLOAD:
9930 return LowerMLOAD(Op, DAG);
9931 case ISD::VECREDUCE_MUL:
9932 case ISD::VECREDUCE_AND:
9933 case ISD::VECREDUCE_OR:
9934 case ISD::VECREDUCE_XOR:
9935 return LowerVecReduce(Op, DAG, Subtarget);
9936 case ISD::VECREDUCE_FADD:
9937 case ISD::VECREDUCE_FMUL:
9938 case ISD::VECREDUCE_FMIN:
9939 case ISD::VECREDUCE_FMAX:
9940 return LowerVecReduceF(Op, DAG, Subtarget);
9941 case ISD::ATOMIC_LOAD:
9942 case ISD::ATOMIC_STORE: return LowerAtomicLoadStore(Op, DAG);
9943 case ISD::FSINCOS: return LowerFSINCOS(Op, DAG);
9944 case ISD::SDIVREM:
9945 case ISD::UDIVREM: return LowerDivRem(Op, DAG);
9946 case ISD::DYNAMIC_STACKALLOC:
9947 if (Subtarget->isTargetWindows())
9948 return LowerDYNAMIC_STACKALLOC(Op, DAG);
9949 llvm_unreachable("Don't know how to custom lower this!")::llvm::llvm_unreachable_internal("Don't know how to custom lower this!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9949)
;
9950 case ISD::STRICT_FP_ROUND:
9951 case ISD::FP_ROUND: return LowerFP_ROUND(Op, DAG);
9952 case ISD::STRICT_FP_EXTEND:
9953 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
9954 case ISD::STRICT_FSETCC:
9955 case ISD::STRICT_FSETCCS: return LowerFSETCC(Op, DAG);
9956 case ARMISD::WIN__DBZCHK: return SDValue();
9957 }
9958}
9959
9960static void ReplaceLongIntrinsic(SDNode *N, SmallVectorImpl<SDValue> &Results,
9961 SelectionDAG &DAG) {
9962 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
9963 unsigned Opc = 0;
9964 if (IntNo == Intrinsic::arm_smlald)
9965 Opc = ARMISD::SMLALD;
9966 else if (IntNo == Intrinsic::arm_smlaldx)
9967 Opc = ARMISD::SMLALDX;
9968 else if (IntNo == Intrinsic::arm_smlsld)
9969 Opc = ARMISD::SMLSLD;
9970 else if (IntNo == Intrinsic::arm_smlsldx)
9971 Opc = ARMISD::SMLSLDX;
9972 else
9973 return;
9974
9975 SDLoc dl(N);
9976 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9977 N->getOperand(3),
9978 DAG.getConstant(0, dl, MVT::i32));
9979 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32,
9980 N->getOperand(3),
9981 DAG.getConstant(1, dl, MVT::i32));
9982
9983 SDValue LongMul = DAG.getNode(Opc, dl,
9984 DAG.getVTList(MVT::i32, MVT::i32),
9985 N->getOperand(1), N->getOperand(2),
9986 Lo, Hi);
9987 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64,
9988 LongMul.getValue(0), LongMul.getValue(1)));
9989}
9990
9991/// ReplaceNodeResults - Replace the results of node with an illegal result
9992/// type with new values built out of custom code.
9993void ARMTargetLowering::ReplaceNodeResults(SDNode *N,
9994 SmallVectorImpl<SDValue> &Results,
9995 SelectionDAG &DAG) const {
9996 SDValue Res;
9997 switch (N->getOpcode()) {
9998 default:
9999 llvm_unreachable("Don't know how to custom expand this!")::llvm::llvm_unreachable_internal("Don't know how to custom expand this!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 9999)
;
10000 case ISD::READ_REGISTER:
10001 ExpandREAD_REGISTER(N, Results, DAG);
10002 break;
10003 case ISD::BITCAST:
10004 Res = ExpandBITCAST(N, DAG, Subtarget);
10005 break;
10006 case ISD::SRL:
10007 case ISD::SRA:
10008 case ISD::SHL:
10009 Res = Expand64BitShift(N, DAG, Subtarget);
10010 break;
10011 case ISD::SREM:
10012 case ISD::UREM:
10013 Res = LowerREM(N, DAG);
10014 break;
10015 case ISD::SDIVREM:
10016 case ISD::UDIVREM:
10017 Res = LowerDivRem(SDValue(N, 0), DAG);
10018 assert(Res.getNumOperands() == 2 && "DivRem needs two values")(static_cast <bool> (Res.getNumOperands() == 2 &&
"DivRem needs two values") ? void (0) : __assert_fail ("Res.getNumOperands() == 2 && \"DivRem needs two values\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10018, __extension__ __PRETTY_FUNCTION__))
;
10019 Results.push_back(Res.getValue(0));
10020 Results.push_back(Res.getValue(1));
10021 return;
10022 case ISD::SADDSAT:
10023 case ISD::SSUBSAT:
10024 Res = LowerSADDSUBSAT(SDValue(N, 0), DAG, Subtarget);
10025 break;
10026 case ISD::READCYCLECOUNTER:
10027 ReplaceREADCYCLECOUNTER(N, Results, DAG, Subtarget);
10028 return;
10029 case ISD::UDIV:
10030 case ISD::SDIV:
10031 assert(Subtarget->isTargetWindows() && "can only expand DIV on Windows")(static_cast <bool> (Subtarget->isTargetWindows() &&
"can only expand DIV on Windows") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"can only expand DIV on Windows\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10031, __extension__ __PRETTY_FUNCTION__))
;
10032 return ExpandDIV_Windows(SDValue(N, 0), DAG, N->getOpcode() == ISD::SDIV,
10033 Results);
10034 case ISD::ATOMIC_CMP_SWAP:
10035 ReplaceCMP_SWAP_64Results(N, Results, DAG);
10036 return;
10037 case ISD::INTRINSIC_WO_CHAIN:
10038 return ReplaceLongIntrinsic(N, Results, DAG);
10039 case ISD::ABS:
10040 lowerABS(N, Results, DAG);
10041 return ;
10042 case ISD::LOAD:
10043 LowerLOAD(N, Results, DAG);
10044 break;
10045 }
10046 if (Res.getNode())
10047 Results.push_back(Res);
10048}
10049
10050//===----------------------------------------------------------------------===//
10051// ARM Scheduler Hooks
10052//===----------------------------------------------------------------------===//
10053
10054/// SetupEntryBlockForSjLj - Insert code into the entry block that creates and
10055/// registers the function context.
10056void ARMTargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
10057 MachineBasicBlock *MBB,
10058 MachineBasicBlock *DispatchBB,
10059 int FI) const {
10060 assert(!Subtarget->isROPI() && !Subtarget->isRWPI() &&(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported with SjLj"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported with SjLj\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10061, __extension__ __PRETTY_FUNCTION__))
10061 "ROPI/RWPI not currently supported with SjLj")(static_cast <bool> (!Subtarget->isROPI() &&
!Subtarget->isRWPI() && "ROPI/RWPI not currently supported with SjLj"
) ? void (0) : __assert_fail ("!Subtarget->isROPI() && !Subtarget->isRWPI() && \"ROPI/RWPI not currently supported with SjLj\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10061, __extension__ __PRETTY_FUNCTION__))
;
10062 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10063 DebugLoc dl = MI.getDebugLoc();
10064 MachineFunction *MF = MBB->getParent();
10065 MachineRegisterInfo *MRI = &MF->getRegInfo();
10066 MachineConstantPool *MCP = MF->getConstantPool();
10067 ARMFunctionInfo *AFI = MF->getInfo<ARMFunctionInfo>();
10068 const Function &F = MF->getFunction();
10069
10070 bool isThumb = Subtarget->isThumb();
10071 bool isThumb2 = Subtarget->isThumb2();
10072
10073 unsigned PCLabelId = AFI->createPICLabelUId();
10074 unsigned PCAdj = (isThumb || isThumb2) ? 4 : 8;
10075 ARMConstantPoolValue *CPV =
10076 ARMConstantPoolMBB::Create(F.getContext(), DispatchBB, PCLabelId, PCAdj);
10077 unsigned CPI = MCP->getConstantPoolIndex(CPV, Align(4));
10078
10079 const TargetRegisterClass *TRC = isThumb ? &ARM::tGPRRegClass
10080 : &ARM::GPRRegClass;
10081
10082 // Grab constant pool and fixed stack memory operands.
10083 MachineMemOperand *CPMMO =
10084 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
10085 MachineMemOperand::MOLoad, 4, Align(4));
10086
10087 MachineMemOperand *FIMMOSt =
10088 MF->getMachineMemOperand(MachinePointerInfo::getFixedStack(*MF, FI),
10089 MachineMemOperand::MOStore, 4, Align(4));
10090
10091 // Load the address of the dispatch MBB into the jump buffer.
10092 if (isThumb2) {
10093 // Incoming value: jbuf
10094 // ldr.n r5, LCPI1_1
10095 // orr r5, r5, #1
10096 // add r5, pc
10097 // str r5, [$jbuf, #+4] ; &jbuf[1]
10098 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10099 BuildMI(*MBB, MI, dl, TII->get(ARM::t2LDRpci), NewVReg1)
10100 .addConstantPoolIndex(CPI)
10101 .addMemOperand(CPMMO)
10102 .add(predOps(ARMCC::AL));
10103 // Set the low bit because of thumb mode.
10104 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10105 BuildMI(*MBB, MI, dl, TII->get(ARM::t2ORRri), NewVReg2)
10106 .addReg(NewVReg1, RegState::Kill)
10107 .addImm(0x01)
10108 .add(predOps(ARMCC::AL))
10109 .add(condCodeOp());
10110 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10111 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg3)
10112 .addReg(NewVReg2, RegState::Kill)
10113 .addImm(PCLabelId);
10114 BuildMI(*MBB, MI, dl, TII->get(ARM::t2STRi12))
10115 .addReg(NewVReg3, RegState::Kill)
10116 .addFrameIndex(FI)
10117 .addImm(36) // &jbuf[1] :: pc
10118 .addMemOperand(FIMMOSt)
10119 .add(predOps(ARMCC::AL));
10120 } else if (isThumb) {
10121 // Incoming value: jbuf
10122 // ldr.n r1, LCPI1_4
10123 // add r1, pc
10124 // mov r2, #1
10125 // orrs r1, r2
10126 // add r2, $jbuf, #+4 ; &jbuf[1]
10127 // str r1, [r2]
10128 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10129 BuildMI(*MBB, MI, dl, TII->get(ARM::tLDRpci), NewVReg1)
10130 .addConstantPoolIndex(CPI)
10131 .addMemOperand(CPMMO)
10132 .add(predOps(ARMCC::AL));
10133 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10134 BuildMI(*MBB, MI, dl, TII->get(ARM::tPICADD), NewVReg2)
10135 .addReg(NewVReg1, RegState::Kill)
10136 .addImm(PCLabelId);
10137 // Set the low bit because of thumb mode.
10138 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10139 BuildMI(*MBB, MI, dl, TII->get(ARM::tMOVi8), NewVReg3)
10140 .addReg(ARM::CPSR, RegState::Define)
10141 .addImm(1)
10142 .add(predOps(ARMCC::AL));
10143 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10144 BuildMI(*MBB, MI, dl, TII->get(ARM::tORR), NewVReg4)
10145 .addReg(ARM::CPSR, RegState::Define)
10146 .addReg(NewVReg2, RegState::Kill)
10147 .addReg(NewVReg3, RegState::Kill)
10148 .add(predOps(ARMCC::AL));
10149 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10150 BuildMI(*MBB, MI, dl, TII->get(ARM::tADDframe), NewVReg5)
10151 .addFrameIndex(FI)
10152 .addImm(36); // &jbuf[1] :: pc
10153 BuildMI(*MBB, MI, dl, TII->get(ARM::tSTRi))
10154 .addReg(NewVReg4, RegState::Kill)
10155 .addReg(NewVReg5, RegState::Kill)
10156 .addImm(0)
10157 .addMemOperand(FIMMOSt)
10158 .add(predOps(ARMCC::AL));
10159 } else {
10160 // Incoming value: jbuf
10161 // ldr r1, LCPI1_1
10162 // add r1, pc, r1
10163 // str r1, [$jbuf, #+4] ; &jbuf[1]
10164 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10165 BuildMI(*MBB, MI, dl, TII->get(ARM::LDRi12), NewVReg1)
10166 .addConstantPoolIndex(CPI)
10167 .addImm(0)
10168 .addMemOperand(CPMMO)
10169 .add(predOps(ARMCC::AL));
10170 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10171 BuildMI(*MBB, MI, dl, TII->get(ARM::PICADD), NewVReg2)
10172 .addReg(NewVReg1, RegState::Kill)
10173 .addImm(PCLabelId)
10174 .add(predOps(ARMCC::AL));
10175 BuildMI(*MBB, MI, dl, TII->get(ARM::STRi12))
10176 .addReg(NewVReg2, RegState::Kill)
10177 .addFrameIndex(FI)
10178 .addImm(36) // &jbuf[1] :: pc
10179 .addMemOperand(FIMMOSt)
10180 .add(predOps(ARMCC::AL));
10181 }
10182}
10183
10184void ARMTargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
10185 MachineBasicBlock *MBB) const {
10186 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10187 DebugLoc dl = MI.getDebugLoc();
10188 MachineFunction *MF = MBB->getParent();
10189 MachineRegisterInfo *MRI = &MF->getRegInfo();
10190 MachineFrameInfo &MFI = MF->getFrameInfo();
10191 int FI = MFI.getFunctionContextIndex();
10192
10193 const TargetRegisterClass *TRC = Subtarget->isThumb() ? &ARM::tGPRRegClass
10194 : &ARM::GPRnopcRegClass;
10195
10196 // Get a mapping of the call site numbers to all of the landing pads they're
10197 // associated with.
10198 DenseMap<unsigned, SmallVector<MachineBasicBlock*, 2>> CallSiteNumToLPad;
10199 unsigned MaxCSNum = 0;
10200 for (MachineFunction::iterator BB = MF->begin(), E = MF->end(); BB != E;
10201 ++BB) {
10202 if (!BB->isEHPad()) continue;
10203
10204 // FIXME: We should assert that the EH_LABEL is the first MI in the landing
10205 // pad.
10206 for (MachineBasicBlock::iterator
10207 II = BB->begin(), IE = BB->end(); II != IE; ++II) {
10208 if (!II->isEHLabel()) continue;
10209
10210 MCSymbol *Sym = II->getOperand(0).getMCSymbol();
10211 if (!MF->hasCallSiteLandingPad(Sym)) continue;
10212
10213 SmallVectorImpl<unsigned> &CallSiteIdxs = MF->getCallSiteLandingPad(Sym);
10214 for (SmallVectorImpl<unsigned>::iterator
10215 CSI = CallSiteIdxs.begin(), CSE = CallSiteIdxs.end();
10216 CSI != CSE; ++CSI) {
10217 CallSiteNumToLPad[*CSI].push_back(&*BB);
10218 MaxCSNum = std::max(MaxCSNum, *CSI);
10219 }
10220 break;
10221 }
10222 }
10223
10224 // Get an ordered list of the machine basic blocks for the jump table.
10225 std::vector<MachineBasicBlock*> LPadList;
10226 SmallPtrSet<MachineBasicBlock*, 32> InvokeBBs;
10227 LPadList.reserve(CallSiteNumToLPad.size());
10228 for (unsigned I = 1; I <= MaxCSNum; ++I) {
10229 SmallVectorImpl<MachineBasicBlock*> &MBBList = CallSiteNumToLPad[I];
10230 for (SmallVectorImpl<MachineBasicBlock*>::iterator
10231 II = MBBList.begin(), IE = MBBList.end(); II != IE; ++II) {
10232 LPadList.push_back(*II);
10233 InvokeBBs.insert((*II)->pred_begin(), (*II)->pred_end());
10234 }
10235 }
10236
10237 assert(!LPadList.empty() &&(static_cast <bool> (!LPadList.empty() && "No landing pad destinations for the dispatch jump table!"
) ? void (0) : __assert_fail ("!LPadList.empty() && \"No landing pad destinations for the dispatch jump table!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10238, __extension__ __PRETTY_FUNCTION__))
10238 "No landing pad destinations for the dispatch jump table!")(static_cast <bool> (!LPadList.empty() && "No landing pad destinations for the dispatch jump table!"
) ? void (0) : __assert_fail ("!LPadList.empty() && \"No landing pad destinations for the dispatch jump table!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10238, __extension__ __PRETTY_FUNCTION__))
;
10239
10240 // Create the jump table and associated information.
10241 MachineJumpTableInfo *JTI =
10242 MF->getOrCreateJumpTableInfo(MachineJumpTableInfo::EK_Inline);
10243 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
10244
10245 // Create the MBBs for the dispatch code.
10246
10247 // Shove the dispatch's address into the return slot in the function context.
10248 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
10249 DispatchBB->setIsEHPad();
10250
10251 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
10252 unsigned trap_opcode;
10253 if (Subtarget->isThumb())
10254 trap_opcode = ARM::tTRAP;
10255 else
10256 trap_opcode = Subtarget->useNaClTrap() ? ARM::TRAPNaCl : ARM::TRAP;
10257
10258 BuildMI(TrapBB, dl, TII->get(trap_opcode));
10259 DispatchBB->addSuccessor(TrapBB);
10260
10261 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
10262 DispatchBB->addSuccessor(DispContBB);
10263
10264 // Insert and MBBs.
10265 MF->insert(MF->end(), DispatchBB);
10266 MF->insert(MF->end(), DispContBB);
10267 MF->insert(MF->end(), TrapBB);
10268
10269 // Insert code into the entry block that creates and registers the function
10270 // context.
10271 SetupEntryBlockForSjLj(MI, MBB, DispatchBB, FI);
10272
10273 MachineMemOperand *FIMMOLd = MF->getMachineMemOperand(
10274 MachinePointerInfo::getFixedStack(*MF, FI),
10275 MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, 4, Align(4));
10276
10277 MachineInstrBuilder MIB;
10278 MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
10279
10280 const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
10281 const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
10282
10283 // Add a register mask with no preserved registers. This results in all
10284 // registers being marked as clobbered. This can't work if the dispatch block
10285 // is in a Thumb1 function and is linked with ARM code which uses the FP
10286 // registers, as there is no way to preserve the FP registers in Thumb1 mode.
10287 MIB.addRegMask(RI.getSjLjDispatchPreservedMask(*MF));
10288
10289 bool IsPositionIndependent = isPositionIndependent();
10290 unsigned NumLPads = LPadList.size();
10291 if (Subtarget->isThumb2()) {
10292 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10293 BuildMI(DispatchBB, dl, TII->get(ARM::t2LDRi12), NewVReg1)
10294 .addFrameIndex(FI)
10295 .addImm(4)
10296 .addMemOperand(FIMMOLd)
10297 .add(predOps(ARMCC::AL));
10298
10299 if (NumLPads < 256) {
10300 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPri))
10301 .addReg(NewVReg1)
10302 .addImm(LPadList.size())
10303 .add(predOps(ARMCC::AL));
10304 } else {
10305 Register VReg1 = MRI->createVirtualRegister(TRC);
10306 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVi16), VReg1)
10307 .addImm(NumLPads & 0xFFFF)
10308 .add(predOps(ARMCC::AL));
10309
10310 unsigned VReg2 = VReg1;
10311 if ((NumLPads & 0xFFFF0000) != 0) {
10312 VReg2 = MRI->createVirtualRegister(TRC);
10313 BuildMI(DispatchBB, dl, TII->get(ARM::t2MOVTi16), VReg2)
10314 .addReg(VReg1)
10315 .addImm(NumLPads >> 16)
10316 .add(predOps(ARMCC::AL));
10317 }
10318
10319 BuildMI(DispatchBB, dl, TII->get(ARM::t2CMPrr))
10320 .addReg(NewVReg1)
10321 .addReg(VReg2)
10322 .add(predOps(ARMCC::AL));
10323 }
10324
10325 BuildMI(DispatchBB, dl, TII->get(ARM::t2Bcc))
10326 .addMBB(TrapBB)
10327 .addImm(ARMCC::HI)
10328 .addReg(ARM::CPSR);
10329
10330 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10331 BuildMI(DispContBB, dl, TII->get(ARM::t2LEApcrelJT), NewVReg3)
10332 .addJumpTableIndex(MJTI)
10333 .add(predOps(ARMCC::AL));
10334
10335 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10336 BuildMI(DispContBB, dl, TII->get(ARM::t2ADDrs), NewVReg4)
10337 .addReg(NewVReg3, RegState::Kill)
10338 .addReg(NewVReg1)
10339 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))
10340 .add(predOps(ARMCC::AL))
10341 .add(condCodeOp());
10342
10343 BuildMI(DispContBB, dl, TII->get(ARM::t2BR_JT))
10344 .addReg(NewVReg4, RegState::Kill)
10345 .addReg(NewVReg1)
10346 .addJumpTableIndex(MJTI);
10347 } else if (Subtarget->isThumb()) {
10348 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10349 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRspi), NewVReg1)
10350 .addFrameIndex(FI)
10351 .addImm(1)
10352 .addMemOperand(FIMMOLd)
10353 .add(predOps(ARMCC::AL));
10354
10355 if (NumLPads < 256) {
10356 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPi8))
10357 .addReg(NewVReg1)
10358 .addImm(NumLPads)
10359 .add(predOps(ARMCC::AL));
10360 } else {
10361 MachineConstantPool *ConstantPool = MF->getConstantPool();
10362 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
10363 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
10364
10365 // MachineConstantPool wants an explicit alignment.
10366 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10367 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10368
10369 Register VReg1 = MRI->createVirtualRegister(TRC);
10370 BuildMI(DispatchBB, dl, TII->get(ARM::tLDRpci))
10371 .addReg(VReg1, RegState::Define)
10372 .addConstantPoolIndex(Idx)
10373 .add(predOps(ARMCC::AL));
10374 BuildMI(DispatchBB, dl, TII->get(ARM::tCMPr))
10375 .addReg(NewVReg1)
10376 .addReg(VReg1)
10377 .add(predOps(ARMCC::AL));
10378 }
10379
10380 BuildMI(DispatchBB, dl, TII->get(ARM::tBcc))
10381 .addMBB(TrapBB)
10382 .addImm(ARMCC::HI)
10383 .addReg(ARM::CPSR);
10384
10385 Register NewVReg2 = MRI->createVirtualRegister(TRC);
10386 BuildMI(DispContBB, dl, TII->get(ARM::tLSLri), NewVReg2)
10387 .addReg(ARM::CPSR, RegState::Define)
10388 .addReg(NewVReg1)
10389 .addImm(2)
10390 .add(predOps(ARMCC::AL));
10391
10392 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10393 BuildMI(DispContBB, dl, TII->get(ARM::tLEApcrelJT), NewVReg3)
10394 .addJumpTableIndex(MJTI)
10395 .add(predOps(ARMCC::AL));
10396
10397 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10398 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg4)
10399 .addReg(ARM::CPSR, RegState::Define)
10400 .addReg(NewVReg2, RegState::Kill)
10401 .addReg(NewVReg3)
10402 .add(predOps(ARMCC::AL));
10403
10404 MachineMemOperand *JTMMOLd =
10405 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
10406 MachineMemOperand::MOLoad, 4, Align(4));
10407
10408 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10409 BuildMI(DispContBB, dl, TII->get(ARM::tLDRi), NewVReg5)
10410 .addReg(NewVReg4, RegState::Kill)
10411 .addImm(0)
10412 .addMemOperand(JTMMOLd)
10413 .add(predOps(ARMCC::AL));
10414
10415 unsigned NewVReg6 = NewVReg5;
10416 if (IsPositionIndependent) {
10417 NewVReg6 = MRI->createVirtualRegister(TRC);
10418 BuildMI(DispContBB, dl, TII->get(ARM::tADDrr), NewVReg6)
10419 .addReg(ARM::CPSR, RegState::Define)
10420 .addReg(NewVReg5, RegState::Kill)
10421 .addReg(NewVReg3)
10422 .add(predOps(ARMCC::AL));
10423 }
10424
10425 BuildMI(DispContBB, dl, TII->get(ARM::tBR_JTr))
10426 .addReg(NewVReg6, RegState::Kill)
10427 .addJumpTableIndex(MJTI);
10428 } else {
10429 Register NewVReg1 = MRI->createVirtualRegister(TRC);
10430 BuildMI(DispatchBB, dl, TII->get(ARM::LDRi12), NewVReg1)
10431 .addFrameIndex(FI)
10432 .addImm(4)
10433 .addMemOperand(FIMMOLd)
10434 .add(predOps(ARMCC::AL));
10435
10436 if (NumLPads < 256) {
10437 BuildMI(DispatchBB, dl, TII->get(ARM::CMPri))
10438 .addReg(NewVReg1)
10439 .addImm(NumLPads)
10440 .add(predOps(ARMCC::AL));
10441 } else if (Subtarget->hasV6T2Ops() && isUInt<16>(NumLPads)) {
10442 Register VReg1 = MRI->createVirtualRegister(TRC);
10443 BuildMI(DispatchBB, dl, TII->get(ARM::MOVi16), VReg1)
10444 .addImm(NumLPads & 0xFFFF)
10445 .add(predOps(ARMCC::AL));
10446
10447 unsigned VReg2 = VReg1;
10448 if ((NumLPads & 0xFFFF0000) != 0) {
10449 VReg2 = MRI->createVirtualRegister(TRC);
10450 BuildMI(DispatchBB, dl, TII->get(ARM::MOVTi16), VReg2)
10451 .addReg(VReg1)
10452 .addImm(NumLPads >> 16)
10453 .add(predOps(ARMCC::AL));
10454 }
10455
10456 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
10457 .addReg(NewVReg1)
10458 .addReg(VReg2)
10459 .add(predOps(ARMCC::AL));
10460 } else {
10461 MachineConstantPool *ConstantPool = MF->getConstantPool();
10462 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
10463 const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
10464
10465 // MachineConstantPool wants an explicit alignment.
10466 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10467 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10468
10469 Register VReg1 = MRI->createVirtualRegister(TRC);
10470 BuildMI(DispatchBB, dl, TII->get(ARM::LDRcp))
10471 .addReg(VReg1, RegState::Define)
10472 .addConstantPoolIndex(Idx)
10473 .addImm(0)
10474 .add(predOps(ARMCC::AL));
10475 BuildMI(DispatchBB, dl, TII->get(ARM::CMPrr))
10476 .addReg(NewVReg1)
10477 .addReg(VReg1, RegState::Kill)
10478 .add(predOps(ARMCC::AL));
10479 }
10480
10481 BuildMI(DispatchBB, dl, TII->get(ARM::Bcc))
10482 .addMBB(TrapBB)
10483 .addImm(ARMCC::HI)
10484 .addReg(ARM::CPSR);
10485
10486 Register NewVReg3 = MRI->createVirtualRegister(TRC);
10487 BuildMI(DispContBB, dl, TII->get(ARM::MOVsi), NewVReg3)
10488 .addReg(NewVReg1)
10489 .addImm(ARM_AM::getSORegOpc(ARM_AM::lsl, 2))
10490 .add(predOps(ARMCC::AL))
10491 .add(condCodeOp());
10492 Register NewVReg4 = MRI->createVirtualRegister(TRC);
10493 BuildMI(DispContBB, dl, TII->get(ARM::LEApcrelJT), NewVReg4)
10494 .addJumpTableIndex(MJTI)
10495 .add(predOps(ARMCC::AL));
10496
10497 MachineMemOperand *JTMMOLd =
10498 MF->getMachineMemOperand(MachinePointerInfo::getJumpTable(*MF),
10499 MachineMemOperand::MOLoad, 4, Align(4));
10500 Register NewVReg5 = MRI->createVirtualRegister(TRC);
10501 BuildMI(DispContBB, dl, TII->get(ARM::LDRrs), NewVReg5)
10502 .addReg(NewVReg3, RegState::Kill)
10503 .addReg(NewVReg4)
10504 .addImm(0)
10505 .addMemOperand(JTMMOLd)
10506 .add(predOps(ARMCC::AL));
10507
10508 if (IsPositionIndependent) {
10509 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTadd))
10510 .addReg(NewVReg5, RegState::Kill)
10511 .addReg(NewVReg4)
10512 .addJumpTableIndex(MJTI);
10513 } else {
10514 BuildMI(DispContBB, dl, TII->get(ARM::BR_JTr))
10515 .addReg(NewVReg5, RegState::Kill)
10516 .addJumpTableIndex(MJTI);
10517 }
10518 }
10519
10520 // Add the jump table entries as successors to the MBB.
10521 SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
10522 for (std::vector<MachineBasicBlock*>::iterator
10523 I = LPadList.begin(), E = LPadList.end(); I != E; ++I) {
10524 MachineBasicBlock *CurMBB = *I;
10525 if (SeenMBBs.insert(CurMBB).second)
10526 DispContBB->addSuccessor(CurMBB);
10527 }
10528
10529 // N.B. the order the invoke BBs are processed in doesn't matter here.
10530 const MCPhysReg *SavedRegs = RI.getCalleeSavedRegs(MF);
10531 SmallVector<MachineBasicBlock*, 64> MBBLPads;
10532 for (MachineBasicBlock *BB : InvokeBBs) {
10533
10534 // Remove the landing pad successor from the invoke block and replace it
10535 // with the new dispatch block.
10536 SmallVector<MachineBasicBlock*, 4> Successors(BB->successors());
10537 while (!Successors.empty()) {
10538 MachineBasicBlock *SMBB = Successors.pop_back_val();
10539 if (SMBB->isEHPad()) {
10540 BB->removeSuccessor(SMBB);
10541 MBBLPads.push_back(SMBB);
10542 }
10543 }
10544
10545 BB->addSuccessor(DispatchBB, BranchProbability::getZero());
10546 BB->normalizeSuccProbs();
10547
10548 // Find the invoke call and mark all of the callee-saved registers as
10549 // 'implicit defined' so that they're spilled. This prevents code from
10550 // moving instructions to before the EH block, where they will never be
10551 // executed.
10552 for (MachineBasicBlock::reverse_iterator
10553 II = BB->rbegin(), IE = BB->rend(); II != IE; ++II) {
10554 if (!II->isCall()) continue;
10555
10556 DenseMap<unsigned, bool> DefRegs;
10557 for (MachineInstr::mop_iterator
10558 OI = II->operands_begin(), OE = II->operands_end();
10559 OI != OE; ++OI) {
10560 if (!OI->isReg()) continue;
10561 DefRegs[OI->getReg()] = true;
10562 }
10563
10564 MachineInstrBuilder MIB(*MF, &*II);
10565
10566 for (unsigned i = 0; SavedRegs[i] != 0; ++i) {
10567 unsigned Reg = SavedRegs[i];
10568 if (Subtarget->isThumb2() &&
10569 !ARM::tGPRRegClass.contains(Reg) &&
10570 !ARM::hGPRRegClass.contains(Reg))
10571 continue;
10572 if (Subtarget->isThumb1Only() && !ARM::tGPRRegClass.contains(Reg))
10573 continue;
10574 if (!Subtarget->isThumb() && !ARM::GPRRegClass.contains(Reg))
10575 continue;
10576 if (!DefRegs[Reg])
10577 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
10578 }
10579
10580 break;
10581 }
10582 }
10583
10584 // Mark all former landing pads as non-landing pads. The dispatch is the only
10585 // landing pad now.
10586 for (SmallVectorImpl<MachineBasicBlock*>::iterator
10587 I = MBBLPads.begin(), E = MBBLPads.end(); I != E; ++I)
10588 (*I)->setIsEHPad(false);
10589
10590 // The instruction is gone now.
10591 MI.eraseFromParent();
10592}
10593
10594static
10595MachineBasicBlock *OtherSucc(MachineBasicBlock *MBB, MachineBasicBlock *Succ) {
10596 for (MachineBasicBlock::succ_iterator I = MBB->succ_begin(),
10597 E = MBB->succ_end(); I != E; ++I)
10598 if (*I != Succ)
10599 return *I;
10600 llvm_unreachable("Expecting a BB with two successors!")::llvm::llvm_unreachable_internal("Expecting a BB with two successors!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10600)
;
10601}
10602
10603/// Return the load opcode for a given load size. If load size >= 8,
10604/// neon opcode will be returned.
10605static unsigned getLdOpcode(unsigned LdSize, bool IsThumb1, bool IsThumb2) {
10606 if (LdSize >= 8)
10607 return LdSize == 16 ? ARM::VLD1q32wb_fixed
10608 : LdSize == 8 ? ARM::VLD1d32wb_fixed : 0;
10609 if (IsThumb1)
10610 return LdSize == 4 ? ARM::tLDRi
10611 : LdSize == 2 ? ARM::tLDRHi
10612 : LdSize == 1 ? ARM::tLDRBi : 0;
10613 if (IsThumb2)
10614 return LdSize == 4 ? ARM::t2LDR_POST
10615 : LdSize == 2 ? ARM::t2LDRH_POST
10616 : LdSize == 1 ? ARM::t2LDRB_POST : 0;
10617 return LdSize == 4 ? ARM::LDR_POST_IMM
10618 : LdSize == 2 ? ARM::LDRH_POST
10619 : LdSize == 1 ? ARM::LDRB_POST_IMM : 0;
10620}
10621
10622/// Return the store opcode for a given store size. If store size >= 8,
10623/// neon opcode will be returned.
10624static unsigned getStOpcode(unsigned StSize, bool IsThumb1, bool IsThumb2) {
10625 if (StSize >= 8)
10626 return StSize == 16 ? ARM::VST1q32wb_fixed
10627 : StSize == 8 ? ARM::VST1d32wb_fixed : 0;
10628 if (IsThumb1)
10629 return StSize == 4 ? ARM::tSTRi
10630 : StSize == 2 ? ARM::tSTRHi
10631 : StSize == 1 ? ARM::tSTRBi : 0;
10632 if (IsThumb2)
10633 return StSize == 4 ? ARM::t2STR_POST
10634 : StSize == 2 ? ARM::t2STRH_POST
10635 : StSize == 1 ? ARM::t2STRB_POST : 0;
10636 return StSize == 4 ? ARM::STR_POST_IMM
10637 : StSize == 2 ? ARM::STRH_POST
10638 : StSize == 1 ? ARM::STRB_POST_IMM : 0;
10639}
10640
10641/// Emit a post-increment load operation with given size. The instructions
10642/// will be added to BB at Pos.
10643static void emitPostLd(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
10644 const TargetInstrInfo *TII, const DebugLoc &dl,
10645 unsigned LdSize, unsigned Data, unsigned AddrIn,
10646 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
10647 unsigned LdOpc = getLdOpcode(LdSize, IsThumb1, IsThumb2);
10648 assert(LdOpc != 0 && "Should have a load opcode")(static_cast <bool> (LdOpc != 0 && "Should have a load opcode"
) ? void (0) : __assert_fail ("LdOpc != 0 && \"Should have a load opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10648, __extension__ __PRETTY_FUNCTION__))
;
10649 if (LdSize >= 8) {
10650 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
10651 .addReg(AddrOut, RegState::Define)
10652 .addReg(AddrIn)
10653 .addImm(0)
10654 .add(predOps(ARMCC::AL));
10655 } else if (IsThumb1) {
10656 // load + update AddrIn
10657 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
10658 .addReg(AddrIn)
10659 .addImm(0)
10660 .add(predOps(ARMCC::AL));
10661 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
10662 .add(t1CondCodeOp())
10663 .addReg(AddrIn)
10664 .addImm(LdSize)
10665 .add(predOps(ARMCC::AL));
10666 } else if (IsThumb2) {
10667 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
10668 .addReg(AddrOut, RegState::Define)
10669 .addReg(AddrIn)
10670 .addImm(LdSize)
10671 .add(predOps(ARMCC::AL));
10672 } else { // arm
10673 BuildMI(*BB, Pos, dl, TII->get(LdOpc), Data)
10674 .addReg(AddrOut, RegState::Define)
10675 .addReg(AddrIn)
10676 .addReg(0)
10677 .addImm(LdSize)
10678 .add(predOps(ARMCC::AL));
10679 }
10680}
10681
10682/// Emit a post-increment store operation with given size. The instructions
10683/// will be added to BB at Pos.
10684static void emitPostSt(MachineBasicBlock *BB, MachineBasicBlock::iterator Pos,
10685 const TargetInstrInfo *TII, const DebugLoc &dl,
10686 unsigned StSize, unsigned Data, unsigned AddrIn,
10687 unsigned AddrOut, bool IsThumb1, bool IsThumb2) {
10688 unsigned StOpc = getStOpcode(StSize, IsThumb1, IsThumb2);
10689 assert(StOpc != 0 && "Should have a store opcode")(static_cast <bool> (StOpc != 0 && "Should have a store opcode"
) ? void (0) : __assert_fail ("StOpc != 0 && \"Should have a store opcode\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10689, __extension__ __PRETTY_FUNCTION__))
;
10690 if (StSize >= 8) {
10691 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
10692 .addReg(AddrIn)
10693 .addImm(0)
10694 .addReg(Data)
10695 .add(predOps(ARMCC::AL));
10696 } else if (IsThumb1) {
10697 // store + update AddrIn
10698 BuildMI(*BB, Pos, dl, TII->get(StOpc))
10699 .addReg(Data)
10700 .addReg(AddrIn)
10701 .addImm(0)
10702 .add(predOps(ARMCC::AL));
10703 BuildMI(*BB, Pos, dl, TII->get(ARM::tADDi8), AddrOut)
10704 .add(t1CondCodeOp())
10705 .addReg(AddrIn)
10706 .addImm(StSize)
10707 .add(predOps(ARMCC::AL));
10708 } else if (IsThumb2) {
10709 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
10710 .addReg(Data)
10711 .addReg(AddrIn)
10712 .addImm(StSize)
10713 .add(predOps(ARMCC::AL));
10714 } else { // arm
10715 BuildMI(*BB, Pos, dl, TII->get(StOpc), AddrOut)
10716 .addReg(Data)
10717 .addReg(AddrIn)
10718 .addReg(0)
10719 .addImm(StSize)
10720 .add(predOps(ARMCC::AL));
10721 }
10722}
10723
10724MachineBasicBlock *
10725ARMTargetLowering::EmitStructByval(MachineInstr &MI,
10726 MachineBasicBlock *BB) const {
10727 // This pseudo instruction has 3 operands: dst, src, size
10728 // We expand it to a loop if size > Subtarget->getMaxInlineSizeThreshold().
10729 // Otherwise, we will generate unrolled scalar copies.
10730 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
10731 const BasicBlock *LLVM_BB = BB->getBasicBlock();
10732 MachineFunction::iterator It = ++BB->getIterator();
10733
10734 Register dest = MI.getOperand(0).getReg();
10735 Register src = MI.getOperand(1).getReg();
10736 unsigned SizeVal = MI.getOperand(2).getImm();
10737 unsigned Alignment = MI.getOperand(3).getImm();
10738 DebugLoc dl = MI.getDebugLoc();
10739
10740 MachineFunction *MF = BB->getParent();
10741 MachineRegisterInfo &MRI = MF->getRegInfo();
10742 unsigned UnitSize = 0;
10743 const TargetRegisterClass *TRC = nullptr;
10744 const TargetRegisterClass *VecTRC = nullptr;
10745
10746 bool IsThumb1 = Subtarget->isThumb1Only();
10747 bool IsThumb2 = Subtarget->isThumb2();
10748 bool IsThumb = Subtarget->isThumb();
10749
10750 if (Alignment & 1) {
10751 UnitSize = 1;
10752 } else if (Alignment & 2) {
10753 UnitSize = 2;
10754 } else {
10755 // Check whether we can use NEON instructions.
10756 if (!MF->getFunction().hasFnAttribute(Attribute::NoImplicitFloat) &&
10757 Subtarget->hasNEON()) {
10758 if ((Alignment % 16 == 0) && SizeVal >= 16)
10759 UnitSize = 16;
10760 else if ((Alignment % 8 == 0) && SizeVal >= 8)
10761 UnitSize = 8;
10762 }
10763 // Can't use NEON instructions.
10764 if (UnitSize == 0)
10765 UnitSize = 4;
10766 }
10767
10768 // Select the correct opcode and register class for unit size load/store
10769 bool IsNeon = UnitSize >= 8;
10770 TRC = IsThumb ? &ARM::tGPRRegClass : &ARM::GPRRegClass;
10771 if (IsNeon)
10772 VecTRC = UnitSize == 16 ? &ARM::DPairRegClass
10773 : UnitSize == 8 ? &ARM::DPRRegClass
10774 : nullptr;
10775
10776 unsigned BytesLeft = SizeVal % UnitSize;
10777 unsigned LoopSize = SizeVal - BytesLeft;
10778
10779 if (SizeVal <= Subtarget->getMaxInlineSizeThreshold()) {
10780 // Use LDR and STR to copy.
10781 // [scratch, srcOut] = LDR_POST(srcIn, UnitSize)
10782 // [destOut] = STR_POST(scratch, destIn, UnitSize)
10783 unsigned srcIn = src;
10784 unsigned destIn = dest;
10785 for (unsigned i = 0; i < LoopSize; i+=UnitSize) {
10786 Register srcOut = MRI.createVirtualRegister(TRC);
10787 Register destOut = MRI.createVirtualRegister(TRC);
10788 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
10789 emitPostLd(BB, MI, TII, dl, UnitSize, scratch, srcIn, srcOut,
10790 IsThumb1, IsThumb2);
10791 emitPostSt(BB, MI, TII, dl, UnitSize, scratch, destIn, destOut,
10792 IsThumb1, IsThumb2);
10793 srcIn = srcOut;
10794 destIn = destOut;
10795 }
10796
10797 // Handle the leftover bytes with LDRB and STRB.
10798 // [scratch, srcOut] = LDRB_POST(srcIn, 1)
10799 // [destOut] = STRB_POST(scratch, destIn, 1)
10800 for (unsigned i = 0; i < BytesLeft; i++) {
10801 Register srcOut = MRI.createVirtualRegister(TRC);
10802 Register destOut = MRI.createVirtualRegister(TRC);
10803 Register scratch = MRI.createVirtualRegister(TRC);
10804 emitPostLd(BB, MI, TII, dl, 1, scratch, srcIn, srcOut,
10805 IsThumb1, IsThumb2);
10806 emitPostSt(BB, MI, TII, dl, 1, scratch, destIn, destOut,
10807 IsThumb1, IsThumb2);
10808 srcIn = srcOut;
10809 destIn = destOut;
10810 }
10811 MI.eraseFromParent(); // The instruction is gone now.
10812 return BB;
10813 }
10814
10815 // Expand the pseudo op to a loop.
10816 // thisMBB:
10817 // ...
10818 // movw varEnd, # --> with thumb2
10819 // movt varEnd, #
10820 // ldrcp varEnd, idx --> without thumb2
10821 // fallthrough --> loopMBB
10822 // loopMBB:
10823 // PHI varPhi, varEnd, varLoop
10824 // PHI srcPhi, src, srcLoop
10825 // PHI destPhi, dst, destLoop
10826 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
10827 // [destLoop] = STR_POST(scratch, destPhi, UnitSize)
10828 // subs varLoop, varPhi, #UnitSize
10829 // bne loopMBB
10830 // fallthrough --> exitMBB
10831 // exitMBB:
10832 // epilogue to handle left-over bytes
10833 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
10834 // [destOut] = STRB_POST(scratch, destLoop, 1)
10835 MachineBasicBlock *loopMBB = MF->CreateMachineBasicBlock(LLVM_BB);
10836 MachineBasicBlock *exitMBB = MF->CreateMachineBasicBlock(LLVM_BB);
10837 MF->insert(It, loopMBB);
10838 MF->insert(It, exitMBB);
10839
10840 // Transfer the remainder of BB and its successor edges to exitMBB.
10841 exitMBB->splice(exitMBB->begin(), BB,
10842 std::next(MachineBasicBlock::iterator(MI)), BB->end());
10843 exitMBB->transferSuccessorsAndUpdatePHIs(BB);
10844
10845 // Load an immediate to varEnd.
10846 Register varEnd = MRI.createVirtualRegister(TRC);
10847 if (Subtarget->useMovt()) {
10848 unsigned Vtmp = varEnd;
10849 if ((LoopSize & 0xFFFF0000) != 0)
10850 Vtmp = MRI.createVirtualRegister(TRC);
10851 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVi16 : ARM::MOVi16), Vtmp)
10852 .addImm(LoopSize & 0xFFFF)
10853 .add(predOps(ARMCC::AL));
10854
10855 if ((LoopSize & 0xFFFF0000) != 0)
10856 BuildMI(BB, dl, TII->get(IsThumb ? ARM::t2MOVTi16 : ARM::MOVTi16), varEnd)
10857 .addReg(Vtmp)
10858 .addImm(LoopSize >> 16)
10859 .add(predOps(ARMCC::AL));
10860 } else {
10861 MachineConstantPool *ConstantPool = MF->getConstantPool();
10862 Type *Int32Ty = Type::getInt32Ty(MF->getFunction().getContext());
10863 const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
10864
10865 // MachineConstantPool wants an explicit alignment.
10866 Align Alignment = MF->getDataLayout().getPrefTypeAlign(Int32Ty);
10867 unsigned Idx = ConstantPool->getConstantPoolIndex(C, Alignment);
10868 MachineMemOperand *CPMMO =
10869 MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF),
10870 MachineMemOperand::MOLoad, 4, Align(4));
10871
10872 if (IsThumb)
10873 BuildMI(*BB, MI, dl, TII->get(ARM::tLDRpci))
10874 .addReg(varEnd, RegState::Define)
10875 .addConstantPoolIndex(Idx)
10876 .add(predOps(ARMCC::AL))
10877 .addMemOperand(CPMMO);
10878 else
10879 BuildMI(*BB, MI, dl, TII->get(ARM::LDRcp))
10880 .addReg(varEnd, RegState::Define)
10881 .addConstantPoolIndex(Idx)
10882 .addImm(0)
10883 .add(predOps(ARMCC::AL))
10884 .addMemOperand(CPMMO);
10885 }
10886 BB->addSuccessor(loopMBB);
10887
10888 // Generate the loop body:
10889 // varPhi = PHI(varLoop, varEnd)
10890 // srcPhi = PHI(srcLoop, src)
10891 // destPhi = PHI(destLoop, dst)
10892 MachineBasicBlock *entryBB = BB;
10893 BB = loopMBB;
10894 Register varLoop = MRI.createVirtualRegister(TRC);
10895 Register varPhi = MRI.createVirtualRegister(TRC);
10896 Register srcLoop = MRI.createVirtualRegister(TRC);
10897 Register srcPhi = MRI.createVirtualRegister(TRC);
10898 Register destLoop = MRI.createVirtualRegister(TRC);
10899 Register destPhi = MRI.createVirtualRegister(TRC);
10900
10901 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), varPhi)
10902 .addReg(varLoop).addMBB(loopMBB)
10903 .addReg(varEnd).addMBB(entryBB);
10904 BuildMI(BB, dl, TII->get(ARM::PHI), srcPhi)
10905 .addReg(srcLoop).addMBB(loopMBB)
10906 .addReg(src).addMBB(entryBB);
10907 BuildMI(BB, dl, TII->get(ARM::PHI), destPhi)
10908 .addReg(destLoop).addMBB(loopMBB)
10909 .addReg(dest).addMBB(entryBB);
10910
10911 // [scratch, srcLoop] = LDR_POST(srcPhi, UnitSize)
10912 // [destLoop] = STR_POST(scratch, destPhi, UnitSiz)
10913 Register scratch = MRI.createVirtualRegister(IsNeon ? VecTRC : TRC);
10914 emitPostLd(BB, BB->end(), TII, dl, UnitSize, scratch, srcPhi, srcLoop,
10915 IsThumb1, IsThumb2);
10916 emitPostSt(BB, BB->end(), TII, dl, UnitSize, scratch, destPhi, destLoop,
10917 IsThumb1, IsThumb2);
10918
10919 // Decrement loop variable by UnitSize.
10920 if (IsThumb1) {
10921 BuildMI(*BB, BB->end(), dl, TII->get(ARM::tSUBi8), varLoop)
10922 .add(t1CondCodeOp())
10923 .addReg(varPhi)
10924 .addImm(UnitSize)
10925 .add(predOps(ARMCC::AL));
10926 } else {
10927 MachineInstrBuilder MIB =
10928 BuildMI(*BB, BB->end(), dl,
10929 TII->get(IsThumb2 ? ARM::t2SUBri : ARM::SUBri), varLoop);
10930 MIB.addReg(varPhi)
10931 .addImm(UnitSize)
10932 .add(predOps(ARMCC::AL))
10933 .add(condCodeOp());
10934 MIB->getOperand(5).setReg(ARM::CPSR);
10935 MIB->getOperand(5).setIsDef(true);
10936 }
10937 BuildMI(*BB, BB->end(), dl,
10938 TII->get(IsThumb1 ? ARM::tBcc : IsThumb2 ? ARM::t2Bcc : ARM::Bcc))
10939 .addMBB(loopMBB).addImm(ARMCC::NE).addReg(ARM::CPSR);
10940
10941 // loopMBB can loop back to loopMBB or fall through to exitMBB.
10942 BB->addSuccessor(loopMBB);
10943 BB->addSuccessor(exitMBB);
10944
10945 // Add epilogue to handle BytesLeft.
10946 BB = exitMBB;
10947 auto StartOfExit = exitMBB->begin();
10948
10949 // [scratch, srcOut] = LDRB_POST(srcLoop, 1)
10950 // [destOut] = STRB_POST(scratch, destLoop, 1)
10951 unsigned srcIn = srcLoop;
10952 unsigned destIn = destLoop;
10953 for (unsigned i = 0; i < BytesLeft; i++) {
10954 Register srcOut = MRI.createVirtualRegister(TRC);
10955 Register destOut = MRI.createVirtualRegister(TRC);
10956 Register scratch = MRI.createVirtualRegister(TRC);
10957 emitPostLd(BB, StartOfExit, TII, dl, 1, scratch, srcIn, srcOut,
10958 IsThumb1, IsThumb2);
10959 emitPostSt(BB, StartOfExit, TII, dl, 1, scratch, destIn, destOut,
10960 IsThumb1, IsThumb2);
10961 srcIn = srcOut;
10962 destIn = destOut;
10963 }
10964
10965 MI.eraseFromParent(); // The instruction is gone now.
10966 return BB;
10967}
10968
10969MachineBasicBlock *
10970ARMTargetLowering::EmitLowered__chkstk(MachineInstr &MI,
10971 MachineBasicBlock *MBB) const {
10972 const TargetMachine &TM = getTargetMachine();
10973 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
10974 DebugLoc DL = MI.getDebugLoc();
10975
10976 assert(Subtarget->isTargetWindows() &&(static_cast <bool> (Subtarget->isTargetWindows() &&
"__chkstk is only supported on Windows") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"__chkstk is only supported on Windows\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10977, __extension__ __PRETTY_FUNCTION__))
10977 "__chkstk is only supported on Windows")(static_cast <bool> (Subtarget->isTargetWindows() &&
"__chkstk is only supported on Windows") ? void (0) : __assert_fail
("Subtarget->isTargetWindows() && \"__chkstk is only supported on Windows\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10977, __extension__ __PRETTY_FUNCTION__))
;
10978 assert(Subtarget->isThumb2() && "Windows on ARM requires Thumb-2 mode")(static_cast <bool> (Subtarget->isThumb2() &&
"Windows on ARM requires Thumb-2 mode") ? void (0) : __assert_fail
("Subtarget->isThumb2() && \"Windows on ARM requires Thumb-2 mode\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 10978, __extension__ __PRETTY_FUNCTION__))
;
10979
10980 // __chkstk takes the number of words to allocate on the stack in R4, and
10981 // returns the stack adjustment in number of bytes in R4. This will not
10982 // clober any other registers (other than the obvious lr).
10983 //
10984 // Although, technically, IP should be considered a register which may be
10985 // clobbered, the call itself will not touch it. Windows on ARM is a pure
10986 // thumb-2 environment, so there is no interworking required. As a result, we
10987 // do not expect a veneer to be emitted by the linker, clobbering IP.
10988 //
10989 // Each module receives its own copy of __chkstk, so no import thunk is
10990 // required, again, ensuring that IP is not clobbered.
10991 //
10992 // Finally, although some linkers may theoretically provide a trampoline for
10993 // out of range calls (which is quite common due to a 32M range limitation of
10994 // branches for Thumb), we can generate the long-call version via
10995 // -mcmodel=large, alleviating the need for the trampoline which may clobber
10996 // IP.
10997
10998 switch (TM.getCodeModel()) {
10999 case CodeModel::Tiny:
11000 llvm_unreachable("Tiny code model not available on ARM.")::llvm::llvm_unreachable_internal("Tiny code model not available on ARM."
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11000)
;
11001 case CodeModel::Small:
11002 case CodeModel::Medium:
11003 case CodeModel::Kernel:
11004 BuildMI(*MBB, MI, DL, TII.get(ARM::tBL))
11005 .add(predOps(ARMCC::AL))
11006 .addExternalSymbol("__chkstk")
11007 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
11008 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
11009 .addReg(ARM::R12,
11010 RegState::Implicit | RegState::Define | RegState::Dead)
11011 .addReg(ARM::CPSR,
11012 RegState::Implicit | RegState::Define | RegState::Dead);
11013 break;
11014 case CodeModel::Large: {
11015 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
11016 Register Reg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11017
11018 BuildMI(*MBB, MI, DL, TII.get(ARM::t2MOVi32imm), Reg)
11019 .addExternalSymbol("__chkstk");
11020 BuildMI(*MBB, MI, DL, TII.get(gettBLXrOpcode(*MBB->getParent())))
11021 .add(predOps(ARMCC::AL))
11022 .addReg(Reg, RegState::Kill)
11023 .addReg(ARM::R4, RegState::Implicit | RegState::Kill)
11024 .addReg(ARM::R4, RegState::Implicit | RegState::Define)
11025 .addReg(ARM::R12,
11026 RegState::Implicit | RegState::Define | RegState::Dead)
11027 .addReg(ARM::CPSR,
11028 RegState::Implicit | RegState::Define | RegState::Dead);
11029 break;
11030 }
11031 }
11032
11033 BuildMI(*MBB, MI, DL, TII.get(ARM::t2SUBrr), ARM::SP)
11034 .addReg(ARM::SP, RegState::Kill)
11035 .addReg(ARM::R4, RegState::Kill)
11036 .setMIFlags(MachineInstr::FrameSetup)
11037 .add(predOps(ARMCC::AL))
11038 .add(condCodeOp());
11039
11040 MI.eraseFromParent();
11041 return MBB;
11042}
11043
11044MachineBasicBlock *
11045ARMTargetLowering::EmitLowered__dbzchk(MachineInstr &MI,
11046 MachineBasicBlock *MBB) const {
11047 DebugLoc DL = MI.getDebugLoc();
11048 MachineFunction *MF = MBB->getParent();
11049 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11050
11051 MachineBasicBlock *ContBB = MF->CreateMachineBasicBlock();
11052 MF->insert(++MBB->getIterator(), ContBB);
11053 ContBB->splice(ContBB->begin(), MBB,
11054 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
11055 ContBB->transferSuccessorsAndUpdatePHIs(MBB);
11056 MBB->addSuccessor(ContBB);
11057
11058 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
11059 BuildMI(TrapBB, DL, TII->get(ARM::t__brkdiv0));
11060 MF->push_back(TrapBB);
11061 MBB->addSuccessor(TrapBB);
11062
11063 BuildMI(*MBB, MI, DL, TII->get(ARM::tCMPi8))
11064 .addReg(MI.getOperand(0).getReg())
11065 .addImm(0)
11066 .add(predOps(ARMCC::AL));
11067 BuildMI(*MBB, MI, DL, TII->get(ARM::t2Bcc))
11068 .addMBB(TrapBB)
11069 .addImm(ARMCC::EQ)
11070 .addReg(ARM::CPSR);
11071
11072 MI.eraseFromParent();
11073 return ContBB;
11074}
11075
11076// The CPSR operand of SelectItr might be missing a kill marker
11077// because there were multiple uses of CPSR, and ISel didn't know
11078// which to mark. Figure out whether SelectItr should have had a
11079// kill marker, and set it if it should. Returns the correct kill
11080// marker value.
11081static bool checkAndUpdateCPSRKill(MachineBasicBlock::iterator SelectItr,
11082 MachineBasicBlock* BB,
11083 const TargetRegisterInfo* TRI) {
11084 // Scan forward through BB for a use/def of CPSR.
11085 MachineBasicBlock::iterator miI(std::next(SelectItr));
11086 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
11087 const MachineInstr& mi = *miI;
11088 if (mi.readsRegister(ARM::CPSR))
11089 return false;
11090 if (mi.definesRegister(ARM::CPSR))
11091 break; // Should have kill-flag - update below.
11092 }
11093
11094 // If we hit the end of the block, check whether CPSR is live into a
11095 // successor.
11096 if (miI == BB->end()) {
11097 for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
11098 sEnd = BB->succ_end();
11099 sItr != sEnd; ++sItr) {
11100 MachineBasicBlock* succ = *sItr;
11101 if (succ->isLiveIn(ARM::CPSR))
11102 return false;
11103 }
11104 }
11105
11106 // We found a def, or hit the end of the basic block and CPSR wasn't live
11107 // out. SelectMI should have a kill flag on CPSR.
11108 SelectItr->addRegisterKilled(ARM::CPSR, TRI);
11109 return true;
11110}
11111
11112/// Adds logic in loop entry MBB to calculate loop iteration count and adds
11113/// t2WhileLoopSetup and t2WhileLoopStart to generate WLS loop
11114static Register genTPEntry(MachineBasicBlock *TpEntry,
11115 MachineBasicBlock *TpLoopBody,
11116 MachineBasicBlock *TpExit, Register OpSizeReg,
11117 const TargetInstrInfo *TII, DebugLoc Dl,
11118 MachineRegisterInfo &MRI) {
11119 // Calculates loop iteration count = ceil(n/16) = (n + 15) >> 4.
11120 Register AddDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11121 BuildMI(TpEntry, Dl, TII->get(ARM::t2ADDri), AddDestReg)
11122 .addUse(OpSizeReg)
11123 .addImm(15)
11124 .add(predOps(ARMCC::AL))
11125 .addReg(0);
11126
11127 Register LsrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11128 BuildMI(TpEntry, Dl, TII->get(ARM::t2LSRri), LsrDestReg)
11129 .addUse(AddDestReg, RegState::Kill)
11130 .addImm(4)
11131 .add(predOps(ARMCC::AL))
11132 .addReg(0);
11133
11134 Register TotalIterationsReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11135 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopSetup), TotalIterationsReg)
11136 .addUse(LsrDestReg, RegState::Kill);
11137
11138 BuildMI(TpEntry, Dl, TII->get(ARM::t2WhileLoopStart))
11139 .addUse(TotalIterationsReg)
11140 .addMBB(TpExit);
11141
11142 BuildMI(TpEntry, Dl, TII->get(ARM::t2B))
11143 .addMBB(TpLoopBody)
11144 .add(predOps(ARMCC::AL));
11145
11146 return TotalIterationsReg;
11147}
11148
11149/// Adds logic in the loopBody MBB to generate MVE_VCTP, t2DoLoopDec and
11150/// t2DoLoopEnd. These are used by later passes to generate tail predicated
11151/// loops.
11152static void genTPLoopBody(MachineBasicBlock *TpLoopBody,
11153 MachineBasicBlock *TpEntry, MachineBasicBlock *TpExit,
11154 const TargetInstrInfo *TII, DebugLoc Dl,
11155 MachineRegisterInfo &MRI, Register OpSrcReg,
11156 Register OpDestReg, Register ElementCountReg,
11157 Register TotalIterationsReg, bool IsMemcpy) {
11158 // First insert 4 PHI nodes for: Current pointer to Src (if memcpy), Dest
11159 // array, loop iteration counter, predication counter.
11160
11161 Register SrcPhiReg, CurrSrcReg;
11162 if (IsMemcpy) {
11163 // Current position in the src array
11164 SrcPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11165 CurrSrcReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11166 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), SrcPhiReg)
11167 .addUse(OpSrcReg)
11168 .addMBB(TpEntry)
11169 .addUse(CurrSrcReg)
11170 .addMBB(TpLoopBody);
11171 }
11172
11173 // Current position in the dest array
11174 Register DestPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11175 Register CurrDestReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11176 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), DestPhiReg)
11177 .addUse(OpDestReg)
11178 .addMBB(TpEntry)
11179 .addUse(CurrDestReg)
11180 .addMBB(TpLoopBody);
11181
11182 // Current loop counter
11183 Register LoopCounterPhiReg = MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11184 Register RemainingLoopIterationsReg =
11185 MRI.createVirtualRegister(&ARM::GPRlrRegClass);
11186 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), LoopCounterPhiReg)
11187 .addUse(TotalIterationsReg)
11188 .addMBB(TpEntry)
11189 .addUse(RemainingLoopIterationsReg)
11190 .addMBB(TpLoopBody);
11191
11192 // Predication counter
11193 Register PredCounterPhiReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11194 Register RemainingElementsReg = MRI.createVirtualRegister(&ARM::rGPRRegClass);
11195 BuildMI(TpLoopBody, Dl, TII->get(ARM::PHI), PredCounterPhiReg)
11196 .addUse(ElementCountReg)
11197 .addMBB(TpEntry)
11198 .addUse(RemainingElementsReg)
11199 .addMBB(TpLoopBody);
11200
11201 // Pass predication counter to VCTP
11202 Register VccrReg = MRI.createVirtualRegister(&ARM::VCCRRegClass);
11203 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VCTP8), VccrReg)
11204 .addUse(PredCounterPhiReg)
11205 .addImm(ARMVCC::None)
11206 .addReg(0);
11207
11208 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2SUBri), RemainingElementsReg)
11209 .addUse(PredCounterPhiReg)
11210 .addImm(16)
11211 .add(predOps(ARMCC::AL))
11212 .addReg(0);
11213
11214 // VLDRB (only if memcpy) and VSTRB instructions, predicated using VPR
11215 Register SrcValueReg;
11216 if (IsMemcpy) {
11217 SrcValueReg = MRI.createVirtualRegister(&ARM::MQPRRegClass);
11218 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VLDRBU8_post))
11219 .addDef(CurrSrcReg)
11220 .addDef(SrcValueReg)
11221 .addReg(SrcPhiReg)
11222 .addImm(16)
11223 .addImm(ARMVCC::Then)
11224 .addUse(VccrReg);
11225 } else
11226 SrcValueReg = OpSrcReg;
11227
11228 BuildMI(TpLoopBody, Dl, TII->get(ARM::MVE_VSTRBU8_post))
11229 .addDef(CurrDestReg)
11230 .addUse(SrcValueReg)
11231 .addReg(DestPhiReg)
11232 .addImm(16)
11233 .addImm(ARMVCC::Then)
11234 .addUse(VccrReg);
11235
11236 // Add the pseudoInstrs for decrementing the loop counter and marking the
11237 // end:t2DoLoopDec and t2DoLoopEnd
11238 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopDec), RemainingLoopIterationsReg)
11239 .addUse(LoopCounterPhiReg)
11240 .addImm(1);
11241
11242 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2LoopEnd))
11243 .addUse(RemainingLoopIterationsReg)
11244 .addMBB(TpLoopBody);
11245
11246 BuildMI(TpLoopBody, Dl, TII->get(ARM::t2B))
11247 .addMBB(TpExit)
11248 .add(predOps(ARMCC::AL));
11249}
11250
11251MachineBasicBlock *
11252ARMTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
11253 MachineBasicBlock *BB) const {
11254 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
11255 DebugLoc dl = MI.getDebugLoc();
11256 bool isThumb2 = Subtarget->isThumb2();
11257 switch (MI.getOpcode()) {
11258 default: {
11259 MI.print(errs());
11260 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11260)
;
11261 }
11262
11263 // Thumb1 post-indexed loads are really just single-register LDMs.
11264 case ARM::tLDR_postidx: {
11265 MachineOperand Def(MI.getOperand(1));
11266 BuildMI(*BB, MI, dl, TII->get(ARM::tLDMIA_UPD))
11267 .add(Def) // Rn_wb
11268 .add(MI.getOperand(2)) // Rn
11269 .add(MI.getOperand(3)) // PredImm
11270 .add(MI.getOperand(4)) // PredReg
11271 .add(MI.getOperand(0)) // Rt
11272 .cloneMemRefs(MI);
11273 MI.eraseFromParent();
11274 return BB;
11275 }
11276
11277 case ARM::MVE_MEMCPYLOOPINST:
11278 case ARM::MVE_MEMSETLOOPINST: {
11279
11280 // Transformation below expands MVE_MEMCPYLOOPINST/MVE_MEMSETLOOPINST Pseudo
11281 // into a Tail Predicated (TP) Loop. It adds the instructions to calculate
11282 // the iteration count =ceil(size_in_bytes/16)) in the TP entry block and
11283 // adds the relevant instructions in the TP loop Body for generation of a
11284 // WLSTP loop.
11285
11286 // Below is relevant portion of the CFG after the transformation.
11287 // The Machine Basic Blocks are shown along with branch conditions (in
11288 // brackets). Note that TP entry/exit MBBs depict the entry/exit of this
11289 // portion of the CFG and may not necessarily be the entry/exit of the
11290 // function.
11291
11292 // (Relevant) CFG after transformation:
11293 // TP entry MBB
11294 // |
11295 // |-----------------|
11296 // (n <= 0) (n > 0)
11297 // | |
11298 // | TP loop Body MBB<--|
11299 // | | |
11300 // \ |___________|
11301 // \ /
11302 // TP exit MBB
11303
11304 MachineFunction *MF = BB->getParent();
11305 MachineFunctionProperties &Properties = MF->getProperties();
11306 MachineRegisterInfo &MRI = MF->getRegInfo();
11307
11308 Register OpDestReg = MI.getOperand(0).getReg();
11309 Register OpSrcReg = MI.getOperand(1).getReg();
11310 Register OpSizeReg = MI.getOperand(2).getReg();
11311
11312 // Allocate the required MBBs and add to parent function.
11313 MachineBasicBlock *TpEntry = BB;
11314 MachineBasicBlock *TpLoopBody = MF->CreateMachineBasicBlock();
11315 MachineBasicBlock *TpExit;
11316
11317 MF->push_back(TpLoopBody);
11318
11319 // If any instructions are present in the current block after
11320 // MVE_MEMCPYLOOPINST or MVE_MEMSETLOOPINST, split the current block and
11321 // move the instructions into the newly created exit block. If there are no
11322 // instructions add an explicit branch to the FallThrough block and then
11323 // split.
11324 //
11325 // The split is required for two reasons:
11326 // 1) A terminator(t2WhileLoopStart) will be placed at that site.
11327 // 2) Since a TPLoopBody will be added later, any phis in successive blocks
11328 // need to be updated. splitAt() already handles this.
11329 TpExit = BB->splitAt(MI, false);
11330 if (TpExit == BB) {
11331 assert(BB->canFallThrough() && "Exit Block must be Fallthrough of the "(static_cast <bool> (BB->canFallThrough() &&
"Exit Block must be Fallthrough of the " "block containing memcpy/memset Pseudo"
) ? void (0) : __assert_fail ("BB->canFallThrough() && \"Exit Block must be Fallthrough of the \" \"block containing memcpy/memset Pseudo\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11332, __extension__ __PRETTY_FUNCTION__))
11332 "block containing memcpy/memset Pseudo")(static_cast <bool> (BB->canFallThrough() &&
"Exit Block must be Fallthrough of the " "block containing memcpy/memset Pseudo"
) ? void (0) : __assert_fail ("BB->canFallThrough() && \"Exit Block must be Fallthrough of the \" \"block containing memcpy/memset Pseudo\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11332, __extension__ __PRETTY_FUNCTION__))
;
11333 TpExit = BB->getFallThrough();
11334 BuildMI(BB, dl, TII->get(ARM::t2B))
11335 .addMBB(TpExit)
11336 .add(predOps(ARMCC::AL));
11337 TpExit = BB->splitAt(MI, false);
11338 }
11339
11340 // Add logic for iteration count
11341 Register TotalIterationsReg =
11342 genTPEntry(TpEntry, TpLoopBody, TpExit, OpSizeReg, TII, dl, MRI);
11343
11344 // Add the vectorized (and predicated) loads/store instructions
11345 bool IsMemcpy = MI.getOpcode() == ARM::MVE_MEMCPYLOOPINST;
11346 genTPLoopBody(TpLoopBody, TpEntry, TpExit, TII, dl, MRI, OpSrcReg,
11347 OpDestReg, OpSizeReg, TotalIterationsReg, IsMemcpy);
11348
11349 // Required to avoid conflict with the MachineVerifier during testing.
11350 Properties.reset(MachineFunctionProperties::Property::NoPHIs);
11351
11352 // Connect the blocks
11353 TpEntry->addSuccessor(TpLoopBody);
11354 TpLoopBody->addSuccessor(TpLoopBody);
11355 TpLoopBody->addSuccessor(TpExit);
11356
11357 // Reorder for a more natural layout
11358 TpLoopBody->moveAfter(TpEntry);
11359 TpExit->moveAfter(TpLoopBody);
11360
11361 // Finally, remove the memcpy Psuedo Instruction
11362 MI.eraseFromParent();
11363
11364 // Return the exit block as it may contain other instructions requiring a
11365 // custom inserter
11366 return TpExit;
11367 }
11368
11369 // The Thumb2 pre-indexed stores have the same MI operands, they just
11370 // define them differently in the .td files from the isel patterns, so
11371 // they need pseudos.
11372 case ARM::t2STR_preidx:
11373 MI.setDesc(TII->get(ARM::t2STR_PRE));
11374 return BB;
11375 case ARM::t2STRB_preidx:
11376 MI.setDesc(TII->get(ARM::t2STRB_PRE));
11377 return BB;
11378 case ARM::t2STRH_preidx:
11379 MI.setDesc(TII->get(ARM::t2STRH_PRE));
11380 return BB;
11381
11382 case ARM::STRi_preidx:
11383 case ARM::STRBi_preidx: {
11384 unsigned NewOpc = MI.getOpcode() == ARM::STRi_preidx ? ARM::STR_PRE_IMM
11385 : ARM::STRB_PRE_IMM;
11386 // Decode the offset.
11387 unsigned Offset = MI.getOperand(4).getImm();
11388 bool isSub = ARM_AM::getAM2Op(Offset) == ARM_AM::sub;
11389 Offset = ARM_AM::getAM2Offset(Offset);
11390 if (isSub)
11391 Offset = -Offset;
11392
11393 MachineMemOperand *MMO = *MI.memoperands_begin();
11394 BuildMI(*BB, MI, dl, TII->get(NewOpc))
11395 .add(MI.getOperand(0)) // Rn_wb
11396 .add(MI.getOperand(1)) // Rt
11397 .add(MI.getOperand(2)) // Rn
11398 .addImm(Offset) // offset (skip GPR==zero_reg)
11399 .add(MI.getOperand(5)) // pred
11400 .add(MI.getOperand(6))
11401 .addMemOperand(MMO);
11402 MI.eraseFromParent();
11403 return BB;
11404 }
11405 case ARM::STRr_preidx:
11406 case ARM::STRBr_preidx:
11407 case ARM::STRH_preidx: {
11408 unsigned NewOpc;
11409 switch (MI.getOpcode()) {
11410 default: llvm_unreachable("unexpected opcode!")::llvm::llvm_unreachable_internal("unexpected opcode!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11410)
;
11411 case ARM::STRr_preidx: NewOpc = ARM::STR_PRE_REG; break;
11412 case ARM::STRBr_preidx: NewOpc = ARM::STRB_PRE_REG; break;
11413 case ARM::STRH_preidx: NewOpc = ARM::STRH_PRE; break;
11414 }
11415 MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(NewOpc));
11416 for (unsigned i = 0; i < MI.getNumOperands(); ++i)
11417 MIB.add(MI.getOperand(i));
11418 MI.eraseFromParent();
11419 return BB;
11420 }
11421
11422 case ARM::tMOVCCr_pseudo: {
11423 // To "insert" a SELECT_CC instruction, we actually have to insert the
11424 // diamond control-flow pattern. The incoming instruction knows the
11425 // destination vreg to set, the condition code register to branch on, the
11426 // true/false values to select between, and a branch opcode to use.
11427 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11428 MachineFunction::iterator It = ++BB->getIterator();
11429
11430 // thisMBB:
11431 // ...
11432 // TrueVal = ...
11433 // cmpTY ccX, r1, r2
11434 // bCC copy1MBB
11435 // fallthrough --> copy0MBB
11436 MachineBasicBlock *thisMBB = BB;
11437 MachineFunction *F = BB->getParent();
11438 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
11439 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
11440 F->insert(It, copy0MBB);
11441 F->insert(It, sinkMBB);
11442
11443 // Check whether CPSR is live past the tMOVCCr_pseudo.
11444 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
11445 if (!MI.killsRegister(ARM::CPSR) &&
11446 !checkAndUpdateCPSRKill(MI, thisMBB, TRI)) {
11447 copy0MBB->addLiveIn(ARM::CPSR);
11448 sinkMBB->addLiveIn(ARM::CPSR);
11449 }
11450
11451 // Transfer the remainder of BB and its successor edges to sinkMBB.
11452 sinkMBB->splice(sinkMBB->begin(), BB,
11453 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11454 sinkMBB->transferSuccessorsAndUpdatePHIs(BB);
11455
11456 BB->addSuccessor(copy0MBB);
11457 BB->addSuccessor(sinkMBB);
11458
11459 BuildMI(BB, dl, TII->get(ARM::tBcc))
11460 .addMBB(sinkMBB)
11461 .addImm(MI.getOperand(3).getImm())
11462 .addReg(MI.getOperand(4).getReg());
11463
11464 // copy0MBB:
11465 // %FalseValue = ...
11466 // # fallthrough to sinkMBB
11467 BB = copy0MBB;
11468
11469 // Update machine-CFG edges
11470 BB->addSuccessor(sinkMBB);
11471
11472 // sinkMBB:
11473 // %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
11474 // ...
11475 BB = sinkMBB;
11476 BuildMI(*BB, BB->begin(), dl, TII->get(ARM::PHI), MI.getOperand(0).getReg())
11477 .addReg(MI.getOperand(1).getReg())
11478 .addMBB(copy0MBB)
11479 .addReg(MI.getOperand(2).getReg())
11480 .addMBB(thisMBB);
11481
11482 MI.eraseFromParent(); // The pseudo instruction is gone now.
11483 return BB;
11484 }
11485
11486 case ARM::BCCi64:
11487 case ARM::BCCZi64: {
11488 // If there is an unconditional branch to the other successor, remove it.
11489 BB->erase(std::next(MachineBasicBlock::iterator(MI)), BB->end());
11490
11491 // Compare both parts that make up the double comparison separately for
11492 // equality.
11493 bool RHSisZero = MI.getOpcode() == ARM::BCCZi64;
11494
11495 Register LHS1 = MI.getOperand(1).getReg();
11496 Register LHS2 = MI.getOperand(2).getReg();
11497 if (RHSisZero) {
11498 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
11499 .addReg(LHS1)
11500 .addImm(0)
11501 .add(predOps(ARMCC::AL));
11502 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
11503 .addReg(LHS2).addImm(0)
11504 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
11505 } else {
11506 Register RHS1 = MI.getOperand(3).getReg();
11507 Register RHS2 = MI.getOperand(4).getReg();
11508 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
11509 .addReg(LHS1)
11510 .addReg(RHS1)
11511 .add(predOps(ARMCC::AL));
11512 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
11513 .addReg(LHS2).addReg(RHS2)
11514 .addImm(ARMCC::EQ).addReg(ARM::CPSR);
11515 }
11516
11517 MachineBasicBlock *destMBB = MI.getOperand(RHSisZero ? 3 : 5).getMBB();
11518 MachineBasicBlock *exitMBB = OtherSucc(BB, destMBB);
11519 if (MI.getOperand(0).getImm() == ARMCC::NE)
11520 std::swap(destMBB, exitMBB);
11521
11522 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc))
11523 .addMBB(destMBB).addImm(ARMCC::EQ).addReg(ARM::CPSR);
11524 if (isThumb2)
11525 BuildMI(BB, dl, TII->get(ARM::t2B))
11526 .addMBB(exitMBB)
11527 .add(predOps(ARMCC::AL));
11528 else
11529 BuildMI(BB, dl, TII->get(ARM::B)) .addMBB(exitMBB);
11530
11531 MI.eraseFromParent(); // The pseudo instruction is gone now.
11532 return BB;
11533 }
11534
11535 case ARM::Int_eh_sjlj_setjmp:
11536 case ARM::Int_eh_sjlj_setjmp_nofp:
11537 case ARM::tInt_eh_sjlj_setjmp:
11538 case ARM::t2Int_eh_sjlj_setjmp:
11539 case ARM::t2Int_eh_sjlj_setjmp_nofp:
11540 return BB;
11541
11542 case ARM::Int_eh_sjlj_setup_dispatch:
11543 EmitSjLjDispatchBlock(MI, BB);
11544 return BB;
11545
11546 case ARM::ABS:
11547 case ARM::t2ABS: {
11548 // To insert an ABS instruction, we have to insert the
11549 // diamond control-flow pattern. The incoming instruction knows the
11550 // source vreg to test against 0, the destination vreg to set,
11551 // the condition code register to branch on, the
11552 // true/false values to select between, and a branch opcode to use.
11553 // It transforms
11554 // V1 = ABS V0
11555 // into
11556 // V2 = MOVS V0
11557 // BCC (branch to SinkBB if V0 >= 0)
11558 // RSBBB: V3 = RSBri V2, 0 (compute ABS if V2 < 0)
11559 // SinkBB: V1 = PHI(V2, V3)
11560 const BasicBlock *LLVM_BB = BB->getBasicBlock();
11561 MachineFunction::iterator BBI = ++BB->getIterator();
11562 MachineFunction *Fn = BB->getParent();
11563 MachineBasicBlock *RSBBB = Fn->CreateMachineBasicBlock(LLVM_BB);
11564 MachineBasicBlock *SinkBB = Fn->CreateMachineBasicBlock(LLVM_BB);
11565 Fn->insert(BBI, RSBBB);
11566 Fn->insert(BBI, SinkBB);
11567
11568 Register ABSSrcReg = MI.getOperand(1).getReg();
11569 Register ABSDstReg = MI.getOperand(0).getReg();
11570 bool ABSSrcKIll = MI.getOperand(1).isKill();
11571 bool isThumb2 = Subtarget->isThumb2();
11572 MachineRegisterInfo &MRI = Fn->getRegInfo();
11573 // In Thumb mode S must not be specified if source register is the SP or
11574 // PC and if destination register is the SP, so restrict register class
11575 Register NewRsbDstReg = MRI.createVirtualRegister(
11576 isThumb2 ? &ARM::rGPRRegClass : &ARM::GPRRegClass);
11577
11578 // Transfer the remainder of BB and its successor edges to sinkMBB.
11579 SinkBB->splice(SinkBB->begin(), BB,
11580 std::next(MachineBasicBlock::iterator(MI)), BB->end());
11581 SinkBB->transferSuccessorsAndUpdatePHIs(BB);
11582
11583 BB->addSuccessor(RSBBB);
11584 BB->addSuccessor(SinkBB);
11585
11586 // fall through to SinkMBB
11587 RSBBB->addSuccessor(SinkBB);
11588
11589 // insert a cmp at the end of BB
11590 BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPri : ARM::CMPri))
11591 .addReg(ABSSrcReg)
11592 .addImm(0)
11593 .add(predOps(ARMCC::AL));
11594
11595 // insert a bcc with opposite CC to ARMCC::MI at the end of BB
11596 BuildMI(BB, dl,
11597 TII->get(isThumb2 ? ARM::t2Bcc : ARM::Bcc)).addMBB(SinkBB)
11598 .addImm(ARMCC::getOppositeCondition(ARMCC::MI)).addReg(ARM::CPSR);
11599
11600 // insert rsbri in RSBBB
11601 // Note: BCC and rsbri will be converted into predicated rsbmi
11602 // by if-conversion pass
11603 BuildMI(*RSBBB, RSBBB->begin(), dl,
11604 TII->get(isThumb2 ? ARM::t2RSBri : ARM::RSBri), NewRsbDstReg)
11605 .addReg(ABSSrcReg, ABSSrcKIll ? RegState::Kill : 0)
11606 .addImm(0)
11607 .add(predOps(ARMCC::AL))
11608 .add(condCodeOp());
11609
11610 // insert PHI in SinkBB,
11611 // reuse ABSDstReg to not change uses of ABS instruction
11612 BuildMI(*SinkBB, SinkBB->begin(), dl,
11613 TII->get(ARM::PHI), ABSDstReg)
11614 .addReg(NewRsbDstReg).addMBB(RSBBB)
11615 .addReg(ABSSrcReg).addMBB(BB);
11616
11617 // remove ABS instruction
11618 MI.eraseFromParent();
11619
11620 // return last added BB
11621 return SinkBB;
11622 }
11623 case ARM::COPY_STRUCT_BYVAL_I32:
11624 ++NumLoopByVals;
11625 return EmitStructByval(MI, BB);
11626 case ARM::WIN__CHKSTK:
11627 return EmitLowered__chkstk(MI, BB);
11628 case ARM::WIN__DBZCHK:
11629 return EmitLowered__dbzchk(MI, BB);
11630 }
11631}
11632
11633/// Attaches vregs to MEMCPY that it will use as scratch registers
11634/// when it is expanded into LDM/STM. This is done as a post-isel lowering
11635/// instead of as a custom inserter because we need the use list from the SDNode.
11636static void attachMEMCPYScratchRegs(const ARMSubtarget *Subtarget,
11637 MachineInstr &MI, const SDNode *Node) {
11638 bool isThumb1 = Subtarget->isThumb1Only();
11639
11640 DebugLoc DL = MI.getDebugLoc();
11641 MachineFunction *MF = MI.getParent()->getParent();
11642 MachineRegisterInfo &MRI = MF->getRegInfo();
11643 MachineInstrBuilder MIB(*MF, MI);
11644
11645 // If the new dst/src is unused mark it as dead.
11646 if (!Node->hasAnyUseOfValue(0)) {
11647 MI.getOperand(0).setIsDead(true);
11648 }
11649 if (!Node->hasAnyUseOfValue(1)) {
11650 MI.getOperand(1).setIsDead(true);
11651 }
11652
11653 // The MEMCPY both defines and kills the scratch registers.
11654 for (unsigned I = 0; I != MI.getOperand(4).getImm(); ++I) {
11655 Register TmpReg = MRI.createVirtualRegister(isThumb1 ? &ARM::tGPRRegClass
11656 : &ARM::GPRRegClass);
11657 MIB.addReg(TmpReg, RegState::Define|RegState::Dead);
11658 }
11659}
11660
11661void ARMTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
11662 SDNode *Node) const {
11663 if (MI.getOpcode() == ARM::MEMCPY) {
11664 attachMEMCPYScratchRegs(Subtarget, MI, Node);
11665 return;
11666 }
11667
11668 const MCInstrDesc *MCID = &MI.getDesc();
11669 // Adjust potentially 's' setting instructions after isel, i.e. ADC, SBC, RSB,
11670 // RSC. Coming out of isel, they have an implicit CPSR def, but the optional
11671 // operand is still set to noreg. If needed, set the optional operand's
11672 // register to CPSR, and remove the redundant implicit def.
11673 //
11674 // e.g. ADCS (..., implicit-def CPSR) -> ADC (... opt:def CPSR).
11675
11676 // Rename pseudo opcodes.
11677 unsigned NewOpc = convertAddSubFlagsOpcode(MI.getOpcode());
11678 unsigned ccOutIdx;
11679 if (NewOpc) {
11680 const ARMBaseInstrInfo *TII = Subtarget->getInstrInfo();
11681 MCID = &TII->get(NewOpc);
11682
11683 assert(MCID->getNumOperands() ==(static_cast <bool> (MCID->getNumOperands() == MI.getDesc
().getNumOperands() + 5 - MI.getDesc().getSize() && "converted opcode should be the same except for cc_out"
" (and, on Thumb1, pred)") ? void (0) : __assert_fail ("MCID->getNumOperands() == MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() && \"converted opcode should be the same except for cc_out\" \" (and, on Thumb1, pred)\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11686, __extension__ __PRETTY_FUNCTION__))
11684 MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize()(static_cast <bool> (MCID->getNumOperands() == MI.getDesc
().getNumOperands() + 5 - MI.getDesc().getSize() && "converted opcode should be the same except for cc_out"
" (and, on Thumb1, pred)") ? void (0) : __assert_fail ("MCID->getNumOperands() == MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() && \"converted opcode should be the same except for cc_out\" \" (and, on Thumb1, pred)\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11686, __extension__ __PRETTY_FUNCTION__))
11685 && "converted opcode should be the same except for cc_out"(static_cast <bool> (MCID->getNumOperands() == MI.getDesc
().getNumOperands() + 5 - MI.getDesc().getSize() && "converted opcode should be the same except for cc_out"
" (and, on Thumb1, pred)") ? void (0) : __assert_fail ("MCID->getNumOperands() == MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() && \"converted opcode should be the same except for cc_out\" \" (and, on Thumb1, pred)\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11686, __extension__ __PRETTY_FUNCTION__))
11686 " (and, on Thumb1, pred)")(static_cast <bool> (MCID->getNumOperands() == MI.getDesc
().getNumOperands() + 5 - MI.getDesc().getSize() && "converted opcode should be the same except for cc_out"
" (and, on Thumb1, pred)") ? void (0) : __assert_fail ("MCID->getNumOperands() == MI.getDesc().getNumOperands() + 5 - MI.getDesc().getSize() && \"converted opcode should be the same except for cc_out\" \" (and, on Thumb1, pred)\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11686, __extension__ __PRETTY_FUNCTION__))
;
11687
11688 MI.setDesc(*MCID);
11689
11690 // Add the optional cc_out operand
11691 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/true));
11692
11693 // On Thumb1, move all input operands to the end, then add the predicate
11694 if (Subtarget->isThumb1Only()) {
11695 for (unsigned c = MCID->getNumOperands() - 4; c--;) {
11696 MI.addOperand(MI.getOperand(1));
11697 MI.RemoveOperand(1);
11698 }
11699
11700 // Restore the ties
11701 for (unsigned i = MI.getNumOperands(); i--;) {
11702 const MachineOperand& op = MI.getOperand(i);
11703 if (op.isReg() && op.isUse()) {
11704 int DefIdx = MCID->getOperandConstraint(i, MCOI::TIED_TO);
11705 if (DefIdx != -1)
11706 MI.tieOperands(DefIdx, i);
11707 }
11708 }
11709
11710 MI.addOperand(MachineOperand::CreateImm(ARMCC::AL));
11711 MI.addOperand(MachineOperand::CreateReg(0, /*isDef=*/false));
11712 ccOutIdx = 1;
11713 } else
11714 ccOutIdx = MCID->getNumOperands() - 1;
11715 } else
11716 ccOutIdx = MCID->getNumOperands() - 1;
11717
11718 // Any ARM instruction that sets the 's' bit should specify an optional
11719 // "cc_out" operand in the last operand position.
11720 if (!MI.hasOptionalDef() || !MCID->OpInfo[ccOutIdx].isOptionalDef()) {
11721 assert(!NewOpc && "Optional cc_out operand required")(static_cast <bool> (!NewOpc && "Optional cc_out operand required"
) ? void (0) : __assert_fail ("!NewOpc && \"Optional cc_out operand required\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11721, __extension__ __PRETTY_FUNCTION__))
;
11722 return;
11723 }
11724 // Look for an implicit def of CPSR added by MachineInstr ctor. Remove it
11725 // since we already have an optional CPSR def.
11726 bool definesCPSR = false;
11727 bool deadCPSR = false;
11728 for (unsigned i = MCID->getNumOperands(), e = MI.getNumOperands(); i != e;
11729 ++i) {
11730 const MachineOperand &MO = MI.getOperand(i);
11731 if (MO.isReg() && MO.isDef() && MO.getReg() == ARM::CPSR) {
11732 definesCPSR = true;
11733 if (MO.isDead())
11734 deadCPSR = true;
11735 MI.RemoveOperand(i);
11736 break;
11737 }
11738 }
11739 if (!definesCPSR) {
11740 assert(!NewOpc && "Optional cc_out operand required")(static_cast <bool> (!NewOpc && "Optional cc_out operand required"
) ? void (0) : __assert_fail ("!NewOpc && \"Optional cc_out operand required\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11740, __extension__ __PRETTY_FUNCTION__))
;
11741 return;
11742 }
11743 assert(deadCPSR == !Node->hasAnyUseOfValue(1) && "inconsistent dead flag")(static_cast <bool> (deadCPSR == !Node->hasAnyUseOfValue
(1) && "inconsistent dead flag") ? void (0) : __assert_fail
("deadCPSR == !Node->hasAnyUseOfValue(1) && \"inconsistent dead flag\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11743, __extension__ __PRETTY_FUNCTION__))
;
11744 if (deadCPSR) {
11745 assert(!MI.getOperand(ccOutIdx).getReg() &&(static_cast <bool> (!MI.getOperand(ccOutIdx).getReg() &&
"expect uninitialized optional cc_out operand") ? void (0) :
__assert_fail ("!MI.getOperand(ccOutIdx).getReg() && \"expect uninitialized optional cc_out operand\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11746, __extension__ __PRETTY_FUNCTION__))
11746 "expect uninitialized optional cc_out operand")(static_cast <bool> (!MI.getOperand(ccOutIdx).getReg() &&
"expect uninitialized optional cc_out operand") ? void (0) :
__assert_fail ("!MI.getOperand(ccOutIdx).getReg() && \"expect uninitialized optional cc_out operand\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 11746, __extension__ __PRETTY_FUNCTION__))
;
11747 // Thumb1 instructions must have the S bit even if the CPSR is dead.
11748 if (!Subtarget->isThumb1Only())
11749 return;
11750 }
11751
11752 // If this instruction was defined with an optional CPSR def and its dag node
11753 // had a live implicit CPSR def, then activate the optional CPSR def.
11754 MachineOperand &MO = MI.getOperand(ccOutIdx);
11755 MO.setReg(ARM::CPSR);
11756 MO.setIsDef(true);
11757}
11758
11759//===----------------------------------------------------------------------===//
11760// ARM Optimization Hooks
11761//===----------------------------------------------------------------------===//
11762
11763// Helper function that checks if N is a null or all ones constant.
11764static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
11765 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
11766}
11767
11768// Return true if N is conditionally 0 or all ones.
11769// Detects these expressions where cc is an i1 value:
11770//
11771// (select cc 0, y) [AllOnes=0]
11772// (select cc y, 0) [AllOnes=0]
11773// (zext cc) [AllOnes=0]
11774// (sext cc) [AllOnes=0/1]
11775// (select cc -1, y) [AllOnes=1]
11776// (select cc y, -1) [AllOnes=1]
11777//
11778// Invert is set when N is the null/all ones constant when CC is false.
11779// OtherOp is set to the alternative value of N.
11780static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
11781 SDValue &CC, bool &Invert,
11782 SDValue &OtherOp,
11783 SelectionDAG &DAG) {
11784 switch (N->getOpcode()) {
11785 default: return false;
11786 case ISD::SELECT: {
11787 CC = N->getOperand(0);
11788 SDValue N1 = N->getOperand(1);
11789 SDValue N2 = N->getOperand(2);
11790 if (isZeroOrAllOnes(N1, AllOnes)) {
11791 Invert = false;
11792 OtherOp = N2;
11793 return true;
11794 }
11795 if (isZeroOrAllOnes(N2, AllOnes)) {
11796 Invert = true;
11797 OtherOp = N1;
11798 return true;
11799 }
11800 return false;
11801 }
11802 case ISD::ZERO_EXTEND:
11803 // (zext cc) can never be the all ones value.
11804 if (AllOnes)
11805 return false;
11806 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11807 case ISD::SIGN_EXTEND: {
11808 SDLoc dl(N);
11809 EVT VT = N->getValueType(0);
11810 CC = N->getOperand(0);
11811 if (CC.getValueType() != MVT::i1 || CC.getOpcode() != ISD::SETCC)
11812 return false;
11813 Invert = !AllOnes;
11814 if (AllOnes)
11815 // When looking for an AllOnes constant, N is an sext, and the 'other'
11816 // value is 0.
11817 OtherOp = DAG.getConstant(0, dl, VT);
11818 else if (N->getOpcode() == ISD::ZERO_EXTEND)
11819 // When looking for a 0 constant, N can be zext or sext.
11820 OtherOp = DAG.getConstant(1, dl, VT);
11821 else
11822 OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
11823 VT);
11824 return true;
11825 }
11826 }
11827}
11828
11829// Combine a constant select operand into its use:
11830//
11831// (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
11832// (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
11833// (and (select cc, -1, c), x) -> (select cc, x, (and, x, c)) [AllOnes=1]
11834// (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
11835// (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
11836//
11837// The transform is rejected if the select doesn't have a constant operand that
11838// is null, or all ones when AllOnes is set.
11839//
11840// Also recognize sext/zext from i1:
11841//
11842// (add (zext cc), x) -> (select cc (add x, 1), x)
11843// (add (sext cc), x) -> (select cc (add x, -1), x)
11844//
11845// These transformations eventually create predicated instructions.
11846//
11847// @param N The node to transform.
11848// @param Slct The N operand that is a select.
11849// @param OtherOp The other N operand (x above).
11850// @param DCI Context.
11851// @param AllOnes Require the select constant to be all ones instead of null.
11852// @returns The new node, or SDValue() on failure.
11853static
11854SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
11855 TargetLowering::DAGCombinerInfo &DCI,
11856 bool AllOnes = false) {
11857 SelectionDAG &DAG = DCI.DAG;
11858 EVT VT = N->getValueType(0);
11859 SDValue NonConstantVal;
11860 SDValue CCOp;
11861 bool SwapSelectOps;
11862 if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
11863 NonConstantVal, DAG))
11864 return SDValue();
11865
11866 // Slct is now know to be the desired identity constant when CC is true.
11867 SDValue TrueVal = OtherOp;
11868 SDValue FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT,
11869 OtherOp, NonConstantVal);
11870 // Unless SwapSelectOps says CC should be false.
11871 if (SwapSelectOps)
11872 std::swap(TrueVal, FalseVal);
11873
11874 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
11875 CCOp, TrueVal, FalseVal);
11876}
11877
11878// Attempt combineSelectAndUse on each operand of a commutative operator N.
11879static
11880SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
11881 TargetLowering::DAGCombinerInfo &DCI) {
11882 SDValue N0 = N->getOperand(0);
11883 SDValue N1 = N->getOperand(1);
11884 if (N0.getNode()->hasOneUse())
11885 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes))
11886 return Result;
11887 if (N1.getNode()->hasOneUse())
11888 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes))
11889 return Result;
11890 return SDValue();
11891}
11892
11893static bool IsVUZPShuffleNode(SDNode *N) {
11894 // VUZP shuffle node.
11895 if (N->getOpcode() == ARMISD::VUZP)
11896 return true;
11897
11898 // "VUZP" on i32 is an alias for VTRN.
11899 if (N->getOpcode() == ARMISD::VTRN && N->getValueType(0) == MVT::v2i32)
11900 return true;
11901
11902 return false;
11903}
11904
11905static SDValue AddCombineToVPADD(SDNode *N, SDValue N0, SDValue N1,
11906 TargetLowering::DAGCombinerInfo &DCI,
11907 const ARMSubtarget *Subtarget) {
11908 // Look for ADD(VUZP.0, VUZP.1).
11909 if (!IsVUZPShuffleNode(N0.getNode()) || N0.getNode() != N1.getNode() ||
11910 N0 == N1)
11911 return SDValue();
11912
11913 // Make sure the ADD is a 64-bit add; there is no 128-bit VPADD.
11914 if (!N->getValueType(0).is64BitVector())
11915 return SDValue();
11916
11917 // Generate vpadd.
11918 SelectionDAG &DAG = DCI.DAG;
11919 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11920 SDLoc dl(N);
11921 SDNode *Unzip = N0.getNode();
11922 EVT VT = N->getValueType(0);
11923
11924 SmallVector<SDValue, 8> Ops;
11925 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpadd, dl,
11926 TLI.getPointerTy(DAG.getDataLayout())));
11927 Ops.push_back(Unzip->getOperand(0));
11928 Ops.push_back(Unzip->getOperand(1));
11929
11930 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
11931}
11932
11933static SDValue AddCombineVUZPToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11934 TargetLowering::DAGCombinerInfo &DCI,
11935 const ARMSubtarget *Subtarget) {
11936 // Check for two extended operands.
11937 if (!(N0.getOpcode() == ISD::SIGN_EXTEND &&
11938 N1.getOpcode() == ISD::SIGN_EXTEND) &&
11939 !(N0.getOpcode() == ISD::ZERO_EXTEND &&
11940 N1.getOpcode() == ISD::ZERO_EXTEND))
11941 return SDValue();
11942
11943 SDValue N00 = N0.getOperand(0);
11944 SDValue N10 = N1.getOperand(0);
11945
11946 // Look for ADD(SEXT(VUZP.0), SEXT(VUZP.1))
11947 if (!IsVUZPShuffleNode(N00.getNode()) || N00.getNode() != N10.getNode() ||
11948 N00 == N10)
11949 return SDValue();
11950
11951 // We only recognize Q register paddl here; this can't be reached until
11952 // after type legalization.
11953 if (!N00.getValueType().is64BitVector() ||
11954 !N0.getValueType().is128BitVector())
11955 return SDValue();
11956
11957 // Generate vpaddl.
11958 SelectionDAG &DAG = DCI.DAG;
11959 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
11960 SDLoc dl(N);
11961 EVT VT = N->getValueType(0);
11962
11963 SmallVector<SDValue, 8> Ops;
11964 // Form vpaddl.sN or vpaddl.uN depending on the kind of extension.
11965 unsigned Opcode;
11966 if (N0.getOpcode() == ISD::SIGN_EXTEND)
11967 Opcode = Intrinsic::arm_neon_vpaddls;
11968 else
11969 Opcode = Intrinsic::arm_neon_vpaddlu;
11970 Ops.push_back(DAG.getConstant(Opcode, dl,
11971 TLI.getPointerTy(DAG.getDataLayout())));
11972 EVT ElemTy = N00.getValueType().getVectorElementType();
11973 unsigned NumElts = VT.getVectorNumElements();
11974 EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), ElemTy, NumElts * 2);
11975 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), ConcatVT,
11976 N00.getOperand(0), N00.getOperand(1));
11977 Ops.push_back(Concat);
11978
11979 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT, Ops);
11980}
11981
11982// FIXME: This function shouldn't be necessary; if we lower BUILD_VECTOR in
11983// an appropriate manner, we end up with ADD(VUZP(ZEXT(N))), which is
11984// much easier to match.
11985static SDValue
11986AddCombineBUILD_VECTORToVPADDL(SDNode *N, SDValue N0, SDValue N1,
11987 TargetLowering::DAGCombinerInfo &DCI,
11988 const ARMSubtarget *Subtarget) {
11989 // Only perform optimization if after legalize, and if NEON is available. We
11990 // also expected both operands to be BUILD_VECTORs.
11991 if (DCI.isBeforeLegalize() || !Subtarget->hasNEON()
11992 || N0.getOpcode() != ISD::BUILD_VECTOR
11993 || N1.getOpcode() != ISD::BUILD_VECTOR)
11994 return SDValue();
11995
11996 // Check output type since VPADDL operand elements can only be 8, 16, or 32.
11997 EVT VT = N->getValueType(0);
11998 if (!VT.isInteger() || VT.getVectorElementType() == MVT::i64)
11999 return SDValue();
12000
12001 // Check that the vector operands are of the right form.
12002 // N0 and N1 are BUILD_VECTOR nodes with N number of EXTRACT_VECTOR
12003 // operands, where N is the size of the formed vector.
12004 // Each EXTRACT_VECTOR should have the same input vector and odd or even
12005 // index such that we have a pair wise add pattern.
12006
12007 // Grab the vector that all EXTRACT_VECTOR nodes should be referencing.
12008 if (N0->getOperand(0)->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
12009 return SDValue();
12010 SDValue Vec = N0->getOperand(0)->getOperand(0);
12011 SDNode *V = Vec.getNode();
12012 unsigned nextIndex = 0;
12013
12014 // For each operands to the ADD which are BUILD_VECTORs,
12015 // check to see if each of their operands are an EXTRACT_VECTOR with
12016 // the same vector and appropriate index.
12017 for (unsigned i = 0, e = N0->getNumOperands(); i != e; ++i) {
12018 if (N0->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT
12019 && N1->getOperand(i)->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
12020
12021 SDValue ExtVec0 = N0->getOperand(i);
12022 SDValue ExtVec1 = N1->getOperand(i);
12023
12024 // First operand is the vector, verify its the same.
12025 if (V != ExtVec0->getOperand(0).getNode() ||
12026 V != ExtVec1->getOperand(0).getNode())
12027 return SDValue();
12028
12029 // Second is the constant, verify its correct.
12030 ConstantSDNode *C0 = dyn_cast<ConstantSDNode>(ExtVec0->getOperand(1));
12031 ConstantSDNode *C1 = dyn_cast<ConstantSDNode>(ExtVec1->getOperand(1));
12032
12033 // For the constant, we want to see all the even or all the odd.
12034 if (!C0 || !C1 || C0->getZExtValue() != nextIndex
12035 || C1->getZExtValue() != nextIndex+1)
12036 return SDValue();
12037
12038 // Increment index.
12039 nextIndex+=2;
12040 } else
12041 return SDValue();
12042 }
12043
12044 // Don't generate vpaddl+vmovn; we'll match it to vpadd later. Also make sure
12045 // we're using the entire input vector, otherwise there's a size/legality
12046 // mismatch somewhere.
12047 if (nextIndex != Vec.getValueType().getVectorNumElements() ||
12048 Vec.getValueType().getVectorElementType() == VT.getVectorElementType())
12049 return SDValue();
12050
12051 // Create VPADDL node.
12052 SelectionDAG &DAG = DCI.DAG;
12053 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12054
12055 SDLoc dl(N);
12056
12057 // Build operand list.
12058 SmallVector<SDValue, 8> Ops;
12059 Ops.push_back(DAG.getConstant(Intrinsic::arm_neon_vpaddls, dl,
12060 TLI.getPointerTy(DAG.getDataLayout())));
12061
12062 // Input is the vector.
12063 Ops.push_back(Vec);
12064
12065 // Get widened type and narrowed type.
12066 MVT widenType;
12067 unsigned numElem = VT.getVectorNumElements();
12068
12069 EVT inputLaneType = Vec.getValueType().getVectorElementType();
12070 switch (inputLaneType.getSimpleVT().SimpleTy) {
12071 case MVT::i8: widenType = MVT::getVectorVT(MVT::i16, numElem); break;
12072 case MVT::i16: widenType = MVT::getVectorVT(MVT::i32, numElem); break;
12073 case MVT::i32: widenType = MVT::getVectorVT(MVT::i64, numElem); break;
12074 default:
12075 llvm_unreachable("Invalid vector element type for padd optimization.")::llvm::llvm_unreachable_internal("Invalid vector element type for padd optimization."
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12075)
;
12076 }
12077
12078 SDValue tmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, widenType, Ops);
12079 unsigned ExtOp = VT.bitsGT(tmp.getValueType()) ? ISD::ANY_EXTEND : ISD::TRUNCATE;
12080 return DAG.getNode(ExtOp, dl, VT, tmp);
12081}
12082
12083static SDValue findMUL_LOHI(SDValue V) {
12084 if (V->getOpcode() == ISD::UMUL_LOHI ||
12085 V->getOpcode() == ISD::SMUL_LOHI)
12086 return V;
12087 return SDValue();
12088}
12089
12090static SDValue AddCombineTo64BitSMLAL16(SDNode *AddcNode, SDNode *AddeNode,
12091 TargetLowering::DAGCombinerInfo &DCI,
12092 const ARMSubtarget *Subtarget) {
12093 if (!Subtarget->hasBaseDSP())
12094 return SDValue();
12095
12096 // SMLALBB, SMLALBT, SMLALTB, SMLALTT multiply two 16-bit values and
12097 // accumulates the product into a 64-bit value. The 16-bit values will
12098 // be sign extended somehow or SRA'd into 32-bit values
12099 // (addc (adde (mul 16bit, 16bit), lo), hi)
12100 SDValue Mul = AddcNode->getOperand(0);
12101 SDValue Lo = AddcNode->getOperand(1);
12102 if (Mul.getOpcode() != ISD::MUL) {
12103 Lo = AddcNode->getOperand(0);
12104 Mul = AddcNode->getOperand(1);
12105 if (Mul.getOpcode() != ISD::MUL)
12106 return SDValue();
12107 }
12108
12109 SDValue SRA = AddeNode->getOperand(0);
12110 SDValue Hi = AddeNode->getOperand(1);
12111 if (SRA.getOpcode() != ISD::SRA) {
12112 SRA = AddeNode->getOperand(1);
12113 Hi = AddeNode->getOperand(0);
12114 if (SRA.getOpcode() != ISD::SRA)
12115 return SDValue();
12116 }
12117 if (auto Const = dyn_cast<ConstantSDNode>(SRA.getOperand(1))) {
12118 if (Const->getZExtValue() != 31)
12119 return SDValue();
12120 } else
12121 return SDValue();
12122
12123 if (SRA.getOperand(0) != Mul)
12124 return SDValue();
12125
12126 SelectionDAG &DAG = DCI.DAG;
12127 SDLoc dl(AddcNode);
12128 unsigned Opcode = 0;
12129 SDValue Op0;
12130 SDValue Op1;
12131
12132 if (isS16(Mul.getOperand(0), DAG) && isS16(Mul.getOperand(1), DAG)) {
12133 Opcode = ARMISD::SMLALBB;
12134 Op0 = Mul.getOperand(0);
12135 Op1 = Mul.getOperand(1);
12136 } else if (isS16(Mul.getOperand(0), DAG) && isSRA16(Mul.getOperand(1))) {
12137 Opcode = ARMISD::SMLALBT;
12138 Op0 = Mul.getOperand(0);
12139 Op1 = Mul.getOperand(1).getOperand(0);
12140 } else if (isSRA16(Mul.getOperand(0)) && isS16(Mul.getOperand(1), DAG)) {
12141 Opcode = ARMISD::SMLALTB;
12142 Op0 = Mul.getOperand(0).getOperand(0);
12143 Op1 = Mul.getOperand(1);
12144 } else if (isSRA16(Mul.getOperand(0)) && isSRA16(Mul.getOperand(1))) {
12145 Opcode = ARMISD::SMLALTT;
12146 Op0 = Mul->getOperand(0).getOperand(0);
12147 Op1 = Mul->getOperand(1).getOperand(0);
12148 }
12149
12150 if (!Op0 || !Op1)
12151 return SDValue();
12152
12153 SDValue SMLAL = DAG.getNode(Opcode, dl, DAG.getVTList(MVT::i32, MVT::i32),
12154 Op0, Op1, Lo, Hi);
12155 // Replace the ADDs' nodes uses by the MLA node's values.
12156 SDValue HiMLALResult(SMLAL.getNode(), 1);
12157 SDValue LoMLALResult(SMLAL.getNode(), 0);
12158
12159 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
12160 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
12161
12162 // Return original node to notify the driver to stop replacing.
12163 SDValue resNode(AddcNode, 0);
12164 return resNode;
12165}
12166
12167static SDValue AddCombineTo64bitMLAL(SDNode *AddeSubeNode,
12168 TargetLowering::DAGCombinerInfo &DCI,
12169 const ARMSubtarget *Subtarget) {
12170 // Look for multiply add opportunities.
12171 // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
12172 // each add nodes consumes a value from ISD::UMUL_LOHI and there is
12173 // a glue link from the first add to the second add.
12174 // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
12175 // a S/UMLAL instruction.
12176 // UMUL_LOHI
12177 // / :lo \ :hi
12178 // V \ [no multiline comment]
12179 // loAdd -> ADDC |
12180 // \ :carry /
12181 // V V
12182 // ADDE <- hiAdd
12183 //
12184 // In the special case where only the higher part of a signed result is used
12185 // and the add to the low part of the result of ISD::UMUL_LOHI adds or subtracts
12186 // a constant with the exact value of 0x80000000, we recognize we are dealing
12187 // with a "rounded multiply and add" (or subtract) and transform it into
12188 // either a ARMISD::SMMLAR or ARMISD::SMMLSR respectively.
12189
12190 assert((AddeSubeNode->getOpcode() == ARMISD::ADDE ||(static_cast <bool> ((AddeSubeNode->getOpcode() == ARMISD
::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
"Expect an ADDE or SUBE") ? void (0) : __assert_fail ("(AddeSubeNode->getOpcode() == ARMISD::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) && \"Expect an ADDE or SUBE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12192, __extension__ __PRETTY_FUNCTION__))
12191 AddeSubeNode->getOpcode() == ARMISD::SUBE) &&(static_cast <bool> ((AddeSubeNode->getOpcode() == ARMISD
::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
"Expect an ADDE or SUBE") ? void (0) : __assert_fail ("(AddeSubeNode->getOpcode() == ARMISD::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) && \"Expect an ADDE or SUBE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12192, __extension__ __PRETTY_FUNCTION__))
12192 "Expect an ADDE or SUBE")(static_cast <bool> ((AddeSubeNode->getOpcode() == ARMISD
::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) &&
"Expect an ADDE or SUBE") ? void (0) : __assert_fail ("(AddeSubeNode->getOpcode() == ARMISD::ADDE || AddeSubeNode->getOpcode() == ARMISD::SUBE) && \"Expect an ADDE or SUBE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12192, __extension__ __PRETTY_FUNCTION__))
;
12193
12194 assert(AddeSubeNode->getNumOperands() == 3 &&(static_cast <bool> (AddeSubeNode->getNumOperands() ==
3 && AddeSubeNode->getOperand(2).getValueType() ==
MVT::i32 && "ADDE node has the wrong inputs") ? void
(0) : __assert_fail ("AddeSubeNode->getNumOperands() == 3 && AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && \"ADDE node has the wrong inputs\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12196, __extension__ __PRETTY_FUNCTION__))
12195 AddeSubeNode->getOperand(2).getValueType() == MVT::i32 &&(static_cast <bool> (AddeSubeNode->getNumOperands() ==
3 && AddeSubeNode->getOperand(2).getValueType() ==
MVT::i32 && "ADDE node has the wrong inputs") ? void
(0) : __assert_fail ("AddeSubeNode->getNumOperands() == 3 && AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && \"ADDE node has the wrong inputs\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12196, __extension__ __PRETTY_FUNCTION__))
12196 "ADDE node has the wrong inputs")(static_cast <bool> (AddeSubeNode->getNumOperands() ==
3 && AddeSubeNode->getOperand(2).getValueType() ==
MVT::i32 && "ADDE node has the wrong inputs") ? void
(0) : __assert_fail ("AddeSubeNode->getNumOperands() == 3 && AddeSubeNode->getOperand(2).getValueType() == MVT::i32 && \"ADDE node has the wrong inputs\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12196, __extension__ __PRETTY_FUNCTION__))
;
12197
12198 // Check that we are chained to the right ADDC or SUBC node.
12199 SDNode *AddcSubcNode = AddeSubeNode->getOperand(2).getNode();
12200 if ((AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12201 AddcSubcNode->getOpcode() != ARMISD::ADDC) ||
12202 (AddeSubeNode->getOpcode() == ARMISD::SUBE &&
12203 AddcSubcNode->getOpcode() != ARMISD::SUBC))
12204 return SDValue();
12205
12206 SDValue AddcSubcOp0 = AddcSubcNode->getOperand(0);
12207 SDValue AddcSubcOp1 = AddcSubcNode->getOperand(1);
12208
12209 // Check if the two operands are from the same mul_lohi node.
12210 if (AddcSubcOp0.getNode() == AddcSubcOp1.getNode())
12211 return SDValue();
12212
12213 assert(AddcSubcNode->getNumValues() == 2 &&(static_cast <bool> (AddcSubcNode->getNumValues() ==
2 && AddcSubcNode->getValueType(0) == MVT::i32 &&
"Expect ADDC with two result values. First: i32") ? void (0)
: __assert_fail ("AddcSubcNode->getNumValues() == 2 && AddcSubcNode->getValueType(0) == MVT::i32 && \"Expect ADDC with two result values. First: i32\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12215, __extension__ __PRETTY_FUNCTION__))
12214 AddcSubcNode->getValueType(0) == MVT::i32 &&(static_cast <bool> (AddcSubcNode->getNumValues() ==
2 && AddcSubcNode->getValueType(0) == MVT::i32 &&
"Expect ADDC with two result values. First: i32") ? void (0)
: __assert_fail ("AddcSubcNode->getNumValues() == 2 && AddcSubcNode->getValueType(0) == MVT::i32 && \"Expect ADDC with two result values. First: i32\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12215, __extension__ __PRETTY_FUNCTION__))
12215 "Expect ADDC with two result values. First: i32")(static_cast <bool> (AddcSubcNode->getNumValues() ==
2 && AddcSubcNode->getValueType(0) == MVT::i32 &&
"Expect ADDC with two result values. First: i32") ? void (0)
: __assert_fail ("AddcSubcNode->getNumValues() == 2 && AddcSubcNode->getValueType(0) == MVT::i32 && \"Expect ADDC with two result values. First: i32\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 12215, __extension__ __PRETTY_FUNCTION__))
;
12216
12217 // Check that the ADDC adds the low result of the S/UMUL_LOHI. If not, it
12218 // maybe a SMLAL which multiplies two 16-bit values.
12219 if (AddeSubeNode->getOpcode() == ARMISD::ADDE &&
12220 AddcSubcOp0->getOpcode() != ISD::UMUL_LOHI &&
12221 AddcSubcOp0->getOpcode() != ISD::SMUL_LOHI &&
12222 AddcSubcOp1->getOpcode() != ISD::UMUL_LOHI &&
12223 AddcSubcOp1->getOpcode() != ISD::SMUL_LOHI)
12224 return AddCombineTo64BitSMLAL16(AddcSubcNode, AddeSubeNode, DCI, Subtarget);
12225
12226 // Check for the triangle shape.
12227 SDValue AddeSubeOp0 = AddeSubeNode->getOperand(0);
12228 SDValue AddeSubeOp1 = AddeSubeNode->getOperand(1);
12229
12230 // Make sure that the ADDE/SUBE operands are not coming from the same node.
12231 if (AddeSubeOp0.getNode() == AddeSubeOp1.getNode())
12232 return SDValue();
12233
12234 // Find the MUL_LOHI node walking up ADDE/SUBE's operands.
12235 bool IsLeftOperandMUL = false;
12236 SDValue MULOp = findMUL_LOHI(AddeSubeOp0);
12237 if (MULOp == SDValue())
12238 MULOp = findMUL_LOHI(AddeSubeOp1);
12239 else
12240 IsLeftOperandMUL = true;
12241 if (MULOp == SDValue())
12242 return SDValue();
12243
12244 // Figure out the right opcode.
12245 unsigned Opc = MULOp->getOpcode();
12246 unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
12247
12248 // Figure out the high and low input values to the MLAL node.
12249 SDValue *HiAddSub = nullptr;
12250 SDValue *LoMul = nullptr;
12251 SDValue *LowAddSub = nullptr;
12252
12253 // Ensure that ADDE/SUBE is from high result of ISD::xMUL_LOHI.
12254 if ((AddeSubeOp0 != MULOp.getValue(1)) && (AddeSubeOp1 != MULOp.getValue(1)))
12255 return SDValue();
12256
12257 if (IsLeftOperandMUL)
12258 HiAddSub = &AddeSubeOp1;
12259 else
12260 HiAddSub = &AddeSubeOp0;
12261
12262 // Ensure that LoMul and LowAddSub are taken from correct ISD::SMUL_LOHI node
12263 // whose low result is fed to the ADDC/SUBC we are checking.
12264
12265 if (AddcSubcOp0 == MULOp.getValue(0)) {
12266 LoMul = &AddcSubcOp0;
12267 LowAddSub = &AddcSubcOp1;
12268 }
12269 if (AddcSubcOp1 == MULOp.getValue(0)) {
12270 LoMul = &AddcSubcOp1;
12271 LowAddSub = &AddcSubcOp0;
12272 }
12273
12274 if (!LoMul)
12275 return SDValue();
12276
12277 // If HiAddSub is the same node as ADDC/SUBC or is a predecessor of ADDC/SUBC
12278 // the replacement below will create a cycle.
12279 if (AddcSubcNode == HiAddSub->getNode() ||
12280 AddcSubcNode->isPredecessorOf(HiAddSub->getNode()))
12281 return SDValue();
12282
12283 // Create the merged node.
12284 SelectionDAG &DAG = DCI.DAG;
12285
12286 // Start building operand list.
12287 SmallVector<SDValue, 8> Ops;
12288 Ops.push_back(LoMul->getOperand(0));
12289 Ops.push_back(LoMul->getOperand(1));
12290
12291 // Check whether we can use SMMLAR, SMMLSR or SMMULR instead. For this to be
12292 // the case, we must be doing signed multiplication and only use the higher
12293 // part of the result of the MLAL, furthermore the LowAddSub must be a constant
12294 // addition or subtraction with the value of 0x800000.
12295 if (Subtarget->hasV6Ops() && Subtarget->hasDSP() && Subtarget->useMulOps() &&
12296 FinalOpc == ARMISD::SMLAL && !AddeSubeNode->hasAnyUseOfValue(1) &&
12297 LowAddSub->getNode()->getOpcode() == ISD::Constant &&
12298 static_cast<ConstantSDNode *>(LowAddSub->getNode())->getZExtValue() ==
12299 0x80000000) {
12300 Ops.push_back(*HiAddSub);
12301 if (AddcSubcNode->getOpcode() == ARMISD::SUBC) {
12302 FinalOpc = ARMISD::SMMLSR;
12303 } else {
12304 FinalOpc = ARMISD::SMMLAR;
12305 }
12306 SDValue NewNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode), MVT::i32, Ops);
12307 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), NewNode);
12308
12309 return SDValue(AddeSubeNode, 0);
12310 } else if (AddcSubcNode->getOpcode() == ARMISD::SUBC)
12311 // SMMLS is generated during instruction selection and the rest of this
12312 // function can not handle the case where AddcSubcNode is a SUBC.
12313 return SDValue();
12314
12315 // Finish building the operand list for {U/S}MLAL
12316 Ops.push_back(*LowAddSub);
12317 Ops.push_back(*HiAddSub);
12318
12319 SDValue MLALNode = DAG.getNode(FinalOpc, SDLoc(AddcSubcNode),
12320 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12321
12322 // Replace the ADDs' nodes uses by the MLA node's values.
12323 SDValue HiMLALResult(MLALNode.getNode(), 1);
12324 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeSubeNode, 0), HiMLALResult);
12325
12326 SDValue LoMLALResult(MLALNode.getNode(), 0);
12327 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcSubcNode, 0), LoMLALResult);
12328
12329 // Return original node to notify the driver to stop replacing.
12330 return SDValue(AddeSubeNode, 0);
12331}
12332
12333static SDValue AddCombineTo64bitUMAAL(SDNode *AddeNode,
12334 TargetLowering::DAGCombinerInfo &DCI,
12335 const ARMSubtarget *Subtarget) {
12336 // UMAAL is similar to UMLAL except that it adds two unsigned values.
12337 // While trying to combine for the other MLAL nodes, first search for the
12338 // chance to use UMAAL. Check if Addc uses a node which has already
12339 // been combined into a UMLAL. The other pattern is UMLAL using Addc/Adde
12340 // as the addend, and it's handled in PerformUMLALCombine.
12341
12342 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12343 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12344
12345 // Check that we have a glued ADDC node.
12346 SDNode* AddcNode = AddeNode->getOperand(2).getNode();
12347 if (AddcNode->getOpcode() != ARMISD::ADDC)
12348 return SDValue();
12349
12350 // Find the converted UMAAL or quit if it doesn't exist.
12351 SDNode *UmlalNode = nullptr;
12352 SDValue AddHi;
12353 if (AddcNode->getOperand(0).getOpcode() == ARMISD::UMLAL) {
12354 UmlalNode = AddcNode->getOperand(0).getNode();
12355 AddHi = AddcNode->getOperand(1);
12356 } else if (AddcNode->getOperand(1).getOpcode() == ARMISD::UMLAL) {
12357 UmlalNode = AddcNode->getOperand(1).getNode();
12358 AddHi = AddcNode->getOperand(0);
12359 } else {
12360 return AddCombineTo64bitMLAL(AddeNode, DCI, Subtarget);
12361 }
12362
12363 // The ADDC should be glued to an ADDE node, which uses the same UMLAL as
12364 // the ADDC as well as Zero.
12365 if (!isNullConstant(UmlalNode->getOperand(3)))
12366 return SDValue();
12367
12368 if ((isNullConstant(AddeNode->getOperand(0)) &&
12369 AddeNode->getOperand(1).getNode() == UmlalNode) ||
12370 (AddeNode->getOperand(0).getNode() == UmlalNode &&
12371 isNullConstant(AddeNode->getOperand(1)))) {
12372 SelectionDAG &DAG = DCI.DAG;
12373 SDValue Ops[] = { UmlalNode->getOperand(0), UmlalNode->getOperand(1),
12374 UmlalNode->getOperand(2), AddHi };
12375 SDValue UMAAL = DAG.getNode(ARMISD::UMAAL, SDLoc(AddcNode),
12376 DAG.getVTList(MVT::i32, MVT::i32), Ops);
12377
12378 // Replace the ADDs' nodes uses by the UMAAL node's values.
12379 DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), SDValue(UMAAL.getNode(), 1));
12380 DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), SDValue(UMAAL.getNode(), 0));
12381
12382 // Return original node to notify the driver to stop replacing.
12383 return SDValue(AddeNode, 0);
12384 }
12385 return SDValue();
12386}
12387
12388static SDValue PerformUMLALCombine(SDNode *N, SelectionDAG &DAG,
12389 const ARMSubtarget *Subtarget) {
12390 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
12391 return SDValue();
12392
12393 // Check that we have a pair of ADDC and ADDE as operands.
12394 // Both addends of the ADDE must be zero.
12395 SDNode* AddcNode = N->getOperand(2).getNode();
12396 SDNode* AddeNode = N->getOperand(3).getNode();
12397 if ((AddcNode->getOpcode() == ARMISD::ADDC) &&
12398 (AddeNode->getOpcode() == ARMISD::ADDE) &&
12399 isNullConstant(AddeNode->getOperand(0)) &&
12400 isNullConstant(AddeNode->getOperand(1)) &&
12401 (AddeNode->getOperand(2).getNode() == AddcNode))
12402 return DAG.getNode(ARMISD::UMAAL, SDLoc(N),
12403 DAG.getVTList(MVT::i32, MVT::i32),
12404 {N->getOperand(0), N->getOperand(1),
12405 AddcNode->getOperand(0), AddcNode->getOperand(1)});
12406 else
12407 return SDValue();
12408}
12409
12410static SDValue PerformAddcSubcCombine(SDNode *N,
12411 TargetLowering::DAGCombinerInfo &DCI,
12412 const ARMSubtarget *Subtarget) {
12413 SelectionDAG &DAG(DCI.DAG);
12414
12415 if (N->getOpcode() == ARMISD::SUBC) {
12416 // (SUBC (ADDE 0, 0, C), 1) -> C
12417 SDValue LHS = N->getOperand(0);
12418 SDValue RHS = N->getOperand(1);
12419 if (LHS->getOpcode() == ARMISD::ADDE &&
12420 isNullConstant(LHS->getOperand(0)) &&
12421 isNullConstant(LHS->getOperand(1)) && isOneConstant(RHS)) {
12422 return DCI.CombineTo(N, SDValue(N, 0), LHS->getOperand(2));
12423 }
12424 }
12425
12426 if (Subtarget->isThumb1Only()) {
12427 SDValue RHS = N->getOperand(1);
12428 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
12429 int32_t imm = C->getSExtValue();
12430 if (imm < 0 && imm > std::numeric_limits<int>::min()) {
12431 SDLoc DL(N);
12432 RHS = DAG.getConstant(-imm, DL, MVT::i32);
12433 unsigned Opcode = (N->getOpcode() == ARMISD::ADDC) ? ARMISD::SUBC
12434 : ARMISD::ADDC;
12435 return DAG.getNode(Opcode, DL, N->getVTList(), N->getOperand(0), RHS);
12436 }
12437 }
12438 }
12439
12440 return SDValue();
12441}
12442
12443static SDValue PerformAddeSubeCombine(SDNode *N,
12444 TargetLowering::DAGCombinerInfo &DCI,
12445 const ARMSubtarget *Subtarget) {
12446 if (Subtarget->isThumb1Only()) {
12447 SelectionDAG &DAG = DCI.DAG;
12448 SDValue RHS = N->getOperand(1);
12449 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(RHS)) {
12450 int64_t imm = C->getSExtValue();
12451 if (imm < 0) {
12452 SDLoc DL(N);
12453
12454 // The with-carry-in form matches bitwise not instead of the negation.
12455 // Effectively, the inverse interpretation of the carry flag already
12456 // accounts for part of the negation.
12457 RHS = DAG.getConstant(~imm, DL, MVT::i32);
12458
12459 unsigned Opcode = (N->getOpcode() == ARMISD::ADDE) ? ARMISD::SUBE
12460 : ARMISD::ADDE;
12461 return DAG.getNode(Opcode, DL, N->getVTList(),
12462 N->getOperand(0), RHS, N->getOperand(2));
12463 }
12464 }
12465 } else if (N->getOperand(1)->getOpcode() == ISD::SMUL_LOHI) {
12466 return AddCombineTo64bitMLAL(N, DCI, Subtarget);
12467 }
12468 return SDValue();
12469}
12470
12471static SDValue PerformSELECTCombine(SDNode *N,
12472 TargetLowering::DAGCombinerInfo &DCI,
12473 const ARMSubtarget *Subtarget) {
12474 if (!Subtarget->hasMVEIntegerOps())
12475 return SDValue();
12476
12477 SDLoc dl(N);
12478 SDValue SetCC;
12479 SDValue LHS;
12480 SDValue RHS;
12481 ISD::CondCode CC;
12482 SDValue TrueVal;
12483 SDValue FalseVal;
12484
12485 if (N->getOpcode() == ISD::SELECT &&
12486 N->getOperand(0)->getOpcode() == ISD::SETCC) {
12487 SetCC = N->getOperand(0);
12488 LHS = SetCC->getOperand(0);
12489 RHS = SetCC->getOperand(1);
12490 CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
12491 TrueVal = N->getOperand(1);
12492 FalseVal = N->getOperand(2);
12493 } else if (N->getOpcode() == ISD::SELECT_CC) {
12494 LHS = N->getOperand(0);
12495 RHS = N->getOperand(1);
12496 CC = cast<CondCodeSDNode>(N->getOperand(4))->get();
12497 TrueVal = N->getOperand(2);
12498 FalseVal = N->getOperand(3);
12499 } else {
12500 return SDValue();
12501 }
12502
12503 unsigned int Opcode = 0;
12504 if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMIN ||
12505 FalseVal->getOpcode() == ISD::VECREDUCE_UMIN) &&
12506 (CC == ISD::SETULT || CC == ISD::SETUGT)) {
12507 Opcode = ARMISD::VMINVu;
12508 if (CC == ISD::SETUGT)
12509 std::swap(TrueVal, FalseVal);
12510 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMIN ||
12511 FalseVal->getOpcode() == ISD::VECREDUCE_SMIN) &&
12512 (CC == ISD::SETLT || CC == ISD::SETGT)) {
12513 Opcode = ARMISD::VMINVs;
12514 if (CC == ISD::SETGT)
12515 std::swap(TrueVal, FalseVal);
12516 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_UMAX ||
12517 FalseVal->getOpcode() == ISD::VECREDUCE_UMAX) &&
12518 (CC == ISD::SETUGT || CC == ISD::SETULT)) {
12519 Opcode = ARMISD::VMAXVu;
12520 if (CC == ISD::SETULT)
12521 std::swap(TrueVal, FalseVal);
12522 } else if ((TrueVal->getOpcode() == ISD::VECREDUCE_SMAX ||
12523 FalseVal->getOpcode() == ISD::VECREDUCE_SMAX) &&
12524 (CC == ISD::SETGT || CC == ISD::SETLT)) {
12525 Opcode = ARMISD::VMAXVs;
12526 if (CC == ISD::SETLT)
12527 std::swap(TrueVal, FalseVal);
12528 } else
12529 return SDValue();
12530
12531 // Normalise to the right hand side being the vector reduction
12532 switch (TrueVal->getOpcode()) {
12533 case ISD::VECREDUCE_UMIN:
12534 case ISD::VECREDUCE_SMIN:
12535 case ISD::VECREDUCE_UMAX:
12536 case ISD::VECREDUCE_SMAX:
12537 std::swap(LHS, RHS);
12538 std::swap(TrueVal, FalseVal);
12539 break;
12540 }
12541
12542 EVT VectorType = FalseVal->getOperand(0).getValueType();
12543
12544 if (VectorType != MVT::v16i8 && VectorType != MVT::v8i16 &&
12545 VectorType != MVT::v4i32)
12546 return SDValue();
12547
12548 EVT VectorScalarType = VectorType.getVectorElementType();
12549
12550 // The values being selected must also be the ones being compared
12551 if (TrueVal != LHS || FalseVal != RHS)
12552 return SDValue();
12553
12554 EVT LeftType = LHS->getValueType(0);
12555 EVT RightType = RHS->getValueType(0);
12556
12557 // The types must match the reduced type too
12558 if (LeftType != VectorScalarType || RightType != VectorScalarType)
12559 return SDValue();
12560
12561 // Legalise the scalar to an i32
12562 if (VectorScalarType != MVT::i32)
12563 LHS = DCI.DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, LHS);
12564
12565 // Generate the reduction as an i32 for legalisation purposes
12566 auto Reduction =
12567 DCI.DAG.getNode(Opcode, dl, MVT::i32, LHS, RHS->getOperand(0));
12568
12569 // The result isn't actually an i32 so truncate it back to its original type
12570 if (VectorScalarType != MVT::i32)
12571 Reduction = DCI.DAG.getNode(ISD::TRUNCATE, dl, VectorScalarType, Reduction);
12572
12573 return Reduction;
12574}
12575
12576// A special combine for the vqdmulh family of instructions. This is one of the
12577// potential set of patterns that could patch this instruction. The base pattern
12578// you would expect to be min(max(ashr(mul(mul(sext(x), 2), sext(y)), 16))).
12579// This matches the different min(max(ashr(mul(mul(sext(x), sext(y)), 2), 16))),
12580// which llvm will have optimized to min(ashr(mul(sext(x), sext(y)), 15))) as
12581// the max is unnecessary.
12582static SDValue PerformVQDMULHCombine(SDNode *N, SelectionDAG &DAG) {
12583 EVT VT = N->getValueType(0);
12584 SDValue Shft;
12585 ConstantSDNode *Clamp;
12586
12587 if (N->getOpcode() == ISD::SMIN) {
12588 Shft = N->getOperand(0);
12589 Clamp = isConstOrConstSplat(N->getOperand(1));
12590 } else if (N->getOpcode() == ISD::VSELECT) {
12591 // Detect a SMIN, which for an i64 node will be a vselect/setcc, not a smin.
12592 SDValue Cmp = N->getOperand(0);
12593 if (Cmp.getOpcode() != ISD::SETCC ||
12594 cast<CondCodeSDNode>(Cmp.getOperand(2))->get() != ISD::SETLT ||
12595 Cmp.getOperand(0) != N->getOperand(1) ||
12596 Cmp.getOperand(1) != N->getOperand(2))
12597 return SDValue();
12598 Shft = N->getOperand(1);
12599 Clamp = isConstOrConstSplat(N->getOperand(2));
12600 } else
12601 return SDValue();
12602
12603 if (!Clamp)
12604 return SDValue();
12605
12606 MVT ScalarType;
12607 int ShftAmt = 0;
12608 switch (Clamp->getSExtValue()) {
12609 case (1 << 7) - 1:
12610 ScalarType = MVT::i8;
12611 ShftAmt = 7;
12612 break;
12613 case (1 << 15) - 1:
12614 ScalarType = MVT::i16;
12615 ShftAmt = 15;
12616 break;
12617 case (1ULL << 31) - 1:
12618 ScalarType = MVT::i32;
12619 ShftAmt = 31;
12620 break;
12621 default:
12622 return SDValue();
12623 }
12624
12625 if (Shft.getOpcode() != ISD::SRA)
12626 return SDValue();
12627 ConstantSDNode *N1 = isConstOrConstSplat(Shft.getOperand(1));
12628 if (!N1 || N1->getSExtValue() != ShftAmt)
12629 return SDValue();
12630
12631 SDValue Mul = Shft.getOperand(0);
12632 if (Mul.getOpcode() != ISD::MUL)
12633 return SDValue();
12634
12635 SDValue Ext0 = Mul.getOperand(0);
12636 SDValue Ext1 = Mul.getOperand(1);
12637 if (Ext0.getOpcode() != ISD::SIGN_EXTEND ||
12638 Ext1.getOpcode() != ISD::SIGN_EXTEND)
12639 return SDValue();
12640 EVT VecVT = Ext0.getOperand(0).getValueType();
12641 if (VecVT != MVT::v4i32 && VecVT != MVT::v8i16 && VecVT != MVT::v16i8)
12642 return SDValue();
12643 if (Ext1.getOperand(0).getValueType() != VecVT ||
12644 VecVT.getScalarType() != ScalarType ||
12645 VT.getScalarSizeInBits() < ScalarType.getScalarSizeInBits() * 2)
12646 return SDValue();
12647
12648 SDLoc DL(Mul);
12649 SDValue VQDMULH = DAG.getNode(ARMISD::VQDMULH, DL, VecVT, Ext0.getOperand(0),
12650 Ext1.getOperand(0));
12651 return DAG.getNode(ISD::SIGN_EXTEND, DL, VT, VQDMULH);
12652}
12653
12654static SDValue PerformVSELECTCombine(SDNode *N,
12655 TargetLowering::DAGCombinerInfo &DCI,
12656 const ARMSubtarget *Subtarget) {
12657 if (!Subtarget->hasMVEIntegerOps())
12658 return SDValue();
12659
12660 if (SDValue V = PerformVQDMULHCombine(N, DCI.DAG))
12661 return V;
12662
12663 // Transforms vselect(not(cond), lhs, rhs) into vselect(cond, rhs, lhs).
12664 //
12665 // We need to re-implement this optimization here as the implementation in the
12666 // Target-Independent DAGCombiner does not handle the kind of constant we make
12667 // (it calls isConstOrConstSplat with AllowTruncation set to false - and for
12668 // good reason, allowing truncation there would break other targets).
12669 //
12670 // Currently, this is only done for MVE, as it's the only target that benefits
12671 // from this transformation (e.g. VPNOT+VPSEL becomes a single VPSEL).
12672 if (N->getOperand(0).getOpcode() != ISD::XOR)
12673 return SDValue();
12674 SDValue XOR = N->getOperand(0);
12675
12676 // Check if the XOR's RHS is either a 1, or a BUILD_VECTOR of 1s.
12677 // It is important to check with truncation allowed as the BUILD_VECTORs we
12678 // generate in those situations will truncate their operands.
12679 ConstantSDNode *Const =
12680 isConstOrConstSplat(XOR->getOperand(1), /*AllowUndefs*/ false,
12681 /*AllowTruncation*/ true);
12682 if (!Const || !Const->isOne())
12683 return SDValue();
12684
12685 // Rewrite into vselect(cond, rhs, lhs).
12686 SDValue Cond = XOR->getOperand(0);
12687 SDValue LHS = N->getOperand(1);
12688 SDValue RHS = N->getOperand(2);
12689 EVT Type = N->getValueType(0);
12690 return DCI.DAG.getNode(ISD::VSELECT, SDLoc(N), Type, Cond, RHS, LHS);
12691}
12692
12693static SDValue PerformABSCombine(SDNode *N,
12694 TargetLowering::DAGCombinerInfo &DCI,
12695 const ARMSubtarget *Subtarget) {
12696 SDValue res;
12697 SelectionDAG &DAG = DCI.DAG;
12698 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
12699
12700 if (TLI.isOperationLegal(N->getOpcode(), N->getValueType(0)))
12701 return SDValue();
12702
12703 if (!TLI.expandABS(N, res, DAG))
12704 return SDValue();
12705
12706 return res;
12707}
12708
12709/// PerformADDECombine - Target-specific dag combine transform from
12710/// ARMISD::ADDC, ARMISD::ADDE, and ISD::MUL_LOHI to MLAL or
12711/// ARMISD::ADDC, ARMISD::ADDE and ARMISD::UMLAL to ARMISD::UMAAL
12712static SDValue PerformADDECombine(SDNode *N,
12713 TargetLowering::DAGCombinerInfo &DCI,
12714 const ARMSubtarget *Subtarget) {
12715 // Only ARM and Thumb2 support UMLAL/SMLAL.
12716 if (Subtarget->isThumb1Only())
12717 return PerformAddeSubeCombine(N, DCI, Subtarget);
12718
12719 // Only perform the checks after legalize when the pattern is available.
12720 if (DCI.isBeforeLegalize()) return SDValue();
12721
12722 return AddCombineTo64bitUMAAL(N, DCI, Subtarget);
12723}
12724
12725/// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
12726/// operands N0 and N1. This is a helper for PerformADDCombine that is
12727/// called with the default operands, and if that fails, with commuted
12728/// operands.
12729static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
12730 TargetLowering::DAGCombinerInfo &DCI,
12731 const ARMSubtarget *Subtarget){
12732 // Attempt to create vpadd for this add.
12733 if (SDValue Result = AddCombineToVPADD(N, N0, N1, DCI, Subtarget))
12734 return Result;
12735
12736 // Attempt to create vpaddl for this add.
12737 if (SDValue Result = AddCombineVUZPToVPADDL(N, N0, N1, DCI, Subtarget))
12738 return Result;
12739 if (SDValue Result = AddCombineBUILD_VECTORToVPADDL(N, N0, N1, DCI,
12740 Subtarget))
12741 return Result;
12742
12743 // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
12744 if (N0.getNode()->hasOneUse())
12745 if (SDValue Result = combineSelectAndUse(N, N0, N1, DCI))
12746 return Result;
12747 return SDValue();
12748}
12749
12750static SDValue PerformADDVecReduce(SDNode *N,
12751 TargetLowering::DAGCombinerInfo &DCI,
12752 const ARMSubtarget *Subtarget) {
12753 if (!Subtarget->hasMVEIntegerOps() || N->getValueType(0) != MVT::i64)
12754 return SDValue();
12755
12756 SDValue N0 = N->getOperand(0);
12757 SDValue N1 = N->getOperand(1);
12758
12759 // We are looking for a i64 add of a VADDLVx. Due to these being i64's, this
12760 // will look like:
12761 // t1: i32,i32 = ARMISD::VADDLVs x
12762 // t2: i64 = build_pair t1, t1:1
12763 // t3: i64 = add t2, y
12764 // We also need to check for sext / zext and commutitive adds.
12765 auto MakeVecReduce = [&](unsigned Opcode, unsigned OpcodeA, SDValue NA,
12766 SDValue NB) {
12767 if (NB->getOpcode() != ISD::BUILD_PAIR)
12768 return SDValue();
12769 SDValue VecRed = NB->getOperand(0);
12770 if (VecRed->getOpcode() != Opcode || VecRed.getResNo() != 0 ||
12771 NB->getOperand(1) != SDValue(VecRed.getNode(), 1))
12772 return SDValue();
12773
12774 SDLoc dl(N);
12775 SmallVector<SDValue, 4> Ops;
12776 Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
12777 DCI.DAG.getConstant(0, dl, MVT::i32)));
12778 Ops.push_back(DCI.DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, NA,
12779 DCI.DAG.getConstant(1, dl, MVT::i32)));
12780 for (unsigned i = 0, e = VecRed.getNumOperands(); i < e; i++)
12781 Ops.push_back(VecRed->getOperand(i));
12782 SDValue Red = DCI.DAG.getNode(OpcodeA, dl,
12783 DCI.DAG.getVTList({MVT::i32, MVT::i32}), Ops);
12784 return DCI.DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Red,
12785 SDValue(Red.getNode(), 1));
12786 };
12787
12788 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N0, N1))
12789 return M;
12790 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N0, N1))
12791 return M;
12792 if (SDValue M = MakeVecReduce(ARMISD::VADDLVs, ARMISD::VADDLVAs, N1, N0))
12793 return M;
12794 if (SDValue M = MakeVecReduce(ARMISD::VADDLVu, ARMISD::VADDLVAu, N1, N0))
12795 return M;
12796 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N0, N1))
12797 return M;
12798 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N0, N1))
12799 return M;
12800 if (SDValue M = MakeVecReduce(ARMISD::VADDLVps, ARMISD::VADDLVAps, N1, N0))
12801 return M;
12802 if (SDValue M = MakeVecReduce(ARMISD::VADDLVpu, ARMISD::VADDLVApu, N1, N0))
12803 return M;
12804 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N0, N1))
12805 return M;
12806 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N0, N1))
12807 return M;
12808 if (SDValue M = MakeVecReduce(ARMISD::VMLALVs, ARMISD::VMLALVAs, N1, N0))
12809 return M;
12810 if (SDValue M = MakeVecReduce(ARMISD::VMLALVu, ARMISD::VMLALVAu, N1, N0))
12811 return M;
12812 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N0, N1))
12813 return M;
12814 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N0, N1))
12815 return M;
12816 if (SDValue M = MakeVecReduce(ARMISD::VMLALVps, ARMISD::VMLALVAps, N1, N0))
12817 return M;
12818 if (SDValue M = MakeVecReduce(ARMISD::VMLALVpu, ARMISD::VMLALVApu, N1, N0))
12819 return M;
12820 return SDValue();
12821}
12822
12823bool
12824ARMTargetLowering::isDesirableToCommuteWithShift(const SDNode *N,
12825 CombineLevel Level) const {
12826 if (Level == BeforeLegalizeTypes)
12827 return true;
12828
12829 if (N->getOpcode() != ISD::SHL)
12830 return true;
12831
12832 if (Subtarget->isThumb1Only()) {
12833 // Avoid making expensive immediates by commuting shifts. (This logic
12834 // only applies to Thumb1 because ARM and Thumb2 immediates can be shifted
12835 // for free.)
12836 if (N->getOpcode() != ISD::SHL)
12837 return true;
12838 SDValue N1 = N->getOperand(0);
12839 if (N1->getOpcode() != ISD::ADD && N1->getOpcode() != ISD::AND &&
12840 N1->getOpcode() != ISD::OR && N1->getOpcode() != ISD::XOR)
12841 return true;
12842 if (auto *Const = dyn_cast<ConstantSDNode>(N1->getOperand(1))) {
12843 if (Const->getAPIntValue().ult(256))
12844 return false;
12845 if (N1->getOpcode() == ISD::ADD && Const->getAPIntValue().slt(0) &&
12846 Const->getAPIntValue().sgt(-256))
12847 return false;
12848 }
12849 return true;
12850 }
12851
12852 // Turn off commute-with-shift transform after legalization, so it doesn't
12853 // conflict with PerformSHLSimplify. (We could try to detect when
12854 // PerformSHLSimplify would trigger more precisely, but it isn't
12855 // really necessary.)
12856 return false;
12857}
12858
12859bool ARMTargetLowering::shouldFoldConstantShiftPairToMask(
12860 const SDNode *N, CombineLevel Level) const {
12861 if (!Subtarget->isThumb1Only())
12862 return true;
12863
12864 if (Level == BeforeLegalizeTypes)
12865 return true;
12866
12867 return false;
12868}
12869
12870bool ARMTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
12871 if (!Subtarget->hasNEON()) {
12872 if (Subtarget->isThumb1Only())
12873 return VT.getScalarSizeInBits() <= 32;
12874 return true;
12875 }
12876 return VT.isScalarInteger();
12877}
12878
12879static SDValue PerformSHLSimplify(SDNode *N,
12880 TargetLowering::DAGCombinerInfo &DCI,
12881 const ARMSubtarget *ST) {
12882 // Allow the generic combiner to identify potential bswaps.
12883 if (DCI.isBeforeLegalize())
12884 return SDValue();
12885
12886 // DAG combiner will fold:
12887 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
12888 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2
12889 // Other code patterns that can be also be modified have the following form:
12890 // b + ((a << 1) | 510)
12891 // b + ((a << 1) & 510)
12892 // b + ((a << 1) ^ 510)
12893 // b + ((a << 1) + 510)
12894
12895 // Many instructions can perform the shift for free, but it requires both
12896 // the operands to be registers. If c1 << c2 is too large, a mov immediate
12897 // instruction will needed. So, unfold back to the original pattern if:
12898 // - if c1 and c2 are small enough that they don't require mov imms.
12899 // - the user(s) of the node can perform an shl
12900
12901 // No shifted operands for 16-bit instructions.
12902 if (ST->isThumb() && ST->isThumb1Only())
12903 return SDValue();
12904
12905 // Check that all the users could perform the shl themselves.
12906 for (auto U : N->uses()) {
12907 switch(U->getOpcode()) {
12908 default:
12909 return SDValue();
12910 case ISD::SUB:
12911 case ISD::ADD:
12912 case ISD::AND:
12913 case ISD::OR:
12914 case ISD::XOR:
12915 case ISD::SETCC:
12916 case ARMISD::CMP:
12917 // Check that the user isn't already using a constant because there
12918 // aren't any instructions that support an immediate operand and a
12919 // shifted operand.
12920 if (isa<ConstantSDNode>(U->getOperand(0)) ||
12921 isa<ConstantSDNode>(U->getOperand(1)))
12922 return SDValue();
12923
12924 // Check that it's not already using a shift.
12925 if (U->getOperand(0).getOpcode() == ISD::SHL ||
12926 U->getOperand(1).getOpcode() == ISD::SHL)
12927 return SDValue();
12928 break;
12929 }
12930 }
12931
12932 if (N->getOpcode() != ISD::ADD && N->getOpcode() != ISD::OR &&
12933 N->getOpcode() != ISD::XOR && N->getOpcode() != ISD::AND)
12934 return SDValue();
12935
12936 if (N->getOperand(0).getOpcode() != ISD::SHL)
12937 return SDValue();
12938
12939 SDValue SHL = N->getOperand(0);
12940
12941 auto *C1ShlC2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
12942 auto *C2 = dyn_cast<ConstantSDNode>(SHL.getOperand(1));
12943 if (!C1ShlC2 || !C2)
12944 return SDValue();
12945
12946 APInt C2Int = C2->getAPIntValue();
12947 APInt C1Int = C1ShlC2->getAPIntValue();
12948
12949 // Check that performing a lshr will not lose any information.
12950 APInt Mask = APInt::getHighBitsSet(C2Int.getBitWidth(),
12951 C2Int.getBitWidth() - C2->getZExtValue());
12952 if ((C1Int & Mask) != C1Int)
12953 return SDValue();
12954
12955 // Shift the first constant.
12956 C1Int.lshrInPlace(C2Int);
12957
12958 // The immediates are encoded as an 8-bit value that can be rotated.
12959 auto LargeImm = [](const APInt &Imm) {
12960 unsigned Zeros = Imm.countLeadingZeros() + Imm.countTrailingZeros();
12961 return Imm.getBitWidth() - Zeros > 8;
12962 };
12963
12964 if (LargeImm(C1Int) || LargeImm(C2Int))
12965 return SDValue();
12966
12967 SelectionDAG &DAG = DCI.DAG;
12968 SDLoc dl(N);
12969 SDValue X = SHL.getOperand(0);
12970 SDValue BinOp = DAG.getNode(N->getOpcode(), dl, MVT::i32, X,
12971 DAG.getConstant(C1Int, dl, MVT::i32));
12972 // Shift left to compensate for the lshr of C1Int.
12973 SDValue Res = DAG.getNode(ISD::SHL, dl, MVT::i32, BinOp, SHL.getOperand(1));
12974
12975 LLVM_DEBUG(dbgs() << "Simplify shl use:\n"; SHL.getOperand(0).dump();do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { dbgs() << "Simplify shl use:\n"; SHL.getOperand
(0).dump(); SHL.dump(); N->dump(); } } while (false)
12976 SHL.dump(); N->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { dbgs() << "Simplify shl use:\n"; SHL.getOperand
(0).dump(); SHL.dump(); N->dump(); } } while (false)
;
12977 LLVM_DEBUG(dbgs() << "Into:\n"; X.dump(); BinOp.dump(); Res.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { dbgs() << "Into:\n"; X.dump(); BinOp.dump
(); Res.dump(); } } while (false)
;
12978 return Res;
12979}
12980
12981
12982/// PerformADDCombine - Target-specific dag combine xforms for ISD::ADD.
12983///
12984static SDValue PerformADDCombine(SDNode *N,
12985 TargetLowering::DAGCombinerInfo &DCI,
12986 const ARMSubtarget *Subtarget) {
12987 SDValue N0 = N->getOperand(0);
12988 SDValue N1 = N->getOperand(1);
12989
12990 // Only works one way, because it needs an immediate operand.
12991 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
12992 return Result;
12993
12994 if (SDValue Result = PerformADDVecReduce(N, DCI, Subtarget))
12995 return Result;
12996
12997 // First try with the default operand order.
12998 if (SDValue Result = PerformADDCombineWithOperands(N, N0, N1, DCI, Subtarget))
12999 return Result;
13000
13001 // If that didn't work, try again with the operands commuted.
13002 return PerformADDCombineWithOperands(N, N1, N0, DCI, Subtarget);
13003}
13004
13005// Combine (sub 0, (csinc X, Y, CC)) -> (csinv -X, Y, CC)
13006// providing -X is as cheap as X (currently, just a constant).
13007static SDValue PerformSubCSINCCombine(SDNode *N,
13008 TargetLowering::DAGCombinerInfo &DCI) {
13009 if (N->getValueType(0) != MVT::i32 || !isNullConstant(N->getOperand(0)))
13010 return SDValue();
13011 SDValue CSINC = N->getOperand(1);
13012 if (CSINC.getOpcode() != ARMISD::CSINC || !CSINC.hasOneUse())
13013 return SDValue();
13014
13015 ConstantSDNode *X = dyn_cast<ConstantSDNode>(CSINC.getOperand(0));
13016 if (!X)
13017 return SDValue();
13018
13019 return DCI.DAG.getNode(ARMISD::CSINV, SDLoc(N), MVT::i32,
13020 DCI.DAG.getNode(ISD::SUB, SDLoc(N), MVT::i32,
13021 N->getOperand(0), CSINC.getOperand(0)),
13022 CSINC.getOperand(1), CSINC.getOperand(2),
13023 CSINC.getOperand(3));
13024}
13025
13026/// PerformSUBCombine - Target-specific dag combine xforms for ISD::SUB.
13027///
13028static SDValue PerformSUBCombine(SDNode *N,
13029 TargetLowering::DAGCombinerInfo &DCI,
13030 const ARMSubtarget *Subtarget) {
13031 SDValue N0 = N->getOperand(0);
13032 SDValue N1 = N->getOperand(1);
13033
13034 // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
13035 if (N1.getNode()->hasOneUse())
13036 if (SDValue Result = combineSelectAndUse(N, N1, N0, DCI))
13037 return Result;
13038
13039 if (SDValue R = PerformSubCSINCCombine(N, DCI))
13040 return R;
13041
13042 if (!Subtarget->hasMVEIntegerOps() || !N->getValueType(0).isVector())
13043 return SDValue();
13044
13045 // Fold (sub (ARMvmovImm 0), (ARMvdup x)) -> (ARMvdup (sub 0, x))
13046 // so that we can readily pattern match more mve instructions which can use
13047 // a scalar operand.
13048 SDValue VDup = N->getOperand(1);
13049 if (VDup->getOpcode() != ARMISD::VDUP)
13050 return SDValue();
13051
13052 SDValue VMov = N->getOperand(0);
13053 if (VMov->getOpcode() == ISD::BITCAST)
13054 VMov = VMov->getOperand(0);
13055
13056 if (VMov->getOpcode() != ARMISD::VMOVIMM || !isZeroVector(VMov))
13057 return SDValue();
13058
13059 SDLoc dl(N);
13060 SDValue Negate = DCI.DAG.getNode(ISD::SUB, dl, MVT::i32,
13061 DCI.DAG.getConstant(0, dl, MVT::i32),
13062 VDup->getOperand(0));
13063 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0), Negate);
13064}
13065
13066/// PerformVMULCombine
13067/// Distribute (A + B) * C to (A * C) + (B * C) to take advantage of the
13068/// special multiplier accumulator forwarding.
13069/// vmul d3, d0, d2
13070/// vmla d3, d1, d2
13071/// is faster than
13072/// vadd d3, d0, d1
13073/// vmul d3, d3, d2
13074// However, for (A + B) * (A + B),
13075// vadd d2, d0, d1
13076// vmul d3, d0, d2
13077// vmla d3, d1, d2
13078// is slower than
13079// vadd d2, d0, d1
13080// vmul d3, d2, d2
13081static SDValue PerformVMULCombine(SDNode *N,
13082 TargetLowering::DAGCombinerInfo &DCI,
13083 const ARMSubtarget *Subtarget) {
13084 if (!Subtarget->hasVMLxForwarding())
13085 return SDValue();
13086
13087 SelectionDAG &DAG = DCI.DAG;
13088 SDValue N0 = N->getOperand(0);
13089 SDValue N1 = N->getOperand(1);
13090 unsigned Opcode = N0.getOpcode();
13091 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13092 Opcode != ISD::FADD && Opcode != ISD::FSUB) {
13093 Opcode = N1.getOpcode();
13094 if (Opcode != ISD::ADD && Opcode != ISD::SUB &&
13095 Opcode != ISD::FADD && Opcode != ISD::FSUB)
13096 return SDValue();
13097 std::swap(N0, N1);
13098 }
13099
13100 if (N0 == N1)
13101 return SDValue();
13102
13103 EVT VT = N->getValueType(0);
13104 SDLoc DL(N);
13105 SDValue N00 = N0->getOperand(0);
13106 SDValue N01 = N0->getOperand(1);
13107 return DAG.getNode(Opcode, DL, VT,
13108 DAG.getNode(ISD::MUL, DL, VT, N00, N1),
13109 DAG.getNode(ISD::MUL, DL, VT, N01, N1));
13110}
13111
13112static SDValue PerformMVEVMULLCombine(SDNode *N, SelectionDAG &DAG,
13113 const ARMSubtarget *Subtarget) {
13114 EVT VT = N->getValueType(0);
13115 if (VT != MVT::v2i64)
13116 return SDValue();
13117
13118 SDValue N0 = N->getOperand(0);
13119 SDValue N1 = N->getOperand(1);
13120
13121 auto IsSignExt = [&](SDValue Op) {
13122 if (Op->getOpcode() != ISD::SIGN_EXTEND_INREG)
13123 return SDValue();
13124 EVT VT = cast<VTSDNode>(Op->getOperand(1))->getVT();
13125 if (VT.getScalarSizeInBits() == 32)
13126 return Op->getOperand(0);
13127 return SDValue();
13128 };
13129 auto IsZeroExt = [&](SDValue Op) {
13130 // Zero extends are a little more awkward. At the point we are matching
13131 // this, we are looking for an AND with a (-1, 0, -1, 0) buildvector mask.
13132 // That might be before of after a bitcast depending on how the and is
13133 // placed. Because this has to look through bitcasts, it is currently only
13134 // supported on LE.
13135 if (!Subtarget->isLittle())
13136 return SDValue();
13137
13138 SDValue And = Op;
13139 if (And->getOpcode() == ISD::BITCAST)
13140 And = And->getOperand(0);
13141 if (And->getOpcode() != ISD::AND)
13142 return SDValue();
13143 SDValue Mask = And->getOperand(1);
13144 if (Mask->getOpcode() == ISD::BITCAST)
13145 Mask = Mask->getOperand(0);
13146
13147 if (Mask->getOpcode() != ISD::BUILD_VECTOR ||
13148 Mask.getValueType() != MVT::v4i32)
13149 return SDValue();
13150 if (isAllOnesConstant(Mask->getOperand(0)) &&
13151 isNullConstant(Mask->getOperand(1)) &&
13152 isAllOnesConstant(Mask->getOperand(2)) &&
13153 isNullConstant(Mask->getOperand(3)))
13154 return And->getOperand(0);
13155 return SDValue();
13156 };
13157
13158 SDLoc dl(N);
13159 if (SDValue Op0 = IsSignExt(N0)) {
13160 if (SDValue Op1 = IsSignExt(N1)) {
13161 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
13162 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
13163 return DAG.getNode(ARMISD::VMULLs, dl, VT, New0a, New1a);
13164 }
13165 }
13166 if (SDValue Op0 = IsZeroExt(N0)) {
13167 if (SDValue Op1 = IsZeroExt(N1)) {
13168 SDValue New0a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op0);
13169 SDValue New1a = DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v4i32, Op1);
13170 return DAG.getNode(ARMISD::VMULLu, dl, VT, New0a, New1a);
13171 }
13172 }
13173
13174 return SDValue();
13175}
13176
13177static SDValue PerformMULCombine(SDNode *N,
13178 TargetLowering::DAGCombinerInfo &DCI,
13179 const ARMSubtarget *Subtarget) {
13180 SelectionDAG &DAG = DCI.DAG;
13181
13182 EVT VT = N->getValueType(0);
13183 if (Subtarget->hasMVEIntegerOps() && VT == MVT::v2i64)
13184 return PerformMVEVMULLCombine(N, DAG, Subtarget);
13185
13186 if (Subtarget->isThumb1Only())
13187 return SDValue();
13188
13189 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
13190 return SDValue();
13191
13192 if (VT.is64BitVector() || VT.is128BitVector())
13193 return PerformVMULCombine(N, DCI, Subtarget);
13194 if (VT != MVT::i32)
13195 return SDValue();
13196
13197 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
13198 if (!C)
13199 return SDValue();
13200
13201 int64_t MulAmt = C->getSExtValue();
13202 unsigned ShiftAmt = countTrailingZeros<uint64_t>(MulAmt);
13203
13204 ShiftAmt = ShiftAmt & (32 - 1);
13205 SDValue V = N->getOperand(0);
13206 SDLoc DL(N);
13207
13208 SDValue Res;
13209 MulAmt >>= ShiftAmt;
13210
13211 if (MulAmt >= 0) {
13212 if (isPowerOf2_32(MulAmt - 1)) {
13213 // (mul x, 2^N + 1) => (add (shl x, N), x)
13214 Res = DAG.getNode(ISD::ADD, DL, VT,
13215 V,
13216 DAG.getNode(ISD::SHL, DL, VT,
13217 V,
13218 DAG.getConstant(Log2_32(MulAmt - 1), DL,
13219 MVT::i32)));
13220 } else if (isPowerOf2_32(MulAmt + 1)) {
13221 // (mul x, 2^N - 1) => (sub (shl x, N), x)
13222 Res = DAG.getNode(ISD::SUB, DL, VT,
13223 DAG.getNode(ISD::SHL, DL, VT,
13224 V,
13225 DAG.getConstant(Log2_32(MulAmt + 1), DL,
13226 MVT::i32)),
13227 V);
13228 } else
13229 return SDValue();
13230 } else {
13231 uint64_t MulAmtAbs = -MulAmt;
13232 if (isPowerOf2_32(MulAmtAbs + 1)) {
13233 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
13234 Res = DAG.getNode(ISD::SUB, DL, VT,
13235 V,
13236 DAG.getNode(ISD::SHL, DL, VT,
13237 V,
13238 DAG.getConstant(Log2_32(MulAmtAbs + 1), DL,
13239 MVT::i32)));
13240 } else if (isPowerOf2_32(MulAmtAbs - 1)) {
13241 // (mul x, -(2^N + 1)) => - (add (shl x, N), x)
13242 Res = DAG.getNode(ISD::ADD, DL, VT,
13243 V,
13244 DAG.getNode(ISD::SHL, DL, VT,
13245 V,
13246 DAG.getConstant(Log2_32(MulAmtAbs - 1), DL,
13247 MVT::i32)));
13248 Res = DAG.getNode(ISD::SUB, DL, VT,
13249 DAG.getConstant(0, DL, MVT::i32), Res);
13250 } else
13251 return SDValue();
13252 }
13253
13254 if (ShiftAmt != 0)
13255 Res = DAG.getNode(ISD::SHL, DL, VT,
13256 Res, DAG.getConstant(ShiftAmt, DL, MVT::i32));
13257
13258 // Do not add new nodes to DAG combiner worklist.
13259 DCI.CombineTo(N, Res, false);
13260 return SDValue();
13261}
13262
13263static SDValue CombineANDShift(SDNode *N,
13264 TargetLowering::DAGCombinerInfo &DCI,
13265 const ARMSubtarget *Subtarget) {
13266 // Allow DAGCombine to pattern-match before we touch the canonical form.
13267 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
13268 return SDValue();
13269
13270 if (N->getValueType(0) != MVT::i32)
13271 return SDValue();
13272
13273 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
13274 if (!N1C)
13275 return SDValue();
13276
13277 uint32_t C1 = (uint32_t)N1C->getZExtValue();
13278 // Don't transform uxtb/uxth.
13279 if (C1 == 255 || C1 == 65535)
13280 return SDValue();
13281
13282 SDNode *N0 = N->getOperand(0).getNode();
13283 if (!N0->hasOneUse())
13284 return SDValue();
13285
13286 if (N0->getOpcode() != ISD::SHL && N0->getOpcode() != ISD::SRL)
13287 return SDValue();
13288
13289 bool LeftShift = N0->getOpcode() == ISD::SHL;
13290
13291 ConstantSDNode *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
13292 if (!N01C)
13293 return SDValue();
13294
13295 uint32_t C2 = (uint32_t)N01C->getZExtValue();
13296 if (!C2 || C2 >= 32)
13297 return SDValue();
13298
13299 // Clear irrelevant bits in the mask.
13300 if (LeftShift)
13301 C1 &= (-1U << C2);
13302 else
13303 C1 &= (-1U >> C2);
13304
13305 SelectionDAG &DAG = DCI.DAG;
13306 SDLoc DL(N);
13307
13308 // We have a pattern of the form "(and (shl x, c2) c1)" or
13309 // "(and (srl x, c2) c1)", where c1 is a shifted mask. Try to
13310 // transform to a pair of shifts, to save materializing c1.
13311
13312 // First pattern: right shift, then mask off leading bits.
13313 // FIXME: Use demanded bits?
13314 if (!LeftShift && isMask_32(C1)) {
13315 uint32_t C3 = countLeadingZeros(C1);
13316 if (C2 < C3) {
13317 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
13318 DAG.getConstant(C3 - C2, DL, MVT::i32));
13319 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
13320 DAG.getConstant(C3, DL, MVT::i32));
13321 }
13322 }
13323
13324 // First pattern, reversed: left shift, then mask off trailing bits.
13325 if (LeftShift && isMask_32(~C1)) {
13326 uint32_t C3 = countTrailingZeros(C1);
13327 if (C2 < C3) {
13328 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
13329 DAG.getConstant(C3 - C2, DL, MVT::i32));
13330 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
13331 DAG.getConstant(C3, DL, MVT::i32));
13332 }
13333 }
13334
13335 // Second pattern: left shift, then mask off leading bits.
13336 // FIXME: Use demanded bits?
13337 if (LeftShift && isShiftedMask_32(C1)) {
13338 uint32_t Trailing = countTrailingZeros(C1);
13339 uint32_t C3 = countLeadingZeros(C1);
13340 if (Trailing == C2 && C2 + C3 < 32) {
13341 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
13342 DAG.getConstant(C2 + C3, DL, MVT::i32));
13343 return DAG.getNode(ISD::SRL, DL, MVT::i32, SHL,
13344 DAG.getConstant(C3, DL, MVT::i32));
13345 }
13346 }
13347
13348 // Second pattern, reversed: right shift, then mask off trailing bits.
13349 // FIXME: Handle other patterns of known/demanded bits.
13350 if (!LeftShift && isShiftedMask_32(C1)) {
13351 uint32_t Leading = countLeadingZeros(C1);
13352 uint32_t C3 = countTrailingZeros(C1);
13353 if (Leading == C2 && C2 + C3 < 32) {
13354 SDValue SHL = DAG.getNode(ISD::SRL, DL, MVT::i32, N0->getOperand(0),
13355 DAG.getConstant(C2 + C3, DL, MVT::i32));
13356 return DAG.getNode(ISD::SHL, DL, MVT::i32, SHL,
13357 DAG.getConstant(C3, DL, MVT::i32));
13358 }
13359 }
13360
13361 // FIXME: Transform "(and (shl x, c2) c1)" ->
13362 // "(shl (and x, c1>>c2), c2)" if "c1 >> c2" is a cheaper immediate than
13363 // c1.
13364 return SDValue();
13365}
13366
13367static SDValue PerformANDCombine(SDNode *N,
13368 TargetLowering::DAGCombinerInfo &DCI,
13369 const ARMSubtarget *Subtarget) {
13370 // Attempt to use immediate-form VBIC
13371 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
13372 SDLoc dl(N);
13373 EVT VT = N->getValueType(0);
13374 SelectionDAG &DAG = DCI.DAG;
13375
13376 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) || VT == MVT::v4i1 ||
13377 VT == MVT::v8i1 || VT == MVT::v16i1)
13378 return SDValue();
13379
13380 APInt SplatBits, SplatUndef;
13381 unsigned SplatBitSize;
13382 bool HasAnyUndefs;
13383 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
13384 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
13385 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
13386 SplatBitSize == 64) {
13387 EVT VbicVT;
13388 SDValue Val = isVMOVModifiedImm((~SplatBits).getZExtValue(),
13389 SplatUndef.getZExtValue(), SplatBitSize,
13390 DAG, dl, VbicVT, VT, OtherModImm);
13391 if (Val.getNode()) {
13392 SDValue Input =
13393 DAG.getNode(ISD::BITCAST, dl, VbicVT, N->getOperand(0));
13394 SDValue Vbic = DAG.getNode(ARMISD::VBICIMM, dl, VbicVT, Input, Val);
13395 return DAG.getNode(ISD::BITCAST, dl, VT, Vbic);
13396 }
13397 }
13398 }
13399
13400 if (!Subtarget->isThumb1Only()) {
13401 // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
13402 if (SDValue Result = combineSelectAndUseCommutative(N, true, DCI))
13403 return Result;
13404
13405 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13406 return Result;
13407 }
13408
13409 if (Subtarget->isThumb1Only())
13410 if (SDValue Result = CombineANDShift(N, DCI, Subtarget))
13411 return Result;
13412
13413 return SDValue();
13414}
13415
13416// Try combining OR nodes to SMULWB, SMULWT.
13417static SDValue PerformORCombineToSMULWBT(SDNode *OR,
13418 TargetLowering::DAGCombinerInfo &DCI,
13419 const ARMSubtarget *Subtarget) {
13420 if (!Subtarget->hasV6Ops() ||
13421 (Subtarget->isThumb() &&
13422 (!Subtarget->hasThumb2() || !Subtarget->hasDSP())))
13423 return SDValue();
13424
13425 SDValue SRL = OR->getOperand(0);
13426 SDValue SHL = OR->getOperand(1);
13427
13428 if (SRL.getOpcode() != ISD::SRL || SHL.getOpcode() != ISD::SHL) {
13429 SRL = OR->getOperand(1);
13430 SHL = OR->getOperand(0);
13431 }
13432 if (!isSRL16(SRL) || !isSHL16(SHL))
13433 return SDValue();
13434
13435 // The first operands to the shifts need to be the two results from the
13436 // same smul_lohi node.
13437 if ((SRL.getOperand(0).getNode() != SHL.getOperand(0).getNode()) ||
13438 SRL.getOperand(0).getOpcode() != ISD::SMUL_LOHI)
13439 return SDValue();
13440
13441 SDNode *SMULLOHI = SRL.getOperand(0).getNode();
13442 if (SRL.getOperand(0) != SDValue(SMULLOHI, 0) ||
13443 SHL.getOperand(0) != SDValue(SMULLOHI, 1))
13444 return SDValue();
13445
13446 // Now we have:
13447 // (or (srl (smul_lohi ?, ?), 16), (shl (smul_lohi ?, ?), 16)))
13448 // For SMUL[B|T] smul_lohi will take a 32-bit and a 16-bit arguments.
13449 // For SMUWB the 16-bit value will signed extended somehow.
13450 // For SMULWT only the SRA is required.
13451 // Check both sides of SMUL_LOHI
13452 SDValue OpS16 = SMULLOHI->getOperand(0);
13453 SDValue OpS32 = SMULLOHI->getOperand(1);
13454
13455 SelectionDAG &DAG = DCI.DAG;
13456 if (!isS16(OpS16, DAG) && !isSRA16(OpS16)) {
13457 OpS16 = OpS32;
13458 OpS32 = SMULLOHI->getOperand(0);
13459 }
13460
13461 SDLoc dl(OR);
13462 unsigned Opcode = 0;
13463 if (isS16(OpS16, DAG))
13464 Opcode = ARMISD::SMULWB;
13465 else if (isSRA16(OpS16)) {
13466 Opcode = ARMISD::SMULWT;
13467 OpS16 = OpS16->getOperand(0);
13468 }
13469 else
13470 return SDValue();
13471
13472 SDValue Res = DAG.getNode(Opcode, dl, MVT::i32, OpS32, OpS16);
13473 DAG.ReplaceAllUsesOfValueWith(SDValue(OR, 0), Res);
13474 return SDValue(OR, 0);
13475}
13476
13477static SDValue PerformORCombineToBFI(SDNode *N,
13478 TargetLowering::DAGCombinerInfo &DCI,
13479 const ARMSubtarget *Subtarget) {
13480 // BFI is only available on V6T2+
13481 if (Subtarget->isThumb1Only() || !Subtarget->hasV6T2Ops())
13482 return SDValue();
13483
13484 EVT VT = N->getValueType(0);
13485 SDValue N0 = N->getOperand(0);
13486 SDValue N1 = N->getOperand(1);
13487 SelectionDAG &DAG = DCI.DAG;
13488 SDLoc DL(N);
13489 // 1) or (and A, mask), val => ARMbfi A, val, mask
13490 // iff (val & mask) == val
13491 //
13492 // 2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
13493 // 2a) iff isBitFieldInvertedMask(mask) && isBitFieldInvertedMask(~mask2)
13494 // && mask == ~mask2
13495 // 2b) iff isBitFieldInvertedMask(~mask) && isBitFieldInvertedMask(mask2)
13496 // && ~mask == mask2
13497 // (i.e., copy a bitfield value into another bitfield of the same width)
13498
13499 if (VT != MVT::i32)
13500 return SDValue();
13501
13502 SDValue N00 = N0.getOperand(0);
13503
13504 // The value and the mask need to be constants so we can verify this is
13505 // actually a bitfield set. If the mask is 0xffff, we can do better
13506 // via a movt instruction, so don't use BFI in that case.
13507 SDValue MaskOp = N0.getOperand(1);
13508 ConstantSDNode *MaskC = dyn_cast<ConstantSDNode>(MaskOp);
13509 if (!MaskC)
13510 return SDValue();
13511 unsigned Mask = MaskC->getZExtValue();
13512 if (Mask == 0xffff)
13513 return SDValue();
13514 SDValue Res;
13515 // Case (1): or (and A, mask), val => ARMbfi A, val, mask
13516 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
13517 if (N1C) {
13518 unsigned Val = N1C->getZExtValue();
13519 if ((Val & ~Mask) != Val)
13520 return SDValue();
13521
13522 if (ARM::isBitFieldInvertedMask(Mask)) {
13523 Val >>= countTrailingZeros(~Mask);
13524
13525 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00,
13526 DAG.getConstant(Val, DL, MVT::i32),
13527 DAG.getConstant(Mask, DL, MVT::i32));
13528
13529 DCI.CombineTo(N, Res, false);
13530 // Return value from the original node to inform the combiner than N is
13531 // now dead.
13532 return SDValue(N, 0);
13533 }
13534 } else if (N1.getOpcode() == ISD::AND) {
13535 // case (2) or (and A, mask), (and B, mask2) => ARMbfi A, (lsr B, amt), mask
13536 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
13537 if (!N11C)
13538 return SDValue();
13539 unsigned Mask2 = N11C->getZExtValue();
13540
13541 // Mask and ~Mask2 (or reverse) must be equivalent for the BFI pattern
13542 // as is to match.
13543 if (ARM::isBitFieldInvertedMask(Mask) &&
13544 (Mask == ~Mask2)) {
13545 // The pack halfword instruction works better for masks that fit it,
13546 // so use that when it's available.
13547 if (Subtarget->hasDSP() &&
13548 (Mask == 0xffff || Mask == 0xffff0000))
13549 return SDValue();
13550 // 2a
13551 unsigned amt = countTrailingZeros(Mask2);
13552 Res = DAG.getNode(ISD::SRL, DL, VT, N1.getOperand(0),
13553 DAG.getConstant(amt, DL, MVT::i32));
13554 Res = DAG.getNode(ARMISD::BFI, DL, VT, N00, Res,
13555 DAG.getConstant(Mask, DL, MVT::i32));
13556 DCI.CombineTo(N, Res, false);
13557 // Return value from the original node to inform the combiner than N is
13558 // now dead.
13559 return SDValue(N, 0);
13560 } else if (ARM::isBitFieldInvertedMask(~Mask) &&
13561 (~Mask == Mask2)) {
13562 // The pack halfword instruction works better for masks that fit it,
13563 // so use that when it's available.
13564 if (Subtarget->hasDSP() &&
13565 (Mask2 == 0xffff || Mask2 == 0xffff0000))
13566 return SDValue();
13567 // 2b
13568 unsigned lsb = countTrailingZeros(Mask);
13569 Res = DAG.getNode(ISD::SRL, DL, VT, N00,
13570 DAG.getConstant(lsb, DL, MVT::i32));
13571 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1.getOperand(0), Res,
13572 DAG.getConstant(Mask2, DL, MVT::i32));
13573 DCI.CombineTo(N, Res, false);
13574 // Return value from the original node to inform the combiner than N is
13575 // now dead.
13576 return SDValue(N, 0);
13577 }
13578 }
13579
13580 if (DAG.MaskedValueIsZero(N1, MaskC->getAPIntValue()) &&
13581 N00.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N00.getOperand(1)) &&
13582 ARM::isBitFieldInvertedMask(~Mask)) {
13583 // Case (3): or (and (shl A, #shamt), mask), B => ARMbfi B, A, ~mask
13584 // where lsb(mask) == #shamt and masked bits of B are known zero.
13585 SDValue ShAmt = N00.getOperand(1);
13586 unsigned ShAmtC = cast<ConstantSDNode>(ShAmt)->getZExtValue();
13587 unsigned LSB = countTrailingZeros(Mask);
13588 if (ShAmtC != LSB)
13589 return SDValue();
13590
13591 Res = DAG.getNode(ARMISD::BFI, DL, VT, N1, N00.getOperand(0),
13592 DAG.getConstant(~Mask, DL, MVT::i32));
13593
13594 DCI.CombineTo(N, Res, false);
13595 // Return value from the original node to inform the combiner than N is
13596 // now dead.
13597 return SDValue(N, 0);
13598 }
13599
13600 return SDValue();
13601}
13602
13603static bool isValidMVECond(unsigned CC, bool IsFloat) {
13604 switch (CC) {
13605 case ARMCC::EQ:
13606 case ARMCC::NE:
13607 case ARMCC::LE:
13608 case ARMCC::GT:
13609 case ARMCC::GE:
13610 case ARMCC::LT:
13611 return true;
13612 case ARMCC::HS:
13613 case ARMCC::HI:
13614 return !IsFloat;
13615 default:
13616 return false;
13617 };
13618}
13619
13620static ARMCC::CondCodes getVCMPCondCode(SDValue N) {
13621 if (N->getOpcode() == ARMISD::VCMP)
13622 return (ARMCC::CondCodes)N->getConstantOperandVal(2);
13623 else if (N->getOpcode() == ARMISD::VCMPZ)
13624 return (ARMCC::CondCodes)N->getConstantOperandVal(1);
13625 else
13626 llvm_unreachable("Not a VCMP/VCMPZ!")::llvm::llvm_unreachable_internal("Not a VCMP/VCMPZ!", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13626)
;
13627}
13628
13629static bool CanInvertMVEVCMP(SDValue N) {
13630 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(getVCMPCondCode(N));
13631 return isValidMVECond(CC, N->getOperand(0).getValueType().isFloatingPoint());
13632}
13633
13634static SDValue PerformORCombine_i1(SDNode *N,
13635 TargetLowering::DAGCombinerInfo &DCI,
13636 const ARMSubtarget *Subtarget) {
13637 // Try to invert "or A, B" -> "and ~A, ~B", as the "and" is easier to chain
13638 // together with predicates
13639 EVT VT = N->getValueType(0);
13640 SDLoc DL(N);
13641 SDValue N0 = N->getOperand(0);
13642 SDValue N1 = N->getOperand(1);
13643
13644 auto IsFreelyInvertable = [&](SDValue V) {
13645 if (V->getOpcode() == ARMISD::VCMP || V->getOpcode() == ARMISD::VCMPZ)
13646 return CanInvertMVEVCMP(V);
13647 return false;
13648 };
13649
13650 // At least one operand must be freely invertable.
13651 if (!(IsFreelyInvertable(N0) || IsFreelyInvertable(N1)))
13652 return SDValue();
13653
13654 SDValue NewN0 = DCI.DAG.getLogicalNOT(DL, N0, VT);
13655 SDValue NewN1 = DCI.DAG.getLogicalNOT(DL, N1, VT);
13656 SDValue And = DCI.DAG.getNode(ISD::AND, DL, VT, NewN0, NewN1);
13657 return DCI.DAG.getLogicalNOT(DL, And, VT);
13658}
13659
13660/// PerformORCombine - Target-specific dag combine xforms for ISD::OR
13661static SDValue PerformORCombine(SDNode *N,
13662 TargetLowering::DAGCombinerInfo &DCI,
13663 const ARMSubtarget *Subtarget) {
13664 // Attempt to use immediate-form VORR
13665 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(N->getOperand(1));
13666 SDLoc dl(N);
13667 EVT VT = N->getValueType(0);
13668 SelectionDAG &DAG = DCI.DAG;
13669
13670 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
13671 return SDValue();
13672
13673 if (Subtarget->hasMVEIntegerOps() &&
13674 (VT == MVT::v4i1 || VT == MVT::v8i1 || VT == MVT::v16i1))
13675 return PerformORCombine_i1(N, DCI, Subtarget);
13676
13677 APInt SplatBits, SplatUndef;
13678 unsigned SplatBitSize;
13679 bool HasAnyUndefs;
13680 if (BVN && (Subtarget->hasNEON() || Subtarget->hasMVEIntegerOps()) &&
13681 BVN->isConstantSplat(SplatBits, SplatUndef, SplatBitSize, HasAnyUndefs)) {
13682 if (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
13683 SplatBitSize == 64) {
13684 EVT VorrVT;
13685 SDValue Val =
13686 isVMOVModifiedImm(SplatBits.getZExtValue(), SplatUndef.getZExtValue(),
13687 SplatBitSize, DAG, dl, VorrVT, VT, OtherModImm);
13688 if (Val.getNode()) {
13689 SDValue Input =
13690 DAG.getNode(ISD::BITCAST, dl, VorrVT, N->getOperand(0));
13691 SDValue Vorr = DAG.getNode(ARMISD::VORRIMM, dl, VorrVT, Input, Val);
13692 return DAG.getNode(ISD::BITCAST, dl, VT, Vorr);
13693 }
13694 }
13695 }
13696
13697 if (!Subtarget->isThumb1Only()) {
13698 // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
13699 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
13700 return Result;
13701 if (SDValue Result = PerformORCombineToSMULWBT(N, DCI, Subtarget))
13702 return Result;
13703 }
13704
13705 SDValue N0 = N->getOperand(0);
13706 SDValue N1 = N->getOperand(1);
13707
13708 // (or (and B, A), (and C, ~A)) => (VBSL A, B, C) when A is a constant.
13709 if (Subtarget->hasNEON() && N1.getOpcode() == ISD::AND && VT.isVector() &&
13710 DAG.getTargetLoweringInfo().isTypeLegal(VT)) {
13711
13712 // The code below optimizes (or (and X, Y), Z).
13713 // The AND operand needs to have a single user to make these optimizations
13714 // profitable.
13715 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
13716 return SDValue();
13717
13718 APInt SplatUndef;
13719 unsigned SplatBitSize;
13720 bool HasAnyUndefs;
13721
13722 APInt SplatBits0, SplatBits1;
13723 BuildVectorSDNode *BVN0 = dyn_cast<BuildVectorSDNode>(N0->getOperand(1));
13724 BuildVectorSDNode *BVN1 = dyn_cast<BuildVectorSDNode>(N1->getOperand(1));
13725 // Ensure that the second operand of both ands are constants
13726 if (BVN0 && BVN0->isConstantSplat(SplatBits0, SplatUndef, SplatBitSize,
13727 HasAnyUndefs) && !HasAnyUndefs) {
13728 if (BVN1 && BVN1->isConstantSplat(SplatBits1, SplatUndef, SplatBitSize,
13729 HasAnyUndefs) && !HasAnyUndefs) {
13730 // Ensure that the bit width of the constants are the same and that
13731 // the splat arguments are logical inverses as per the pattern we
13732 // are trying to simplify.
13733 if (SplatBits0.getBitWidth() == SplatBits1.getBitWidth() &&
13734 SplatBits0 == ~SplatBits1) {
13735 // Canonicalize the vector type to make instruction selection
13736 // simpler.
13737 EVT CanonicalVT = VT.is128BitVector() ? MVT::v4i32 : MVT::v2i32;
13738 SDValue Result = DAG.getNode(ARMISD::VBSP, dl, CanonicalVT,
13739 N0->getOperand(1),
13740 N0->getOperand(0),
13741 N1->getOperand(0));
13742 return DAG.getNode(ISD::BITCAST, dl, VT, Result);
13743 }
13744 }
13745 }
13746 }
13747
13748 // Try to use the ARM/Thumb2 BFI (bitfield insert) instruction when
13749 // reasonable.
13750 if (N0.getOpcode() == ISD::AND && N0.hasOneUse()) {
13751 if (SDValue Res = PerformORCombineToBFI(N, DCI, Subtarget))
13752 return Res;
13753 }
13754
13755 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13756 return Result;
13757
13758 return SDValue();
13759}
13760
13761static SDValue PerformXORCombine(SDNode *N,
13762 TargetLowering::DAGCombinerInfo &DCI,
13763 const ARMSubtarget *Subtarget) {
13764 EVT VT = N->getValueType(0);
13765 SelectionDAG &DAG = DCI.DAG;
13766
13767 if(!DAG.getTargetLoweringInfo().isTypeLegal(VT))
13768 return SDValue();
13769
13770 if (!Subtarget->isThumb1Only()) {
13771 // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
13772 if (SDValue Result = combineSelectAndUseCommutative(N, false, DCI))
13773 return Result;
13774
13775 if (SDValue Result = PerformSHLSimplify(N, DCI, Subtarget))
13776 return Result;
13777 }
13778
13779 if (Subtarget->hasMVEIntegerOps()) {
13780 // fold (xor(vcmp/z, 1)) into a vcmp with the opposite condition.
13781 SDValue N0 = N->getOperand(0);
13782 SDValue N1 = N->getOperand(1);
13783 const TargetLowering *TLI = Subtarget->getTargetLowering();
13784 if (TLI->isConstTrueVal(N1.getNode()) &&
13785 (N0->getOpcode() == ARMISD::VCMP || N0->getOpcode() == ARMISD::VCMPZ)) {
13786 if (CanInvertMVEVCMP(N0)) {
13787 SDLoc DL(N0);
13788 ARMCC::CondCodes CC = ARMCC::getOppositeCondition(getVCMPCondCode(N0));
13789
13790 SmallVector<SDValue, 4> Ops;
13791 Ops.push_back(N0->getOperand(0));
13792 if (N0->getOpcode() == ARMISD::VCMP)
13793 Ops.push_back(N0->getOperand(1));
13794 Ops.push_back(DCI.DAG.getConstant(CC, DL, MVT::i32));
13795 return DCI.DAG.getNode(N0->getOpcode(), DL, N0->getValueType(0), Ops);
13796 }
13797 }
13798 }
13799
13800 return SDValue();
13801}
13802
13803// ParseBFI - given a BFI instruction in N, extract the "from" value (Rn) and return it,
13804// and fill in FromMask and ToMask with (consecutive) bits in "from" to be extracted and
13805// their position in "to" (Rd).
13806static SDValue ParseBFI(SDNode *N, APInt &ToMask, APInt &FromMask) {
13807 assert(N->getOpcode() == ARMISD::BFI)(static_cast <bool> (N->getOpcode() == ARMISD::BFI) ?
void (0) : __assert_fail ("N->getOpcode() == ARMISD::BFI"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13807, __extension__ __PRETTY_FUNCTION__))
;
13808
13809 SDValue From = N->getOperand(1);
13810 ToMask = ~cast<ConstantSDNode>(N->getOperand(2))->getAPIntValue();
13811 FromMask = APInt::getLowBitsSet(ToMask.getBitWidth(), ToMask.countPopulation());
13812
13813 // If the Base came from a SHR #C, we can deduce that it is really testing bit
13814 // #C in the base of the SHR.
13815 if (From->getOpcode() == ISD::SRL &&
13816 isa<ConstantSDNode>(From->getOperand(1))) {
13817 APInt Shift = cast<ConstantSDNode>(From->getOperand(1))->getAPIntValue();
13818 assert(Shift.getLimitedValue() < 32 && "Shift too large!")(static_cast <bool> (Shift.getLimitedValue() < 32 &&
"Shift too large!") ? void (0) : __assert_fail ("Shift.getLimitedValue() < 32 && \"Shift too large!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13818, __extension__ __PRETTY_FUNCTION__))
;
13819 FromMask <<= Shift.getLimitedValue(31);
13820 From = From->getOperand(0);
13821 }
13822
13823 return From;
13824}
13825
13826// If A and B contain one contiguous set of bits, does A | B == A . B?
13827//
13828// Neither A nor B must be zero.
13829static bool BitsProperlyConcatenate(const APInt &A, const APInt &B) {
13830 unsigned LastActiveBitInA = A.countTrailingZeros();
13831 unsigned FirstActiveBitInB = B.getBitWidth() - B.countLeadingZeros() - 1;
13832 return LastActiveBitInA - 1 == FirstActiveBitInB;
13833}
13834
13835static SDValue FindBFIToCombineWith(SDNode *N) {
13836 // We have a BFI in N. Follow a possible chain of BFIs and find a BFI it can combine with,
13837 // if one exists.
13838 APInt ToMask, FromMask;
13839 SDValue From = ParseBFI(N, ToMask, FromMask);
13840 SDValue To = N->getOperand(0);
13841
13842 // Now check for a compatible BFI to merge with. We can pass through BFIs that
13843 // aren't compatible, but not if they set the same bit in their destination as
13844 // we do (or that of any BFI we're going to combine with).
13845 SDValue V = To;
13846 APInt CombinedToMask = ToMask;
13847 while (V.getOpcode() == ARMISD::BFI) {
13848 APInt NewToMask, NewFromMask;
13849 SDValue NewFrom = ParseBFI(V.getNode(), NewToMask, NewFromMask);
13850 if (NewFrom != From) {
13851 // This BFI has a different base. Keep going.
13852 CombinedToMask |= NewToMask;
13853 V = V.getOperand(0);
13854 continue;
13855 }
13856
13857 // Do the written bits conflict with any we've seen so far?
13858 if ((NewToMask & CombinedToMask).getBoolValue())
13859 // Conflicting bits - bail out because going further is unsafe.
13860 return SDValue();
13861
13862 // Are the new bits contiguous when combined with the old bits?
13863 if (BitsProperlyConcatenate(ToMask, NewToMask) &&
13864 BitsProperlyConcatenate(FromMask, NewFromMask))
13865 return V;
13866 if (BitsProperlyConcatenate(NewToMask, ToMask) &&
13867 BitsProperlyConcatenate(NewFromMask, FromMask))
13868 return V;
13869
13870 // We've seen a write to some bits, so track it.
13871 CombinedToMask |= NewToMask;
13872 // Keep going...
13873 V = V.getOperand(0);
13874 }
13875
13876 return SDValue();
13877}
13878
13879static SDValue PerformBFICombine(SDNode *N,
13880 TargetLowering::DAGCombinerInfo &DCI) {
13881 SDValue N1 = N->getOperand(1);
13882 if (N1.getOpcode() == ISD::AND) {
13883 // (bfi A, (and B, Mask1), Mask2) -> (bfi A, B, Mask2) iff
13884 // the bits being cleared by the AND are not demanded by the BFI.
13885 ConstantSDNode *N11C = dyn_cast<ConstantSDNode>(N1.getOperand(1));
13886 if (!N11C)
13887 return SDValue();
13888 unsigned InvMask = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
13889 unsigned LSB = countTrailingZeros(~InvMask);
13890 unsigned Width = (32 - countLeadingZeros(~InvMask)) - LSB;
13891 assert(Width <(static_cast <bool> (Width < static_cast<unsigned
>(std::numeric_limits<unsigned>::digits) && "undefined behavior"
) ? void (0) : __assert_fail ("Width < static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && \"undefined behavior\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13893, __extension__ __PRETTY_FUNCTION__))
13892 static_cast<unsigned>(std::numeric_limits<unsigned>::digits) &&(static_cast <bool> (Width < static_cast<unsigned
>(std::numeric_limits<unsigned>::digits) && "undefined behavior"
) ? void (0) : __assert_fail ("Width < static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && \"undefined behavior\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13893, __extension__ __PRETTY_FUNCTION__))
13893 "undefined behavior")(static_cast <bool> (Width < static_cast<unsigned
>(std::numeric_limits<unsigned>::digits) && "undefined behavior"
) ? void (0) : __assert_fail ("Width < static_cast<unsigned>(std::numeric_limits<unsigned>::digits) && \"undefined behavior\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13893, __extension__ __PRETTY_FUNCTION__))
;
13894 unsigned Mask = (1u << Width) - 1;
13895 unsigned Mask2 = N11C->getZExtValue();
13896 if ((Mask & (~Mask2)) == 0)
13897 return DCI.DAG.getNode(ARMISD::BFI, SDLoc(N), N->getValueType(0),
13898 N->getOperand(0), N1.getOperand(0),
13899 N->getOperand(2));
13900 } else if (N->getOperand(0).getOpcode() == ARMISD::BFI) {
13901 // We have a BFI of a BFI. Walk up the BFI chain to see how long it goes.
13902 // Keep track of any consecutive bits set that all come from the same base
13903 // value. We can combine these together into a single BFI.
13904 SDValue CombineBFI = FindBFIToCombineWith(N);
13905 if (CombineBFI == SDValue())
13906 return SDValue();
13907
13908 // We've found a BFI.
13909 APInt ToMask1, FromMask1;
13910 SDValue From1 = ParseBFI(N, ToMask1, FromMask1);
13911
13912 APInt ToMask2, FromMask2;
13913 SDValue From2 = ParseBFI(CombineBFI.getNode(), ToMask2, FromMask2);
13914 assert(From1 == From2)(static_cast <bool> (From1 == From2) ? void (0) : __assert_fail
("From1 == From2", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 13914, __extension__ __PRETTY_FUNCTION__))
;
13915 (void)From2;
13916
13917 // First, unlink CombineBFI.
13918 DCI.DAG.ReplaceAllUsesWith(CombineBFI, CombineBFI.getOperand(0));
13919 // Then create a new BFI, combining the two together.
13920 APInt NewFromMask = FromMask1 | FromMask2;
13921 APInt NewToMask = ToMask1 | ToMask2;
13922
13923 EVT VT = N->getValueType(0);
13924 SDLoc dl(N);
13925
13926 if (NewFromMask[0] == 0)
13927 From1 = DCI.DAG.getNode(
13928 ISD::SRL, dl, VT, From1,
13929 DCI.DAG.getConstant(NewFromMask.countTrailingZeros(), dl, VT));
13930 return DCI.DAG.getNode(ARMISD::BFI, dl, VT, N->getOperand(0), From1,
13931 DCI.DAG.getConstant(~NewToMask, dl, VT));
13932 }
13933 return SDValue();
13934}
13935
13936/// PerformVMOVRRDCombine - Target-specific dag combine xforms for
13937/// ARMISD::VMOVRRD.
13938static SDValue PerformVMOVRRDCombine(SDNode *N,
13939 TargetLowering::DAGCombinerInfo &DCI,
13940 const ARMSubtarget *Subtarget) {
13941 // vmovrrd(vmovdrr x, y) -> x,y
13942 SDValue InDouble = N->getOperand(0);
13943 if (InDouble.getOpcode() == ARMISD::VMOVDRR && Subtarget->hasFP64())
13944 return DCI.CombineTo(N, InDouble.getOperand(0), InDouble.getOperand(1));
13945
13946 // vmovrrd(load f64) -> (load i32), (load i32)
13947 SDNode *InNode = InDouble.getNode();
13948 if (ISD::isNormalLoad(InNode) && InNode->hasOneUse() &&
13949 InNode->getValueType(0) == MVT::f64 &&
13950 InNode->getOperand(1).getOpcode() == ISD::FrameIndex &&
13951 !cast<LoadSDNode>(InNode)->isVolatile()) {
13952 // TODO: Should this be done for non-FrameIndex operands?
13953 LoadSDNode *LD = cast<LoadSDNode>(InNode);
13954
13955 SelectionDAG &DAG = DCI.DAG;
13956 SDLoc DL(LD);
13957 SDValue BasePtr = LD->getBasePtr();
13958 SDValue NewLD1 =
13959 DAG.getLoad(MVT::i32, DL, LD->getChain(), BasePtr, LD->getPointerInfo(),
13960 LD->getAlignment(), LD->getMemOperand()->getFlags());
13961
13962 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
13963 DAG.getConstant(4, DL, MVT::i32));
13964
13965 SDValue NewLD2 = DAG.getLoad(MVT::i32, DL, LD->getChain(), OffsetPtr,
13966 LD->getPointerInfo().getWithOffset(4),
13967 std::min(4U, LD->getAlignment()),
13968 LD->getMemOperand()->getFlags());
13969
13970 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewLD2.getValue(1));
13971 if (DCI.DAG.getDataLayout().isBigEndian())
13972 std::swap (NewLD1, NewLD2);
13973 SDValue Result = DCI.CombineTo(N, NewLD1, NewLD2);
13974 return Result;
13975 }
13976
13977 // VMOVRRD(extract(..(build_vector(a, b, c, d)))) -> a,b or c,d
13978 // VMOVRRD(extract(insert_vector(insert_vector(.., a, l1), b, l2))) -> a,b
13979 if (InDouble.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
13980 isa<ConstantSDNode>(InDouble.getOperand(1))) {
13981 SDValue BV = InDouble.getOperand(0);
13982 // Look up through any nop bitcasts and vector_reg_casts. bitcasts may
13983 // change lane order under big endian.
13984 bool BVSwap = BV.getOpcode() == ISD::BITCAST;
13985 while (
13986 (BV.getOpcode() == ISD::BITCAST ||
13987 BV.getOpcode() == ARMISD::VECTOR_REG_CAST) &&
13988 (BV.getValueType() == MVT::v2f64 || BV.getValueType() == MVT::v2i64)) {
13989 BVSwap = BV.getOpcode() == ISD::BITCAST;
13990 BV = BV.getOperand(0);
13991 }
13992 if (BV.getValueType() != MVT::v4i32)
13993 return SDValue();
13994
13995 // Handle buildvectors, pulling out the correct lane depending on
13996 // endianness.
13997 unsigned Offset = InDouble.getConstantOperandVal(1) == 1 ? 2 : 0;
13998 if (BV.getOpcode() == ISD::BUILD_VECTOR) {
13999 SDValue Op0 = BV.getOperand(Offset);
14000 SDValue Op1 = BV.getOperand(Offset + 1);
14001 if (!Subtarget->isLittle() && BVSwap)
14002 std::swap(Op0, Op1);
14003
14004 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
14005 }
14006
14007 // A chain of insert_vectors, grabbing the correct value of the chain of
14008 // inserts.
14009 SDValue Op0, Op1;
14010 while (BV.getOpcode() == ISD::INSERT_VECTOR_ELT) {
14011 if (isa<ConstantSDNode>(BV.getOperand(2))) {
14012 if (BV.getConstantOperandVal(2) == Offset)
14013 Op0 = BV.getOperand(1);
14014 if (BV.getConstantOperandVal(2) == Offset + 1)
14015 Op1 = BV.getOperand(1);
14016 }
14017 BV = BV.getOperand(0);
14018 }
14019 if (!Subtarget->isLittle() && BVSwap)
14020 std::swap(Op0, Op1);
14021 if (Op0 && Op1)
14022 return DCI.DAG.getMergeValues({Op0, Op1}, SDLoc(N));
14023 }
14024
14025 return SDValue();
14026}
14027
14028/// PerformVMOVDRRCombine - Target-specific dag combine xforms for
14029/// ARMISD::VMOVDRR. This is also used for BUILD_VECTORs with 2 operands.
14030static SDValue PerformVMOVDRRCombine(SDNode *N, SelectionDAG &DAG) {
14031 // N=vmovrrd(X); vmovdrr(N:0, N:1) -> bit_convert(X)
14032 SDValue Op0 = N->getOperand(0);
14033 SDValue Op1 = N->getOperand(1);
14034 if (Op0.getOpcode() == ISD::BITCAST)
14035 Op0 = Op0.getOperand(0);
14036 if (Op1.getOpcode() == ISD::BITCAST)
14037 Op1 = Op1.getOperand(0);
14038 if (Op0.getOpcode() == ARMISD::VMOVRRD &&
14039 Op0.getNode() == Op1.getNode() &&
14040 Op0.getResNo() == 0 && Op1.getResNo() == 1)
14041 return DAG.getNode(ISD::BITCAST, SDLoc(N),
14042 N->getValueType(0), Op0.getOperand(0));
14043 return SDValue();
14044}
14045
14046static SDValue PerformVMOVhrCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14047 SDValue Op0 = N->getOperand(0);
14048
14049 // VMOVhr (VMOVrh (X)) -> X
14050 if (Op0->getOpcode() == ARMISD::VMOVrh)
14051 return Op0->getOperand(0);
14052
14053 // FullFP16: half values are passed in S-registers, and we don't
14054 // need any of the bitcast and moves:
14055 //
14056 // t2: f32,ch = CopyFromReg t0, Register:f32 %0
14057 // t5: i32 = bitcast t2
14058 // t18: f16 = ARMISD::VMOVhr t5
14059 if (Op0->getOpcode() == ISD::BITCAST) {
14060 SDValue Copy = Op0->getOperand(0);
14061 if (Copy.getValueType() == MVT::f32 &&
14062 Copy->getOpcode() == ISD::CopyFromReg) {
14063 SDValue Ops[] = {Copy->getOperand(0), Copy->getOperand(1)};
14064 SDValue NewCopy =
14065 DCI.DAG.getNode(ISD::CopyFromReg, SDLoc(N), N->getValueType(0), Ops);
14066 return NewCopy;
14067 }
14068 }
14069
14070 // fold (VMOVhr (load x)) -> (load (f16*)x)
14071 if (LoadSDNode *LN0 = dyn_cast<LoadSDNode>(Op0)) {
14072 if (LN0->hasOneUse() && LN0->isUnindexed() &&
14073 LN0->getMemoryVT() == MVT::i16) {
14074 SDValue Load =
14075 DCI.DAG.getLoad(N->getValueType(0), SDLoc(N), LN0->getChain(),
14076 LN0->getBasePtr(), LN0->getMemOperand());
14077 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
14078 DCI.DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Load.getValue(1));
14079 return Load;
14080 }
14081 }
14082
14083 // Only the bottom 16 bits of the source register are used.
14084 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
14085 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
14086 if (TLI.SimplifyDemandedBits(Op0, DemandedMask, DCI))
14087 return SDValue(N, 0);
14088
14089 return SDValue();
14090}
14091
14092static SDValue PerformVMOVrhCombine(SDNode *N,
14093 TargetLowering::DAGCombinerInfo &DCI) {
14094 SDValue N0 = N->getOperand(0);
14095 EVT VT = N->getValueType(0);
14096
14097 // fold (VMOVrh (fpconst x)) -> const x
14098 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(N0)) {
14099 APFloat V = C->getValueAPF();
14100 return DCI.DAG.getConstant(V.bitcastToAPInt().getZExtValue(), SDLoc(N), VT);
14101 }
14102
14103 // fold (VMOVrh (load x)) -> (zextload (i16*)x)
14104 if (ISD::isNormalLoad(N0.getNode()) && N0.hasOneUse()) {
14105 LoadSDNode *LN0 = cast<LoadSDNode>(N0);
14106
14107 SDValue Load =
14108 DCI.DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N), VT, LN0->getChain(),
14109 LN0->getBasePtr(), MVT::i16, LN0->getMemOperand());
14110 DCI.DAG.ReplaceAllUsesOfValueWith(SDValue(N, 0), Load.getValue(0));
14111 DCI.DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1));
14112 return Load;
14113 }
14114
14115 // Fold VMOVrh(extract(x, n)) -> vgetlaneu(x, n)
14116 if (N0->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
14117 isa<ConstantSDNode>(N0->getOperand(1)))
14118 return DCI.DAG.getNode(ARMISD::VGETLANEu, SDLoc(N), VT, N0->getOperand(0),
14119 N0->getOperand(1));
14120
14121 return SDValue();
14122}
14123
14124/// hasNormalLoadOperand - Check if any of the operands of a BUILD_VECTOR node
14125/// are normal, non-volatile loads. If so, it is profitable to bitcast an
14126/// i64 vector to have f64 elements, since the value can then be loaded
14127/// directly into a VFP register.
14128static bool hasNormalLoadOperand(SDNode *N) {
14129 unsigned NumElts = N->getValueType(0).getVectorNumElements();
14130 for (unsigned i = 0; i < NumElts; ++i) {
14131 SDNode *Elt = N->getOperand(i).getNode();
14132 if (ISD::isNormalLoad(Elt) && !cast<LoadSDNode>(Elt)->isVolatile())
14133 return true;
14134 }
14135 return false;
14136}
14137
14138/// PerformBUILD_VECTORCombine - Target-specific dag combine xforms for
14139/// ISD::BUILD_VECTOR.
14140static SDValue PerformBUILD_VECTORCombine(SDNode *N,
14141 TargetLowering::DAGCombinerInfo &DCI,
14142 const ARMSubtarget *Subtarget) {
14143 // build_vector(N=ARMISD::VMOVRRD(X), N:1) -> bit_convert(X):
14144 // VMOVRRD is introduced when legalizing i64 types. It forces the i64 value
14145 // into a pair of GPRs, which is fine when the value is used as a scalar,
14146 // but if the i64 value is converted to a vector, we need to undo the VMOVRRD.
14147 SelectionDAG &DAG = DCI.DAG;
14148 if (N->getNumOperands() == 2)
14149 if (SDValue RV = PerformVMOVDRRCombine(N, DAG))
14150 return RV;
14151
14152 // Load i64 elements as f64 values so that type legalization does not split
14153 // them up into i32 values.
14154 EVT VT = N->getValueType(0);
14155 if (VT.getVectorElementType() != MVT::i64 || !hasNormalLoadOperand(N))
14156 return SDValue();
14157 SDLoc dl(N);
14158 SmallVector<SDValue, 8> Ops;
14159 unsigned NumElts = VT.getVectorNumElements();
14160 for (unsigned i = 0; i < NumElts; ++i) {
14161 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(i));
14162 Ops.push_back(V);
14163 // Make the DAGCombiner fold the bitcast.
14164 DCI.AddToWorklist(V.getNode());
14165 }
14166 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, NumElts);
14167 SDValue BV = DAG.getBuildVector(FloatVT, dl, Ops);
14168 return DAG.getNode(ISD::BITCAST, dl, VT, BV);
14169}
14170
14171/// Target-specific dag combine xforms for ARMISD::BUILD_VECTOR.
14172static SDValue
14173PerformARMBUILD_VECTORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14174 // ARMISD::BUILD_VECTOR is introduced when legalizing ISD::BUILD_VECTOR.
14175 // At that time, we may have inserted bitcasts from integer to float.
14176 // If these bitcasts have survived DAGCombine, change the lowering of this
14177 // BUILD_VECTOR in something more vector friendly, i.e., that does not
14178 // force to use floating point types.
14179
14180 // Make sure we can change the type of the vector.
14181 // This is possible iff:
14182 // 1. The vector is only used in a bitcast to a integer type. I.e.,
14183 // 1.1. Vector is used only once.
14184 // 1.2. Use is a bit convert to an integer type.
14185 // 2. The size of its operands are 32-bits (64-bits are not legal).
14186 EVT VT = N->getValueType(0);
14187 EVT EltVT = VT.getVectorElementType();
14188
14189 // Check 1.1. and 2.
14190 if (EltVT.getSizeInBits() != 32 || !N->hasOneUse())
14191 return SDValue();
14192
14193 // By construction, the input type must be float.
14194 assert(EltVT == MVT::f32 && "Unexpected type!")(static_cast <bool> (EltVT == MVT::f32 && "Unexpected type!"
) ? void (0) : __assert_fail ("EltVT == MVT::f32 && \"Unexpected type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14194, __extension__ __PRETTY_FUNCTION__))
;
14195
14196 // Check 1.2.
14197 SDNode *Use = *N->use_begin();
14198 if (Use->getOpcode() != ISD::BITCAST ||
14199 Use->getValueType(0).isFloatingPoint())
14200 return SDValue();
14201
14202 // Check profitability.
14203 // Model is, if more than half of the relevant operands are bitcast from
14204 // i32, turn the build_vector into a sequence of insert_vector_elt.
14205 // Relevant operands are everything that is not statically
14206 // (i.e., at compile time) bitcasted.
14207 unsigned NumOfBitCastedElts = 0;
14208 unsigned NumElts = VT.getVectorNumElements();
14209 unsigned NumOfRelevantElts = NumElts;
14210 for (unsigned Idx = 0; Idx < NumElts; ++Idx) {
14211 SDValue Elt = N->getOperand(Idx);
14212 if (Elt->getOpcode() == ISD::BITCAST) {
14213 // Assume only bit cast to i32 will go away.
14214 if (Elt->getOperand(0).getValueType() == MVT::i32)
14215 ++NumOfBitCastedElts;
14216 } else if (Elt.isUndef() || isa<ConstantSDNode>(Elt))
14217 // Constants are statically casted, thus do not count them as
14218 // relevant operands.
14219 --NumOfRelevantElts;
14220 }
14221
14222 // Check if more than half of the elements require a non-free bitcast.
14223 if (NumOfBitCastedElts <= NumOfRelevantElts / 2)
14224 return SDValue();
14225
14226 SelectionDAG &DAG = DCI.DAG;
14227 // Create the new vector type.
14228 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
14229 // Check if the type is legal.
14230 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14231 if (!TLI.isTypeLegal(VecVT))
14232 return SDValue();
14233
14234 // Combine:
14235 // ARMISD::BUILD_VECTOR E1, E2, ..., EN.
14236 // => BITCAST INSERT_VECTOR_ELT
14237 // (INSERT_VECTOR_ELT (...), (BITCAST EN-1), N-1),
14238 // (BITCAST EN), N.
14239 SDValue Vec = DAG.getUNDEF(VecVT);
14240 SDLoc dl(N);
14241 for (unsigned Idx = 0 ; Idx < NumElts; ++Idx) {
14242 SDValue V = N->getOperand(Idx);
14243 if (V.isUndef())
14244 continue;
14245 if (V.getOpcode() == ISD::BITCAST &&
14246 V->getOperand(0).getValueType() == MVT::i32)
14247 // Fold obvious case.
14248 V = V.getOperand(0);
14249 else {
14250 V = DAG.getNode(ISD::BITCAST, SDLoc(V), MVT::i32, V);
14251 // Make the DAGCombiner fold the bitcasts.
14252 DCI.AddToWorklist(V.getNode());
14253 }
14254 SDValue LaneIdx = DAG.getConstant(Idx, dl, MVT::i32);
14255 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecVT, Vec, V, LaneIdx);
14256 }
14257 Vec = DAG.getNode(ISD::BITCAST, dl, VT, Vec);
14258 // Make the DAGCombiner fold the bitcasts.
14259 DCI.AddToWorklist(Vec.getNode());
14260 return Vec;
14261}
14262
14263static SDValue
14264PerformPREDICATE_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14265 EVT VT = N->getValueType(0);
14266 SDValue Op = N->getOperand(0);
14267 SDLoc dl(N);
14268
14269 // PREDICATE_CAST(PREDICATE_CAST(x)) == PREDICATE_CAST(x)
14270 if (Op->getOpcode() == ARMISD::PREDICATE_CAST) {
14271 // If the valuetypes are the same, we can remove the cast entirely.
14272 if (Op->getOperand(0).getValueType() == VT)
14273 return Op->getOperand(0);
14274 return DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
14275 }
14276
14277 // Turn pred_cast(xor x, -1) into xor(pred_cast x, -1), in order to produce
14278 // more VPNOT which might get folded as else predicates.
14279 if (Op.getValueType() == MVT::i32 && isBitwiseNot(Op)) {
14280 SDValue X =
14281 DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT, Op->getOperand(0));
14282 SDValue C = DCI.DAG.getNode(ARMISD::PREDICATE_CAST, dl, VT,
14283 DCI.DAG.getConstant(65535, dl, MVT::i32));
14284 return DCI.DAG.getNode(ISD::XOR, dl, VT, X, C);
14285 }
14286
14287 // Only the bottom 16 bits of the source register are used.
14288 if (Op.getValueType() == MVT::i32) {
14289 APInt DemandedMask = APInt::getLowBitsSet(32, 16);
14290 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
14291 if (TLI.SimplifyDemandedBits(Op, DemandedMask, DCI))
14292 return SDValue(N, 0);
14293 }
14294 return SDValue();
14295}
14296
14297static SDValue
14298PerformVECTOR_REG_CASTCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
14299 const ARMSubtarget *ST) {
14300 EVT VT = N->getValueType(0);
14301 SDValue Op = N->getOperand(0);
14302 SDLoc dl(N);
14303
14304 // Under Little endian, a VECTOR_REG_CAST is equivalent to a BITCAST
14305 if (ST->isLittle())
14306 return DCI.DAG.getNode(ISD::BITCAST, dl, VT, Op);
14307
14308 // VECTOR_REG_CAST undef -> undef
14309 if (Op.isUndef())
14310 return DCI.DAG.getUNDEF(VT);
14311
14312 // VECTOR_REG_CAST(VECTOR_REG_CAST(x)) == VECTOR_REG_CAST(x)
14313 if (Op->getOpcode() == ARMISD::VECTOR_REG_CAST) {
14314 // If the valuetypes are the same, we can remove the cast entirely.
14315 if (Op->getOperand(0).getValueType() == VT)
14316 return Op->getOperand(0);
14317 return DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, VT, Op->getOperand(0));
14318 }
14319
14320 return SDValue();
14321}
14322
14323static SDValue PerformVCMPCombine(SDNode *N,
14324 TargetLowering::DAGCombinerInfo &DCI,
14325 const ARMSubtarget *Subtarget) {
14326 if (!Subtarget->hasMVEIntegerOps())
14327 return SDValue();
14328
14329 EVT VT = N->getValueType(0);
14330 SDValue Op0 = N->getOperand(0);
14331 SDValue Op1 = N->getOperand(1);
14332 ARMCC::CondCodes Cond =
14333 (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
14334 SDLoc dl(N);
14335
14336 // vcmp X, 0, cc -> vcmpz X, cc
14337 if (isZeroVector(Op1))
14338 return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op0,
14339 N->getOperand(2));
14340
14341 unsigned SwappedCond = getSwappedCondition(Cond);
14342 if (isValidMVECond(SwappedCond, VT.isFloatingPoint())) {
14343 // vcmp 0, X, cc -> vcmpz X, reversed(cc)
14344 if (isZeroVector(Op0))
14345 return DCI.DAG.getNode(ARMISD::VCMPZ, dl, VT, Op1,
14346 DCI.DAG.getConstant(SwappedCond, dl, MVT::i32));
14347 // vcmp vdup(Y), X, cc -> vcmp X, vdup(Y), reversed(cc)
14348 if (Op0->getOpcode() == ARMISD::VDUP && Op1->getOpcode() != ARMISD::VDUP)
14349 return DCI.DAG.getNode(ARMISD::VCMP, dl, VT, Op1, Op0,
14350 DCI.DAG.getConstant(SwappedCond, dl, MVT::i32));
14351 }
14352
14353 return SDValue();
14354}
14355
14356/// PerformInsertEltCombine - Target-specific dag combine xforms for
14357/// ISD::INSERT_VECTOR_ELT.
14358static SDValue PerformInsertEltCombine(SDNode *N,
14359 TargetLowering::DAGCombinerInfo &DCI) {
14360 // Bitcast an i64 load inserted into a vector to f64.
14361 // Otherwise, the i64 value will be legalized to a pair of i32 values.
14362 EVT VT = N->getValueType(0);
14363 SDNode *Elt = N->getOperand(1).getNode();
14364 if (VT.getVectorElementType() != MVT::i64 ||
14365 !ISD::isNormalLoad(Elt) || cast<LoadSDNode>(Elt)->isVolatile())
14366 return SDValue();
14367
14368 SelectionDAG &DAG = DCI.DAG;
14369 SDLoc dl(N);
14370 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
14371 VT.getVectorNumElements());
14372 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, N->getOperand(0));
14373 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::f64, N->getOperand(1));
14374 // Make the DAGCombiner fold the bitcasts.
14375 DCI.AddToWorklist(Vec.getNode());
14376 DCI.AddToWorklist(V.getNode());
14377 SDValue InsElt = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, FloatVT,
14378 Vec, V, N->getOperand(2));
14379 return DAG.getNode(ISD::BITCAST, dl, VT, InsElt);
14380}
14381
14382// Convert a pair of extracts from the same base vector to a VMOVRRD. Either
14383// directly or bitcast to an integer if the original is a float vector.
14384// extract(x, n); extract(x, n+1) -> VMOVRRD(extract v2f64 x, n/2)
14385// bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD(extract x, n/2)
14386static SDValue
14387PerformExtractEltToVMOVRRD(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14388 EVT VT = N->getValueType(0);
14389 SDLoc dl(N);
14390
14391 if (!DCI.isAfterLegalizeDAG() || VT != MVT::i32 ||
14392 !DCI.DAG.getTargetLoweringInfo().isTypeLegal(MVT::f64))
14393 return SDValue();
14394
14395 SDValue Ext = SDValue(N, 0);
14396 if (Ext.getOpcode() == ISD::BITCAST &&
14397 Ext.getOperand(0).getValueType() == MVT::f32)
14398 Ext = Ext.getOperand(0);
14399 if (Ext.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
14400 !isa<ConstantSDNode>(Ext.getOperand(1)) ||
14401 Ext.getConstantOperandVal(1) % 2 != 0)
14402 return SDValue();
14403 if (Ext->use_size() == 1 &&
14404 (Ext->use_begin()->getOpcode() == ISD::SINT_TO_FP ||
14405 Ext->use_begin()->getOpcode() == ISD::UINT_TO_FP))
14406 return SDValue();
14407
14408 SDValue Op0 = Ext.getOperand(0);
14409 EVT VecVT = Op0.getValueType();
14410 unsigned Lane = Ext.getConstantOperandVal(1);
14411 if (VecVT.getVectorNumElements() != 4)
14412 return SDValue();
14413
14414 // Find another extract, of Lane + 1
14415 auto OtherIt = find_if(Op0->uses(), [&](SDNode *V) {
14416 return V->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
14417 isa<ConstantSDNode>(V->getOperand(1)) &&
14418 V->getConstantOperandVal(1) == Lane + 1;
14419 });
14420 if (OtherIt == Op0->uses().end())
14421 return SDValue();
14422
14423 // For float extracts, we need to be converting to a i32 for both vector
14424 // lanes.
14425 SDValue OtherExt(*OtherIt, 0);
14426 if (OtherExt.getValueType() != MVT::i32) {
14427 if (OtherExt->use_size() != 1 ||
14428 OtherExt->use_begin()->getOpcode() != ISD::BITCAST ||
14429 OtherExt->use_begin()->getValueType(0) != MVT::i32)
14430 return SDValue();
14431 OtherExt = SDValue(*OtherExt->use_begin(), 0);
14432 }
14433
14434 // Convert the type to a f64 and extract with a VMOVRRD.
14435 SDValue F64 = DCI.DAG.getNode(
14436 ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
14437 DCI.DAG.getNode(ARMISD::VECTOR_REG_CAST, dl, MVT::v2f64, Op0),
14438 DCI.DAG.getConstant(Ext.getConstantOperandVal(1) / 2, dl, MVT::i32));
14439 SDValue VMOVRRD =
14440 DCI.DAG.getNode(ARMISD::VMOVRRD, dl, {MVT::i32, MVT::i32}, F64);
14441
14442 DCI.CombineTo(OtherExt.getNode(), SDValue(VMOVRRD.getNode(), 1));
14443 return VMOVRRD;
14444}
14445
14446static SDValue PerformExtractEltCombine(SDNode *N,
14447 TargetLowering::DAGCombinerInfo &DCI,
14448 const ARMSubtarget *ST) {
14449 SDValue Op0 = N->getOperand(0);
14450 EVT VT = N->getValueType(0);
14451 SDLoc dl(N);
14452
14453 // extract (vdup x) -> x
14454 if (Op0->getOpcode() == ARMISD::VDUP) {
14455 SDValue X = Op0->getOperand(0);
14456 if (VT == MVT::f16 && X.getValueType() == MVT::i32)
14457 return DCI.DAG.getNode(ARMISD::VMOVhr, dl, VT, X);
14458 if (VT == MVT::i32 && X.getValueType() == MVT::f16)
14459 return DCI.DAG.getNode(ARMISD::VMOVrh, dl, VT, X);
14460
14461 while (X.getValueType() != VT && X->getOpcode() == ISD::BITCAST)
14462 X = X->getOperand(0);
14463 if (X.getValueType() == VT)
14464 return X;
14465 }
14466
14467 // extract(bitcast(BUILD_VECTOR(VMOVDRR(a, b), ..))) -> a or b
14468 if (Op0.getValueType() == MVT::v4i32 &&
14469 isa<ConstantSDNode>(N->getOperand(1)) &&
14470 Op0.getOpcode() == ISD::BITCAST &&
14471 Op0.getOperand(0).getOpcode() == ISD::BUILD_VECTOR &&
14472 Op0.getOperand(0).getValueType() == MVT::v2f64) {
14473 SDValue BV = Op0.getOperand(0);
14474 unsigned Offset = N->getConstantOperandVal(1);
14475 SDValue MOV = BV.getOperand(Offset < 2 ? 0 : 1);
14476 if (MOV.getOpcode() == ARMISD::VMOVDRR)
14477 return MOV.getOperand(ST->isLittle() ? Offset % 2 : 1 - Offset % 2);
14478 }
14479
14480 // extract x, n; extract x, n+1 -> VMOVRRD x
14481 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
14482 return R;
14483
14484 return SDValue();
14485}
14486
14487static SDValue PerformSignExtendInregCombine(SDNode *N, SelectionDAG &DAG) {
14488 SDValue Op = N->getOperand(0);
14489 EVT VT = N->getValueType(0);
14490
14491 // sext_inreg(VGETLANEu) -> VGETLANEs
14492 if (Op.getOpcode() == ARMISD::VGETLANEu &&
14493 cast<VTSDNode>(N->getOperand(1))->getVT() ==
14494 Op.getOperand(0).getValueType().getScalarType())
14495 return DAG.getNode(ARMISD::VGETLANEs, SDLoc(N), VT, Op.getOperand(0),
14496 Op.getOperand(1));
14497
14498 return SDValue();
14499}
14500
14501// When lowering complex nodes that we recognize, like VQDMULH and MULH, we
14502// can end up with shuffle(binop(shuffle, shuffle)), that can be simplified to
14503// binop as the shuffles cancel out.
14504static SDValue FlattenVectorShuffle(ShuffleVectorSDNode *N, SelectionDAG &DAG) {
14505 EVT VT = N->getValueType(0);
14506 if (!N->getOperand(1).isUndef() || N->getOperand(0).getValueType() != VT)
14507 return SDValue();
14508 SDValue Op = N->getOperand(0);
14509
14510 // Looking for binary operators that will have been folded from
14511 // truncates/extends.
14512 switch (Op.getOpcode()) {
14513 case ARMISD::VQDMULH:
14514 case ISD::MULHS:
14515 case ISD::MULHU:
14516 break;
14517 default:
14518 return SDValue();
14519 }
14520
14521 ShuffleVectorSDNode *Op0 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(0));
14522 ShuffleVectorSDNode *Op1 = dyn_cast<ShuffleVectorSDNode>(Op.getOperand(1));
14523 if (!Op0 || !Op1 || !Op0->getOperand(1).isUndef() ||
14524 !Op1->getOperand(1).isUndef() || Op0->getMask() != Op1->getMask() ||
14525 Op0->getOperand(0).getValueType() != VT)
14526 return SDValue();
14527
14528 // Check the mask turns into an identity shuffle.
14529 ArrayRef<int> NMask = N->getMask();
14530 ArrayRef<int> OpMask = Op0->getMask();
14531 for (int i = 0, e = NMask.size(); i != e; i++) {
14532 if (NMask[i] > 0 && OpMask[NMask[i]] > 0 && OpMask[NMask[i]] != i)
14533 return SDValue();
14534 }
14535
14536 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(),
14537 Op0->getOperand(0), Op1->getOperand(0));
14538}
14539
14540/// PerformVECTOR_SHUFFLECombine - Target-specific dag combine xforms for
14541/// ISD::VECTOR_SHUFFLE.
14542static SDValue PerformVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG) {
14543 if (SDValue R = FlattenVectorShuffle(cast<ShuffleVectorSDNode>(N), DAG))
14544 return R;
14545
14546 // The LLVM shufflevector instruction does not require the shuffle mask
14547 // length to match the operand vector length, but ISD::VECTOR_SHUFFLE does
14548 // have that requirement. When translating to ISD::VECTOR_SHUFFLE, if the
14549 // operands do not match the mask length, they are extended by concatenating
14550 // them with undef vectors. That is probably the right thing for other
14551 // targets, but for NEON it is better to concatenate two double-register
14552 // size vector operands into a single quad-register size vector. Do that
14553 // transformation here:
14554 // shuffle(concat(v1, undef), concat(v2, undef)) ->
14555 // shuffle(concat(v1, v2), undef)
14556 SDValue Op0 = N->getOperand(0);
14557 SDValue Op1 = N->getOperand(1);
14558 if (Op0.getOpcode() != ISD::CONCAT_VECTORS ||
14559 Op1.getOpcode() != ISD::CONCAT_VECTORS ||
14560 Op0.getNumOperands() != 2 ||
14561 Op1.getNumOperands() != 2)
14562 return SDValue();
14563 SDValue Concat0Op1 = Op0.getOperand(1);
14564 SDValue Concat1Op1 = Op1.getOperand(1);
14565 if (!Concat0Op1.isUndef() || !Concat1Op1.isUndef())
14566 return SDValue();
14567 // Skip the transformation if any of the types are illegal.
14568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
14569 EVT VT = N->getValueType(0);
14570 if (!TLI.isTypeLegal(VT) ||
14571 !TLI.isTypeLegal(Concat0Op1.getValueType()) ||
14572 !TLI.isTypeLegal(Concat1Op1.getValueType()))
14573 return SDValue();
14574
14575 SDValue NewConcat = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT,
14576 Op0.getOperand(0), Op1.getOperand(0));
14577 // Translate the shuffle mask.
14578 SmallVector<int, 16> NewMask;
14579 unsigned NumElts = VT.getVectorNumElements();
14580 unsigned HalfElts = NumElts/2;
14581 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(N);
14582 for (unsigned n = 0; n < NumElts; ++n) {
14583 int MaskElt = SVN->getMaskElt(n);
14584 int NewElt = -1;
14585 if (MaskElt < (int)HalfElts)
14586 NewElt = MaskElt;
14587 else if (MaskElt >= (int)NumElts && MaskElt < (int)(NumElts + HalfElts))
14588 NewElt = HalfElts + MaskElt - NumElts;
14589 NewMask.push_back(NewElt);
14590 }
14591 return DAG.getVectorShuffle(VT, SDLoc(N), NewConcat,
14592 DAG.getUNDEF(VT), NewMask);
14593}
14594
14595/// CombineBaseUpdate - Target-specific DAG combine function for VLDDUP,
14596/// NEON load/store intrinsics, and generic vector load/stores, to merge
14597/// base address updates.
14598/// For generic load/stores, the memory type is assumed to be a vector.
14599/// The caller is assumed to have checked legality.
14600static SDValue CombineBaseUpdate(SDNode *N,
14601 TargetLowering::DAGCombinerInfo &DCI) {
14602 SelectionDAG &DAG = DCI.DAG;
14603 const bool isIntrinsic = (N->getOpcode() == ISD::INTRINSIC_VOID ||
14604 N->getOpcode() == ISD::INTRINSIC_W_CHAIN);
14605 const bool isStore = N->getOpcode() == ISD::STORE;
14606 const unsigned AddrOpIdx = ((isIntrinsic || isStore) ? 2 : 1);
14607 SDValue Addr = N->getOperand(AddrOpIdx);
14608 MemSDNode *MemN = cast<MemSDNode>(N);
14609 SDLoc dl(N);
14610
14611 // Search for a use of the address operand that is an increment.
14612 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
14613 UE = Addr.getNode()->use_end(); UI != UE; ++UI) {
14614 SDNode *User = *UI;
14615 if (User->getOpcode() != ISD::ADD ||
14616 UI.getUse().getResNo() != Addr.getResNo())
14617 continue;
14618
14619 // Check that the add is independent of the load/store. Otherwise, folding
14620 // it would create a cycle. We can avoid searching through Addr as it's a
14621 // predecessor to both.
14622 SmallPtrSet<const SDNode *, 32> Visited;
14623 SmallVector<const SDNode *, 16> Worklist;
14624 Visited.insert(Addr.getNode());
14625 Worklist.push_back(N);
14626 Worklist.push_back(User);
14627 if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
14628 SDNode::hasPredecessorHelper(User, Visited, Worklist))
14629 continue;
14630
14631 // Find the new opcode for the updating load/store.
14632 bool isLoadOp = true;
14633 bool isLaneOp = false;
14634 // Workaround for vst1x and vld1x intrinsics which do not have alignment
14635 // as an operand.
14636 bool hasAlignment = true;
14637 unsigned NewOpc = 0;
14638 unsigned NumVecs = 0;
14639 if (isIntrinsic) {
14640 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
14641 switch (IntNo) {
14642 default: llvm_unreachable("unexpected intrinsic for Neon base update")::llvm::llvm_unreachable_internal("unexpected intrinsic for Neon base update"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14642)
;
14643 case Intrinsic::arm_neon_vld1: NewOpc = ARMISD::VLD1_UPD;
14644 NumVecs = 1; break;
14645 case Intrinsic::arm_neon_vld2: NewOpc = ARMISD::VLD2_UPD;
14646 NumVecs = 2; break;
14647 case Intrinsic::arm_neon_vld3: NewOpc = ARMISD::VLD3_UPD;
14648 NumVecs = 3; break;
14649 case Intrinsic::arm_neon_vld4: NewOpc = ARMISD::VLD4_UPD;
14650 NumVecs = 4; break;
14651 case Intrinsic::arm_neon_vld1x2: NewOpc = ARMISD::VLD1x2_UPD;
14652 NumVecs = 2; hasAlignment = false; break;
14653 case Intrinsic::arm_neon_vld1x3: NewOpc = ARMISD::VLD1x3_UPD;
14654 NumVecs = 3; hasAlignment = false; break;
14655 case Intrinsic::arm_neon_vld1x4: NewOpc = ARMISD::VLD1x4_UPD;
14656 NumVecs = 4; hasAlignment = false; break;
14657 case Intrinsic::arm_neon_vld2dup:
14658 case Intrinsic::arm_neon_vld3dup:
14659 case Intrinsic::arm_neon_vld4dup:
14660 // TODO: Support updating VLDxDUP nodes. For now, we just skip
14661 // combining base updates for such intrinsics.
14662 continue;
14663 case Intrinsic::arm_neon_vld2lane: NewOpc = ARMISD::VLD2LN_UPD;
14664 NumVecs = 2; isLaneOp = true; break;
14665 case Intrinsic::arm_neon_vld3lane: NewOpc = ARMISD::VLD3LN_UPD;
14666 NumVecs = 3; isLaneOp = true; break;
14667 case Intrinsic::arm_neon_vld4lane: NewOpc = ARMISD::VLD4LN_UPD;
14668 NumVecs = 4; isLaneOp = true; break;
14669 case Intrinsic::arm_neon_vst1: NewOpc = ARMISD::VST1_UPD;
14670 NumVecs = 1; isLoadOp = false; break;
14671 case Intrinsic::arm_neon_vst2: NewOpc = ARMISD::VST2_UPD;
14672 NumVecs = 2; isLoadOp = false; break;
14673 case Intrinsic::arm_neon_vst3: NewOpc = ARMISD::VST3_UPD;
14674 NumVecs = 3; isLoadOp = false; break;
14675 case Intrinsic::arm_neon_vst4: NewOpc = ARMISD::VST4_UPD;
14676 NumVecs = 4; isLoadOp = false; break;
14677 case Intrinsic::arm_neon_vst2lane: NewOpc = ARMISD::VST2LN_UPD;
14678 NumVecs = 2; isLoadOp = false; isLaneOp = true; break;
14679 case Intrinsic::arm_neon_vst3lane: NewOpc = ARMISD::VST3LN_UPD;
14680 NumVecs = 3; isLoadOp = false; isLaneOp = true; break;
14681 case Intrinsic::arm_neon_vst4lane: NewOpc = ARMISD::VST4LN_UPD;
14682 NumVecs = 4; isLoadOp = false; isLaneOp = true; break;
14683 case Intrinsic::arm_neon_vst1x2: NewOpc = ARMISD::VST1x2_UPD;
14684 NumVecs = 2; isLoadOp = false; hasAlignment = false; break;
14685 case Intrinsic::arm_neon_vst1x3: NewOpc = ARMISD::VST1x3_UPD;
14686 NumVecs = 3; isLoadOp = false; hasAlignment = false; break;
14687 case Intrinsic::arm_neon_vst1x4: NewOpc = ARMISD::VST1x4_UPD;
14688 NumVecs = 4; isLoadOp = false; hasAlignment = false; break;
14689 }
14690 } else {
14691 isLaneOp = true;
14692 switch (N->getOpcode()) {
14693 default: llvm_unreachable("unexpected opcode for Neon base update")::llvm::llvm_unreachable_internal("unexpected opcode for Neon base update"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14693)
;
14694 case ARMISD::VLD1DUP: NewOpc = ARMISD::VLD1DUP_UPD; NumVecs = 1; break;
14695 case ARMISD::VLD2DUP: NewOpc = ARMISD::VLD2DUP_UPD; NumVecs = 2; break;
14696 case ARMISD::VLD3DUP: NewOpc = ARMISD::VLD3DUP_UPD; NumVecs = 3; break;
14697 case ARMISD::VLD4DUP: NewOpc = ARMISD::VLD4DUP_UPD; NumVecs = 4; break;
14698 case ISD::LOAD: NewOpc = ARMISD::VLD1_UPD;
14699 NumVecs = 1; isLaneOp = false; break;
14700 case ISD::STORE: NewOpc = ARMISD::VST1_UPD;
14701 NumVecs = 1; isLaneOp = false; isLoadOp = false; break;
14702 }
14703 }
14704
14705 // Find the size of memory referenced by the load/store.
14706 EVT VecTy;
14707 if (isLoadOp) {
14708 VecTy = N->getValueType(0);
14709 } else if (isIntrinsic) {
14710 VecTy = N->getOperand(AddrOpIdx+1).getValueType();
14711 } else {
14712 assert(isStore && "Node has to be a load, a store, or an intrinsic!")(static_cast <bool> (isStore && "Node has to be a load, a store, or an intrinsic!"
) ? void (0) : __assert_fail ("isStore && \"Node has to be a load, a store, or an intrinsic!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14712, __extension__ __PRETTY_FUNCTION__))
;
14713 VecTy = N->getOperand(1).getValueType();
14714 }
14715
14716 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
14717 if (isLaneOp)
14718 NumBytes /= VecTy.getVectorNumElements();
14719
14720 // If the increment is a constant, it must match the memory ref size.
14721 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14722 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode());
14723 if (NumBytes >= 3 * 16 && (!CInc || CInc->getZExtValue() != NumBytes)) {
14724 // VLD3/4 and VST3/4 for 128-bit vectors are implemented with two
14725 // separate instructions that make it harder to use a non-constant update.
14726 continue;
14727 }
14728
14729 // OK, we found an ADD we can fold into the base update.
14730 // Now, create a _UPD node, taking care of not breaking alignment.
14731
14732 EVT AlignedVecTy = VecTy;
14733 unsigned Alignment = MemN->getAlignment();
14734
14735 // If this is a less-than-standard-aligned load/store, change the type to
14736 // match the standard alignment.
14737 // The alignment is overlooked when selecting _UPD variants; and it's
14738 // easier to introduce bitcasts here than fix that.
14739 // There are 3 ways to get to this base-update combine:
14740 // - intrinsics: they are assumed to be properly aligned (to the standard
14741 // alignment of the memory type), so we don't need to do anything.
14742 // - ARMISD::VLDx nodes: they are only generated from the aforementioned
14743 // intrinsics, so, likewise, there's nothing to do.
14744 // - generic load/store instructions: the alignment is specified as an
14745 // explicit operand, rather than implicitly as the standard alignment
14746 // of the memory type (like the intrisics). We need to change the
14747 // memory type to match the explicit alignment. That way, we don't
14748 // generate non-standard-aligned ARMISD::VLDx nodes.
14749 if (isa<LSBaseSDNode>(N)) {
14750 if (Alignment == 0)
14751 Alignment = 1;
14752 if (Alignment < VecTy.getScalarSizeInBits() / 8) {
14753 MVT EltTy = MVT::getIntegerVT(Alignment * 8);
14754 assert(NumVecs == 1 && "Unexpected multi-element generic load/store.")(static_cast <bool> (NumVecs == 1 && "Unexpected multi-element generic load/store."
) ? void (0) : __assert_fail ("NumVecs == 1 && \"Unexpected multi-element generic load/store.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14754, __extension__ __PRETTY_FUNCTION__))
;
14755 assert(!isLaneOp && "Unexpected generic load/store lane.")(static_cast <bool> (!isLaneOp && "Unexpected generic load/store lane."
) ? void (0) : __assert_fail ("!isLaneOp && \"Unexpected generic load/store lane.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14755, __extension__ __PRETTY_FUNCTION__))
;
14756 unsigned NumElts = NumBytes / (EltTy.getSizeInBits() / 8);
14757 AlignedVecTy = MVT::getVectorVT(EltTy, NumElts);
14758 }
14759 // Don't set an explicit alignment on regular load/stores that we want
14760 // to transform to VLD/VST 1_UPD nodes.
14761 // This matches the behavior of regular load/stores, which only get an
14762 // explicit alignment if the MMO alignment is larger than the standard
14763 // alignment of the memory type.
14764 // Intrinsics, however, always get an explicit alignment, set to the
14765 // alignment of the MMO.
14766 Alignment = 1;
14767 }
14768
14769 // Create the new updating load/store node.
14770 // First, create an SDVTList for the new updating node's results.
14771 EVT Tys[6];
14772 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
14773 unsigned n;
14774 for (n = 0; n < NumResultVecs; ++n)
14775 Tys[n] = AlignedVecTy;
14776 Tys[n++] = MVT::i32;
14777 Tys[n] = MVT::Other;
14778 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs+2));
14779
14780 // Then, gather the new node's operands.
14781 SmallVector<SDValue, 8> Ops;
14782 Ops.push_back(N->getOperand(0)); // incoming chain
14783 Ops.push_back(N->getOperand(AddrOpIdx));
14784 Ops.push_back(Inc);
14785
14786 if (StoreSDNode *StN = dyn_cast<StoreSDNode>(N)) {
14787 // Try to match the intrinsic's signature
14788 Ops.push_back(StN->getValue());
14789 } else {
14790 // Loads (and of course intrinsics) match the intrinsics' signature,
14791 // so just add all but the alignment operand.
14792 unsigned LastOperand =
14793 hasAlignment ? N->getNumOperands() - 1 : N->getNumOperands();
14794 for (unsigned i = AddrOpIdx + 1; i < LastOperand; ++i)
14795 Ops.push_back(N->getOperand(i));
14796 }
14797
14798 // For all node types, the alignment operand is always the last one.
14799 Ops.push_back(DAG.getConstant(Alignment, dl, MVT::i32));
14800
14801 // If this is a non-standard-aligned STORE, the penultimate operand is the
14802 // stored value. Bitcast it to the aligned type.
14803 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::STORE) {
14804 SDValue &StVal = Ops[Ops.size()-2];
14805 StVal = DAG.getNode(ISD::BITCAST, dl, AlignedVecTy, StVal);
14806 }
14807
14808 EVT LoadVT = isLaneOp ? VecTy.getVectorElementType() : AlignedVecTy;
14809 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, LoadVT,
14810 MemN->getMemOperand());
14811
14812 // Update the uses.
14813 SmallVector<SDValue, 5> NewResults;
14814 for (unsigned i = 0; i < NumResultVecs; ++i)
14815 NewResults.push_back(SDValue(UpdN.getNode(), i));
14816
14817 // If this is an non-standard-aligned LOAD, the first result is the loaded
14818 // value. Bitcast it to the expected result type.
14819 if (AlignedVecTy != VecTy && N->getOpcode() == ISD::LOAD) {
14820 SDValue &LdVal = NewResults[0];
14821 LdVal = DAG.getNode(ISD::BITCAST, dl, VecTy, LdVal);
14822 }
14823
14824 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs+1)); // chain
14825 DCI.CombineTo(N, NewResults);
14826 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14827
14828 break;
14829 }
14830 return SDValue();
14831}
14832
14833static SDValue PerformVLDCombine(SDNode *N,
14834 TargetLowering::DAGCombinerInfo &DCI) {
14835 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14836 return SDValue();
14837
14838 return CombineBaseUpdate(N, DCI);
14839}
14840
14841static SDValue PerformMVEVLDCombine(SDNode *N,
14842 TargetLowering::DAGCombinerInfo &DCI) {
14843 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
14844 return SDValue();
14845
14846 SelectionDAG &DAG = DCI.DAG;
14847 SDValue Addr = N->getOperand(2);
14848 MemSDNode *MemN = cast<MemSDNode>(N);
14849 SDLoc dl(N);
14850
14851 // For the stores, where there are multiple intrinsics we only actually want
14852 // to post-inc the last of the them.
14853 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
14854 if (IntNo == Intrinsic::arm_mve_vst2q &&
14855 cast<ConstantSDNode>(N->getOperand(5))->getZExtValue() != 1)
14856 return SDValue();
14857 if (IntNo == Intrinsic::arm_mve_vst4q &&
14858 cast<ConstantSDNode>(N->getOperand(7))->getZExtValue() != 3)
14859 return SDValue();
14860
14861 // Search for a use of the address operand that is an increment.
14862 for (SDNode::use_iterator UI = Addr.getNode()->use_begin(),
14863 UE = Addr.getNode()->use_end();
14864 UI != UE; ++UI) {
14865 SDNode *User = *UI;
14866 if (User->getOpcode() != ISD::ADD ||
14867 UI.getUse().getResNo() != Addr.getResNo())
14868 continue;
14869
14870 // Check that the add is independent of the load/store. Otherwise, folding
14871 // it would create a cycle. We can avoid searching through Addr as it's a
14872 // predecessor to both.
14873 SmallPtrSet<const SDNode *, 32> Visited;
14874 SmallVector<const SDNode *, 16> Worklist;
14875 Visited.insert(Addr.getNode());
14876 Worklist.push_back(N);
14877 Worklist.push_back(User);
14878 if (SDNode::hasPredecessorHelper(N, Visited, Worklist) ||
14879 SDNode::hasPredecessorHelper(User, Visited, Worklist))
14880 continue;
14881
14882 // Find the new opcode for the updating load/store.
14883 bool isLoadOp = true;
14884 unsigned NewOpc = 0;
14885 unsigned NumVecs = 0;
14886 switch (IntNo) {
14887 default:
14888 llvm_unreachable("unexpected intrinsic for MVE VLDn combine")::llvm::llvm_unreachable_internal("unexpected intrinsic for MVE VLDn combine"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 14888)
;
14889 case Intrinsic::arm_mve_vld2q:
14890 NewOpc = ARMISD::VLD2_UPD;
14891 NumVecs = 2;
14892 break;
14893 case Intrinsic::arm_mve_vld4q:
14894 NewOpc = ARMISD::VLD4_UPD;
14895 NumVecs = 4;
14896 break;
14897 case Intrinsic::arm_mve_vst2q:
14898 NewOpc = ARMISD::VST2_UPD;
14899 NumVecs = 2;
14900 isLoadOp = false;
14901 break;
14902 case Intrinsic::arm_mve_vst4q:
14903 NewOpc = ARMISD::VST4_UPD;
14904 NumVecs = 4;
14905 isLoadOp = false;
14906 break;
14907 }
14908
14909 // Find the size of memory referenced by the load/store.
14910 EVT VecTy;
14911 if (isLoadOp) {
14912 VecTy = N->getValueType(0);
14913 } else {
14914 VecTy = N->getOperand(3).getValueType();
14915 }
14916
14917 unsigned NumBytes = NumVecs * VecTy.getSizeInBits() / 8;
14918
14919 // If the increment is a constant, it must match the memory ref size.
14920 SDValue Inc = User->getOperand(User->getOperand(0) == Addr ? 1 : 0);
14921 ConstantSDNode *CInc = dyn_cast<ConstantSDNode>(Inc.getNode());
14922 if (!CInc || CInc->getZExtValue() != NumBytes)
14923 continue;
14924
14925 // Create the new updating load/store node.
14926 // First, create an SDVTList for the new updating node's results.
14927 EVT Tys[6];
14928 unsigned NumResultVecs = (isLoadOp ? NumVecs : 0);
14929 unsigned n;
14930 for (n = 0; n < NumResultVecs; ++n)
14931 Tys[n] = VecTy;
14932 Tys[n++] = MVT::i32;
14933 Tys[n] = MVT::Other;
14934 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumResultVecs + 2));
14935
14936 // Then, gather the new node's operands.
14937 SmallVector<SDValue, 8> Ops;
14938 Ops.push_back(N->getOperand(0)); // incoming chain
14939 Ops.push_back(N->getOperand(2)); // ptr
14940 Ops.push_back(Inc);
14941
14942 for (unsigned i = 3; i < N->getNumOperands(); ++i)
14943 Ops.push_back(N->getOperand(i));
14944
14945 SDValue UpdN = DAG.getMemIntrinsicNode(NewOpc, dl, SDTys, Ops, VecTy,
14946 MemN->getMemOperand());
14947
14948 // Update the uses.
14949 SmallVector<SDValue, 5> NewResults;
14950 for (unsigned i = 0; i < NumResultVecs; ++i)
14951 NewResults.push_back(SDValue(UpdN.getNode(), i));
14952
14953 NewResults.push_back(SDValue(UpdN.getNode(), NumResultVecs + 1)); // chain
14954 DCI.CombineTo(N, NewResults);
14955 DCI.CombineTo(User, SDValue(UpdN.getNode(), NumResultVecs));
14956
14957 break;
14958 }
14959
14960 return SDValue();
14961}
14962
14963/// CombineVLDDUP - For a VDUPLANE node N, check if its source operand is a
14964/// vldN-lane (N > 1) intrinsic, and if all the other uses of that intrinsic
14965/// are also VDUPLANEs. If so, combine them to a vldN-dup operation and
14966/// return true.
14967static bool CombineVLDDUP(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
14968 SelectionDAG &DAG = DCI.DAG;
14969 EVT VT = N->getValueType(0);
14970 // vldN-dup instructions only support 64-bit vectors for N > 1.
14971 if (!VT.is64BitVector())
14972 return false;
14973
14974 // Check if the VDUPLANE operand is a vldN-dup intrinsic.
14975 SDNode *VLD = N->getOperand(0).getNode();
14976 if (VLD->getOpcode() != ISD::INTRINSIC_W_CHAIN)
14977 return false;
14978 unsigned NumVecs = 0;
14979 unsigned NewOpc = 0;
14980 unsigned IntNo = cast<ConstantSDNode>(VLD->getOperand(1))->getZExtValue();
14981 if (IntNo == Intrinsic::arm_neon_vld2lane) {
14982 NumVecs = 2;
14983 NewOpc = ARMISD::VLD2DUP;
14984 } else if (IntNo == Intrinsic::arm_neon_vld3lane) {
14985 NumVecs = 3;
14986 NewOpc = ARMISD::VLD3DUP;
14987 } else if (IntNo == Intrinsic::arm_neon_vld4lane) {
14988 NumVecs = 4;
14989 NewOpc = ARMISD::VLD4DUP;
14990 } else {
14991 return false;
14992 }
14993
14994 // First check that all the vldN-lane uses are VDUPLANEs and that the lane
14995 // numbers match the load.
14996 unsigned VLDLaneNo =
14997 cast<ConstantSDNode>(VLD->getOperand(NumVecs+3))->getZExtValue();
14998 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
14999 UI != UE; ++UI) {
15000 // Ignore uses of the chain result.
15001 if (UI.getUse().getResNo() == NumVecs)
15002 continue;
15003 SDNode *User = *UI;
15004 if (User->getOpcode() != ARMISD::VDUPLANE ||
15005 VLDLaneNo != cast<ConstantSDNode>(User->getOperand(1))->getZExtValue())
15006 return false;
15007 }
15008
15009 // Create the vldN-dup node.
15010 EVT Tys[5];
15011 unsigned n;
15012 for (n = 0; n < NumVecs; ++n)
15013 Tys[n] = VT;
15014 Tys[n] = MVT::Other;
15015 SDVTList SDTys = DAG.getVTList(makeArrayRef(Tys, NumVecs+1));
15016 SDValue Ops[] = { VLD->getOperand(0), VLD->getOperand(2) };
15017 MemIntrinsicSDNode *VLDMemInt = cast<MemIntrinsicSDNode>(VLD);
15018 SDValue VLDDup = DAG.getMemIntrinsicNode(NewOpc, SDLoc(VLD), SDTys,
15019 Ops, VLDMemInt->getMemoryVT(),
15020 VLDMemInt->getMemOperand());
15021
15022 // Update the uses.
15023 for (SDNode::use_iterator UI = VLD->use_begin(), UE = VLD->use_end();
15024 UI != UE; ++UI) {
15025 unsigned ResNo = UI.getUse().getResNo();
15026 // Ignore uses of the chain result.
15027 if (ResNo == NumVecs)
15028 continue;
15029 SDNode *User = *UI;
15030 DCI.CombineTo(User, SDValue(VLDDup.getNode(), ResNo));
15031 }
15032
15033 // Now the vldN-lane intrinsic is dead except for its chain result.
15034 // Update uses of the chain.
15035 std::vector<SDValue> VLDDupResults;
15036 for (unsigned n = 0; n < NumVecs; ++n)
15037 VLDDupResults.push_back(SDValue(VLDDup.getNode(), n));
15038 VLDDupResults.push_back(SDValue(VLDDup.getNode(), NumVecs));
15039 DCI.CombineTo(VLD, VLDDupResults);
15040
15041 return true;
15042}
15043
15044/// PerformVDUPLANECombine - Target-specific dag combine xforms for
15045/// ARMISD::VDUPLANE.
15046static SDValue PerformVDUPLANECombine(SDNode *N,
15047 TargetLowering::DAGCombinerInfo &DCI,
15048 const ARMSubtarget *Subtarget) {
15049 SDValue Op = N->getOperand(0);
15050 EVT VT = N->getValueType(0);
15051
15052 // On MVE, we just convert the VDUPLANE to a VDUP with an extract.
15053 if (Subtarget->hasMVEIntegerOps()) {
15054 EVT ExtractVT = VT.getVectorElementType();
15055 // We need to ensure we are creating a legal type.
15056 if (!DCI.DAG.getTargetLoweringInfo().isTypeLegal(ExtractVT))
15057 ExtractVT = MVT::i32;
15058 SDValue Extract = DCI.DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), ExtractVT,
15059 N->getOperand(0), N->getOperand(1));
15060 return DCI.DAG.getNode(ARMISD::VDUP, SDLoc(N), VT, Extract);
15061 }
15062
15063 // If the source is a vldN-lane (N > 1) intrinsic, and all the other uses
15064 // of that intrinsic are also VDUPLANEs, combine them to a vldN-dup operation.
15065 if (CombineVLDDUP(N, DCI))
15066 return SDValue(N, 0);
15067
15068 // If the source is already a VMOVIMM or VMVNIMM splat, the VDUPLANE is
15069 // redundant. Ignore bit_converts for now; element sizes are checked below.
15070 while (Op.getOpcode() == ISD::BITCAST)
15071 Op = Op.getOperand(0);
15072 if (Op.getOpcode() != ARMISD::VMOVIMM && Op.getOpcode() != ARMISD::VMVNIMM)
15073 return SDValue();
15074
15075 // Make sure the VMOV element size is not bigger than the VDUPLANE elements.
15076 unsigned EltSize = Op.getScalarValueSizeInBits();
15077 // The canonical VMOV for a zero vector uses a 32-bit element size.
15078 unsigned Imm = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
15079 unsigned EltBits;
15080 if (ARM_AM::decodeVMOVModImm(Imm, EltBits) == 0)
15081 EltSize = 8;
15082 if (EltSize > VT.getScalarSizeInBits())
15083 return SDValue();
15084
15085 return DCI.DAG.getNode(ISD::BITCAST, SDLoc(N), VT, Op);
15086}
15087
15088/// PerformVDUPCombine - Target-specific dag combine xforms for ARMISD::VDUP.
15089static SDValue PerformVDUPCombine(SDNode *N,
15090 TargetLowering::DAGCombinerInfo &DCI,
15091 const ARMSubtarget *Subtarget) {
15092 SelectionDAG &DAG = DCI.DAG;
15093 SDValue Op = N->getOperand(0);
15094 SDLoc dl(N);
15095
15096 if (Subtarget->hasMVEIntegerOps()) {
15097 // Convert VDUP f32 -> VDUP BITCAST i32 under MVE, as we know the value will
15098 // need to come from a GPR.
15099 if (Op.getValueType() == MVT::f32)
15100 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
15101 DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op));
15102 else if (Op.getValueType() == MVT::f16)
15103 return DCI.DAG.getNode(ARMISD::VDUP, dl, N->getValueType(0),
15104 DAG.getNode(ARMISD::VMOVrh, dl, MVT::i32, Op));
15105 }
15106
15107 if (!Subtarget->hasNEON())
15108 return SDValue();
15109
15110 // Match VDUP(LOAD) -> VLD1DUP.
15111 // We match this pattern here rather than waiting for isel because the
15112 // transform is only legal for unindexed loads.
15113 LoadSDNode *LD = dyn_cast<LoadSDNode>(Op.getNode());
15114 if (LD && Op.hasOneUse() && LD->isUnindexed() &&
15115 LD->getMemoryVT() == N->getValueType(0).getVectorElementType()) {
15116 SDValue Ops[] = { LD->getOperand(0), LD->getOperand(1),
15117 DAG.getConstant(LD->getAlignment(), SDLoc(N), MVT::i32) };
15118 SDVTList SDTys = DAG.getVTList(N->getValueType(0), MVT::Other);
15119 SDValue VLDDup = DAG.getMemIntrinsicNode(ARMISD::VLD1DUP, SDLoc(N), SDTys,
15120 Ops, LD->getMemoryVT(),
15121 LD->getMemOperand());
15122 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), VLDDup.getValue(1));
15123 return VLDDup;
15124 }
15125
15126 return SDValue();
15127}
15128
15129static SDValue PerformLOADCombine(SDNode *N,
15130 TargetLowering::DAGCombinerInfo &DCI) {
15131 EVT VT = N->getValueType(0);
15132
15133 // If this is a legal vector load, try to combine it into a VLD1_UPD.
15134 if (ISD::isNormalLoad(N) && VT.isVector() &&
15135 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
15136 return CombineBaseUpdate(N, DCI);
15137
15138 return SDValue();
15139}
15140
15141// Optimize trunc store (of multiple scalars) to shuffle and store. First,
15142// pack all of the elements in one place. Next, store to memory in fewer
15143// chunks.
15144static SDValue PerformTruncatingStoreCombine(StoreSDNode *St,
15145 SelectionDAG &DAG) {
15146 SDValue StVal = St->getValue();
15147 EVT VT = StVal.getValueType();
15148 if (!St->isTruncatingStore() || !VT.isVector())
15149 return SDValue();
15150 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15151 EVT StVT = St->getMemoryVT();
15152 unsigned NumElems = VT.getVectorNumElements();
15153 assert(StVT != VT && "Cannot truncate to the same type")(static_cast <bool> (StVT != VT && "Cannot truncate to the same type"
) ? void (0) : __assert_fail ("StVT != VT && \"Cannot truncate to the same type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15153, __extension__ __PRETTY_FUNCTION__))
;
15154 unsigned FromEltSz = VT.getScalarSizeInBits();
15155 unsigned ToEltSz = StVT.getScalarSizeInBits();
15156
15157 // From, To sizes and ElemCount must be pow of two
15158 if (!isPowerOf2_32(NumElems * FromEltSz * ToEltSz))
15159 return SDValue();
15160
15161 // We are going to use the original vector elt for storing.
15162 // Accumulated smaller vector elements must be a multiple of the store size.
15163 if (0 != (NumElems * FromEltSz) % ToEltSz)
15164 return SDValue();
15165
15166 unsigned SizeRatio = FromEltSz / ToEltSz;
15167 assert(SizeRatio * NumElems * ToEltSz == VT.getSizeInBits())(static_cast <bool> (SizeRatio * NumElems * ToEltSz == VT
.getSizeInBits()) ? void (0) : __assert_fail ("SizeRatio * NumElems * ToEltSz == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15167, __extension__ __PRETTY_FUNCTION__))
;
15168
15169 // Create a type on which we perform the shuffle.
15170 EVT WideVecVT = EVT::getVectorVT(*DAG.getContext(), StVT.getScalarType(),
15171 NumElems * SizeRatio);
15172 assert(WideVecVT.getSizeInBits() == VT.getSizeInBits())(static_cast <bool> (WideVecVT.getSizeInBits() == VT.getSizeInBits
()) ? void (0) : __assert_fail ("WideVecVT.getSizeInBits() == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15172, __extension__ __PRETTY_FUNCTION__))
;
15173
15174 SDLoc DL(St);
15175 SDValue WideVec = DAG.getNode(ISD::BITCAST, DL, WideVecVT, StVal);
15176 SmallVector<int, 8> ShuffleVec(NumElems * SizeRatio, -1);
15177 for (unsigned i = 0; i < NumElems; ++i)
15178 ShuffleVec[i] = DAG.getDataLayout().isBigEndian() ? (i + 1) * SizeRatio - 1
15179 : i * SizeRatio;
15180
15181 // Can't shuffle using an illegal type.
15182 if (!TLI.isTypeLegal(WideVecVT))
15183 return SDValue();
15184
15185 SDValue Shuff = DAG.getVectorShuffle(
15186 WideVecVT, DL, WideVec, DAG.getUNDEF(WideVec.getValueType()), ShuffleVec);
15187 // At this point all of the data is stored at the bottom of the
15188 // register. We now need to save it to mem.
15189
15190 // Find the largest store unit
15191 MVT StoreType = MVT::i8;
15192 for (MVT Tp : MVT::integer_valuetypes()) {
15193 if (TLI.isTypeLegal(Tp) && Tp.getSizeInBits() <= NumElems * ToEltSz)
15194 StoreType = Tp;
15195 }
15196 // Didn't find a legal store type.
15197 if (!TLI.isTypeLegal(StoreType))
15198 return SDValue();
15199
15200 // Bitcast the original vector into a vector of store-size units
15201 EVT StoreVecVT =
15202 EVT::getVectorVT(*DAG.getContext(), StoreType,
15203 VT.getSizeInBits() / EVT(StoreType).getSizeInBits());
15204 assert(StoreVecVT.getSizeInBits() == VT.getSizeInBits())(static_cast <bool> (StoreVecVT.getSizeInBits() == VT.getSizeInBits
()) ? void (0) : __assert_fail ("StoreVecVT.getSizeInBits() == VT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15204, __extension__ __PRETTY_FUNCTION__))
;
15205 SDValue ShuffWide = DAG.getNode(ISD::BITCAST, DL, StoreVecVT, Shuff);
15206 SmallVector<SDValue, 8> Chains;
15207 SDValue Increment = DAG.getConstant(StoreType.getSizeInBits() / 8, DL,
15208 TLI.getPointerTy(DAG.getDataLayout()));
15209 SDValue BasePtr = St->getBasePtr();
15210
15211 // Perform one or more big stores into memory.
15212 unsigned E = (ToEltSz * NumElems) / StoreType.getSizeInBits();
15213 for (unsigned I = 0; I < E; I++) {
15214 SDValue SubVec = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreType,
15215 ShuffWide, DAG.getIntPtrConstant(I, DL));
15216 SDValue Ch =
15217 DAG.getStore(St->getChain(), DL, SubVec, BasePtr, St->getPointerInfo(),
15218 St->getAlignment(), St->getMemOperand()->getFlags());
15219 BasePtr =
15220 DAG.getNode(ISD::ADD, DL, BasePtr.getValueType(), BasePtr, Increment);
15221 Chains.push_back(Ch);
15222 }
15223 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
15224}
15225
15226// Try taking a single vector store from an truncate (which would otherwise turn
15227// into an expensive buildvector) and splitting it into a series of narrowing
15228// stores.
15229static SDValue PerformSplittingToNarrowingStores(StoreSDNode *St,
15230 SelectionDAG &DAG) {
15231 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
15232 return SDValue();
15233 SDValue Trunc = St->getValue();
15234 if (Trunc->getOpcode() != ISD::TRUNCATE && Trunc->getOpcode() != ISD::FP_ROUND)
15235 return SDValue();
15236 EVT FromVT = Trunc->getOperand(0).getValueType();
15237 EVT ToVT = Trunc.getValueType();
15238 if (!ToVT.isVector())
15239 return SDValue();
15240 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements())(static_cast <bool> (FromVT.getVectorNumElements() == ToVT
.getVectorNumElements()) ? void (0) : __assert_fail ("FromVT.getVectorNumElements() == ToVT.getVectorNumElements()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15240, __extension__ __PRETTY_FUNCTION__))
;
15241 EVT ToEltVT = ToVT.getVectorElementType();
15242 EVT FromEltVT = FromVT.getVectorElementType();
15243
15244 unsigned NumElements = 0;
15245 if (FromEltVT == MVT::i32 && (ToEltVT == MVT::i16 || ToEltVT == MVT::i8))
15246 NumElements = 4;
15247 if (FromEltVT == MVT::i16 && ToEltVT == MVT::i8)
15248 NumElements = 8;
15249 if (FromEltVT == MVT::f32 && ToEltVT == MVT::f16)
15250 NumElements = 4;
15251 if (NumElements == 0 ||
15252 (FromEltVT != MVT::f32 && FromVT.getVectorNumElements() == NumElements) ||
15253 FromVT.getVectorNumElements() % NumElements != 0)
15254 return SDValue();
15255
15256 // Test if the Trunc will be convertable to a VMOVN with a shuffle, and if so
15257 // use the VMOVN over splitting the store. We are looking for patterns of:
15258 // !rev: 0 N 1 N+1 2 N+2 ...
15259 // rev: N 0 N+1 1 N+2 2 ...
15260 // The shuffle may either be a single source (in which case N = NumElts/2) or
15261 // two inputs extended with concat to the same size (in which case N =
15262 // NumElts).
15263 auto isVMOVNShuffle = [&](ShuffleVectorSDNode *SVN, bool Rev) {
15264 ArrayRef<int> M = SVN->getMask();
15265 unsigned NumElts = ToVT.getVectorNumElements();
15266 if (SVN->getOperand(1).isUndef())
15267 NumElts /= 2;
15268
15269 unsigned Off0 = Rev ? NumElts : 0;
15270 unsigned Off1 = Rev ? 0 : NumElts;
15271
15272 for (unsigned I = 0; I < NumElts; I += 2) {
15273 if (M[I] >= 0 && M[I] != (int)(Off0 + I / 2))
15274 return false;
15275 if (M[I + 1] >= 0 && M[I + 1] != (int)(Off1 + I / 2))
15276 return false;
15277 }
15278
15279 return true;
15280 };
15281
15282 // It may be preferable to keep the store unsplit as the trunc may end up
15283 // being removed. Check that here.
15284 if (Trunc.getOperand(0).getOpcode() == ISD::SMIN) {
15285 if (SDValue U = PerformVQDMULHCombine(Trunc.getOperand(0).getNode(), DAG)) {
15286 DAG.ReplaceAllUsesWith(Trunc.getOperand(0), U);
15287 return SDValue();
15288 }
15289 }
15290 if (auto *Shuffle = dyn_cast<ShuffleVectorSDNode>(Trunc.getOperand(0)))
15291 if (isVMOVNShuffle(Shuffle, false) || isVMOVNShuffle(Shuffle, true))
15292 return SDValue();
15293
15294 LLVMContext &C = *DAG.getContext();
15295 SDLoc DL(St);
15296 // Details about the old store
15297 SDValue Ch = St->getChain();
15298 SDValue BasePtr = St->getBasePtr();
15299 Align Alignment = St->getOriginalAlign();
15300 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
15301 AAMDNodes AAInfo = St->getAAInfo();
15302
15303 // We split the store into slices of NumElements. fp16 trunc stores are vcvt
15304 // and then stored as truncating integer stores.
15305 EVT NewFromVT = EVT::getVectorVT(C, FromEltVT, NumElements);
15306 EVT NewToVT = EVT::getVectorVT(
15307 C, EVT::getIntegerVT(C, ToEltVT.getSizeInBits()), NumElements);
15308
15309 SmallVector<SDValue, 4> Stores;
15310 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
15311 unsigned NewOffset = i * NumElements * ToEltVT.getSizeInBits() / 8;
15312 SDValue NewPtr =
15313 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
15314
15315 SDValue Extract =
15316 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewFromVT, Trunc.getOperand(0),
15317 DAG.getConstant(i * NumElements, DL, MVT::i32));
15318
15319 if (ToEltVT == MVT::f16) {
15320 SDValue FPTrunc =
15321 DAG.getNode(ARMISD::VCVTN, DL, MVT::v8f16, DAG.getUNDEF(MVT::v8f16),
15322 Extract, DAG.getConstant(0, DL, MVT::i32));
15323 Extract = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v4i32, FPTrunc);
15324 }
15325
15326 SDValue Store = DAG.getTruncStore(
15327 Ch, DL, Extract, NewPtr, St->getPointerInfo().getWithOffset(NewOffset),
15328 NewToVT, Alignment.value(), MMOFlags, AAInfo);
15329 Stores.push_back(Store);
15330 }
15331 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
15332}
15333
15334// Given a floating point store from an extracted vector, with an integer
15335// VGETLANE that already exists, store the existing VGETLANEu directly. This can
15336// help reduce fp register pressure, doesn't require the fp extract and allows
15337// use of more integer post-inc stores not available with vstr.
15338static SDValue PerformExtractFpToIntStores(StoreSDNode *St, SelectionDAG &DAG) {
15339 if (!St->isSimple() || St->isTruncatingStore() || !St->isUnindexed())
15340 return SDValue();
15341 SDValue Extract = St->getValue();
15342 EVT VT = Extract.getValueType();
15343 // For now only uses f16. This may be useful for f32 too, but that will
15344 // be bitcast(extract), not the VGETLANEu we currently check here.
15345 if (VT != MVT::f16 || Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
15346 return SDValue();
15347
15348 SDNode *GetLane =
15349 DAG.getNodeIfExists(ARMISD::VGETLANEu, DAG.getVTList(MVT::i32),
15350 {Extract.getOperand(0), Extract.getOperand(1)});
15351 if (!GetLane)
15352 return SDValue();
15353
15354 LLVMContext &C = *DAG.getContext();
15355 SDLoc DL(St);
15356 // Create a new integer store to replace the existing floating point version.
15357 SDValue Ch = St->getChain();
15358 SDValue BasePtr = St->getBasePtr();
15359 Align Alignment = St->getOriginalAlign();
15360 MachineMemOperand::Flags MMOFlags = St->getMemOperand()->getFlags();
15361 AAMDNodes AAInfo = St->getAAInfo();
15362 EVT NewToVT = EVT::getIntegerVT(C, VT.getSizeInBits());
15363 SDValue Store = DAG.getTruncStore(Ch, DL, SDValue(GetLane, 0), BasePtr,
15364 St->getPointerInfo(), NewToVT,
15365 Alignment.value(), MMOFlags, AAInfo);
15366
15367 return Store;
15368}
15369
15370/// PerformSTORECombine - Target-specific dag combine xforms for
15371/// ISD::STORE.
15372static SDValue PerformSTORECombine(SDNode *N,
15373 TargetLowering::DAGCombinerInfo &DCI,
15374 const ARMSubtarget *Subtarget) {
15375 StoreSDNode *St = cast<StoreSDNode>(N);
15376 if (St->isVolatile())
15377 return SDValue();
15378 SDValue StVal = St->getValue();
15379 EVT VT = StVal.getValueType();
15380
15381 if (Subtarget->hasNEON())
15382 if (SDValue Store = PerformTruncatingStoreCombine(St, DCI.DAG))
15383 return Store;
15384
15385 if (Subtarget->hasMVEIntegerOps()) {
15386 if (SDValue NewToken = PerformSplittingToNarrowingStores(St, DCI.DAG))
15387 return NewToken;
15388 if (SDValue NewChain = PerformExtractFpToIntStores(St, DCI.DAG))
15389 return NewChain;
15390 }
15391
15392 if (!ISD::isNormalStore(St))
15393 return SDValue();
15394
15395 // Split a store of a VMOVDRR into two integer stores to avoid mixing NEON and
15396 // ARM stores of arguments in the same cache line.
15397 if (StVal.getNode()->getOpcode() == ARMISD::VMOVDRR &&
15398 StVal.getNode()->hasOneUse()) {
15399 SelectionDAG &DAG = DCI.DAG;
15400 bool isBigEndian = DAG.getDataLayout().isBigEndian();
15401 SDLoc DL(St);
15402 SDValue BasePtr = St->getBasePtr();
15403 SDValue NewST1 = DAG.getStore(
15404 St->getChain(), DL, StVal.getNode()->getOperand(isBigEndian ? 1 : 0),
15405 BasePtr, St->getPointerInfo(), St->getOriginalAlign(),
15406 St->getMemOperand()->getFlags());
15407
15408 SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i32, BasePtr,
15409 DAG.getConstant(4, DL, MVT::i32));
15410 return DAG.getStore(NewST1.getValue(0), DL,
15411 StVal.getNode()->getOperand(isBigEndian ? 0 : 1),
15412 OffsetPtr, St->getPointerInfo().getWithOffset(4),
15413 St->getOriginalAlign(),
15414 St->getMemOperand()->getFlags());
15415 }
15416
15417 if (StVal.getValueType() == MVT::i64 &&
15418 StVal.getNode()->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
15419
15420 // Bitcast an i64 store extracted from a vector to f64.
15421 // Otherwise, the i64 value will be legalized to a pair of i32 values.
15422 SelectionDAG &DAG = DCI.DAG;
15423 SDLoc dl(StVal);
15424 SDValue IntVec = StVal.getOperand(0);
15425 EVT FloatVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64,
15426 IntVec.getValueType().getVectorNumElements());
15427 SDValue Vec = DAG.getNode(ISD::BITCAST, dl, FloatVT, IntVec);
15428 SDValue ExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
15429 Vec, StVal.getOperand(1));
15430 dl = SDLoc(N);
15431 SDValue V = DAG.getNode(ISD::BITCAST, dl, MVT::i64, ExtElt);
15432 // Make the DAGCombiner fold the bitcasts.
15433 DCI.AddToWorklist(Vec.getNode());
15434 DCI.AddToWorklist(ExtElt.getNode());
15435 DCI.AddToWorklist(V.getNode());
15436 return DAG.getStore(St->getChain(), dl, V, St->getBasePtr(),
15437 St->getPointerInfo(), St->getAlignment(),
15438 St->getMemOperand()->getFlags(), St->getAAInfo());
15439 }
15440
15441 // If this is a legal vector store, try to combine it into a VST1_UPD.
15442 if (Subtarget->hasNEON() && ISD::isNormalStore(N) && VT.isVector() &&
15443 DCI.DAG.getTargetLoweringInfo().isTypeLegal(VT))
15444 return CombineBaseUpdate(N, DCI);
15445
15446 return SDValue();
15447}
15448
15449/// PerformVCVTCombine - VCVT (floating-point to fixed-point, Advanced SIMD)
15450/// can replace combinations of VMUL and VCVT (floating-point to integer)
15451/// when the VMUL has a constant operand that is a power of 2.
15452///
15453/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
15454/// vmul.f32 d16, d17, d16
15455/// vcvt.s32.f32 d16, d16
15456/// becomes:
15457/// vcvt.s32.f32 d16, d16, #3
15458static SDValue PerformVCVTCombine(SDNode *N, SelectionDAG &DAG,
15459 const ARMSubtarget *Subtarget) {
15460 if (!Subtarget->hasNEON())
15461 return SDValue();
15462
15463 SDValue Op = N->getOperand(0);
15464 if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() ||
15465 Op.getOpcode() != ISD::FMUL)
15466 return SDValue();
15467
15468 SDValue ConstVec = Op->getOperand(1);
15469 if (!isa<BuildVectorSDNode>(ConstVec))
15470 return SDValue();
15471
15472 MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
15473 uint32_t FloatBits = FloatTy.getSizeInBits();
15474 MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
15475 uint32_t IntBits = IntTy.getSizeInBits();
15476 unsigned NumLanes = Op.getValueType().getVectorNumElements();
15477 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
15478 // These instructions only exist converting from f32 to i32. We can handle
15479 // smaller integers by generating an extra truncate, but larger ones would
15480 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
15481 // these intructions only support v2i32/v4i32 types.
15482 return SDValue();
15483 }
15484
15485 BitVector UndefElements;
15486 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
15487 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
15488 if (C == -1 || C == 0 || C > 32)
15489 return SDValue();
15490
15491 SDLoc dl(N);
15492 bool isSigned = N->getOpcode() == ISD::FP_TO_SINT;
15493 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfp2fxs :
15494 Intrinsic::arm_neon_vcvtfp2fxu;
15495 SDValue FixConv = DAG.getNode(
15496 ISD::INTRINSIC_WO_CHAIN, dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
15497 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32), Op->getOperand(0),
15498 DAG.getConstant(C, dl, MVT::i32));
15499
15500 if (IntBits < FloatBits)
15501 FixConv = DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), FixConv);
15502
15503 return FixConv;
15504}
15505
15506/// PerformVDIVCombine - VCVT (fixed-point to floating-point, Advanced SIMD)
15507/// can replace combinations of VCVT (integer to floating-point) and VDIV
15508/// when the VDIV has a constant operand that is a power of 2.
15509///
15510/// Example (assume d17 = <float 8.000000e+00, float 8.000000e+00>):
15511/// vcvt.f32.s32 d16, d16
15512/// vdiv.f32 d16, d17, d16
15513/// becomes:
15514/// vcvt.f32.s32 d16, d16, #3
15515static SDValue PerformVDIVCombine(SDNode *N, SelectionDAG &DAG,
15516 const ARMSubtarget *Subtarget) {
15517 if (!Subtarget->hasNEON())
15518 return SDValue();
15519
15520 SDValue Op = N->getOperand(0);
15521 unsigned OpOpcode = Op.getNode()->getOpcode();
15522 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple() ||
15523 (OpOpcode != ISD::SINT_TO_FP && OpOpcode != ISD::UINT_TO_FP))
15524 return SDValue();
15525
15526 SDValue ConstVec = N->getOperand(1);
15527 if (!isa<BuildVectorSDNode>(ConstVec))
15528 return SDValue();
15529
15530 MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
15531 uint32_t FloatBits = FloatTy.getSizeInBits();
15532 MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
15533 uint32_t IntBits = IntTy.getSizeInBits();
15534 unsigned NumLanes = Op.getValueType().getVectorNumElements();
15535 if (FloatBits != 32 || IntBits > 32 || (NumLanes != 4 && NumLanes != 2)) {
15536 // These instructions only exist converting from i32 to f32. We can handle
15537 // smaller integers by generating an extra extend, but larger ones would
15538 // be lossy. We also can't handle anything other than 2 or 4 lanes, since
15539 // these intructions only support v2i32/v4i32 types.
15540 return SDValue();
15541 }
15542
15543 BitVector UndefElements;
15544 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(ConstVec);
15545 int32_t C = BV->getConstantFPSplatPow2ToLog2Int(&UndefElements, 33);
15546 if (C == -1 || C == 0 || C > 32)
15547 return SDValue();
15548
15549 SDLoc dl(N);
15550 bool isSigned = OpOpcode == ISD::SINT_TO_FP;
15551 SDValue ConvInput = Op.getOperand(0);
15552 if (IntBits < FloatBits)
15553 ConvInput = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
15554 dl, NumLanes == 2 ? MVT::v2i32 : MVT::v4i32,
15555 ConvInput);
15556
15557 unsigned IntrinsicOpcode = isSigned ? Intrinsic::arm_neon_vcvtfxs2fp :
15558 Intrinsic::arm_neon_vcvtfxu2fp;
15559 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl,
15560 Op.getValueType(),
15561 DAG.getConstant(IntrinsicOpcode, dl, MVT::i32),
15562 ConvInput, DAG.getConstant(C, dl, MVT::i32));
15563}
15564
15565static SDValue PerformVECREDUCE_ADDCombine(SDNode *N, SelectionDAG &DAG,
15566 const ARMSubtarget *ST) {
15567 if (!ST->hasMVEIntegerOps())
15568 return SDValue();
15569
15570 assert(N->getOpcode() == ISD::VECREDUCE_ADD)(static_cast <bool> (N->getOpcode() == ISD::VECREDUCE_ADD
) ? void (0) : __assert_fail ("N->getOpcode() == ISD::VECREDUCE_ADD"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15570, __extension__ __PRETTY_FUNCTION__))
;
15571 EVT ResVT = N->getValueType(0);
15572 SDValue N0 = N->getOperand(0);
15573 SDLoc dl(N);
15574
15575 // We are looking for something that will have illegal types if left alone,
15576 // but that we can convert to a single instruction undef MVE. For example
15577 // vecreduce_add(sext(A, v8i32)) => VADDV.s16 A
15578 // or
15579 // vecreduce_add(mul(zext(A, v16i32), zext(B, v16i32))) => VMLADAV.u8 A, B
15580
15581 // Cases:
15582 // VADDV u/s 8/16/32
15583 // VMLAV u/s 8/16/32
15584 // VADDLV u/s 32
15585 // VMLALV u/s 16/32
15586
15587 // If the input vector is smaller than legal (v4i8/v4i16 for example) we can
15588 // extend it and use v4i32 instead.
15589 auto ExtendIfNeeded = [&](SDValue A, unsigned ExtendCode) {
15590 EVT AVT = A.getValueType();
15591 if (!AVT.is128BitVector())
15592 A = DAG.getNode(ExtendCode, dl,
15593 AVT.changeVectorElementType(MVT::getIntegerVT(
15594 128 / AVT.getVectorMinNumElements())),
15595 A);
15596 return A;
15597 };
15598 auto IsVADDV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes) {
15599 if (ResVT != RetTy || N0->getOpcode() != ExtendCode)
15600 return SDValue();
15601 SDValue A = N0->getOperand(0);
15602 if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
15603 return ExtendIfNeeded(A, ExtendCode);
15604 return SDValue();
15605 };
15606 auto IsPredVADDV = [&](MVT RetTy, unsigned ExtendCode,
15607 ArrayRef<MVT> ExtTypes, SDValue &Mask) {
15608 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
15609 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
15610 return SDValue();
15611 Mask = N0->getOperand(0);
15612 SDValue Ext = N0->getOperand(1);
15613 if (Ext->getOpcode() != ExtendCode)
15614 return SDValue();
15615 SDValue A = Ext->getOperand(0);
15616 if (llvm::any_of(ExtTypes, [&A](MVT Ty) { return A.getValueType() == Ty; }))
15617 return ExtendIfNeeded(A, ExtendCode);
15618 return SDValue();
15619 };
15620 auto IsVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
15621 SDValue &A, SDValue &B) {
15622 // For a vmla we are trying to match a larger pattern:
15623 // ExtA = sext/zext A
15624 // ExtB = sext/zext B
15625 // Mul = mul ExtA, ExtB
15626 // vecreduce.add Mul
15627 // There might also be en extra extend between the mul and the addreduce, so
15628 // long as the bitwidth is high enough to make them equivalent (for example
15629 // original v8i16 might be mul at v8i32 and the reduce happens at v8i64).
15630 if (ResVT != RetTy)
15631 return false;
15632 SDValue Mul = N0;
15633 if (Mul->getOpcode() == ExtendCode &&
15634 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
15635 ResVT.getScalarSizeInBits())
15636 Mul = Mul->getOperand(0);
15637 if (Mul->getOpcode() != ISD::MUL)
15638 return false;
15639 SDValue ExtA = Mul->getOperand(0);
15640 SDValue ExtB = Mul->getOperand(1);
15641 if (ExtA->getOpcode() != ExtendCode && ExtB->getOpcode() != ExtendCode)
15642 return false;
15643 A = ExtA->getOperand(0);
15644 B = ExtB->getOperand(0);
15645 if (A.getValueType() == B.getValueType() &&
15646 llvm::any_of(ExtTypes,
15647 [&A](MVT Ty) { return A.getValueType() == Ty; })) {
15648 A = ExtendIfNeeded(A, ExtendCode);
15649 B = ExtendIfNeeded(B, ExtendCode);
15650 return true;
15651 }
15652 return false;
15653 };
15654 auto IsPredVMLAV = [&](MVT RetTy, unsigned ExtendCode, ArrayRef<MVT> ExtTypes,
15655 SDValue &A, SDValue &B, SDValue &Mask) {
15656 // Same as the pattern above with a select for the zero predicated lanes
15657 // ExtA = sext/zext A
15658 // ExtB = sext/zext B
15659 // Mul = mul ExtA, ExtB
15660 // N0 = select Mask, Mul, 0
15661 // vecreduce.add N0
15662 if (ResVT != RetTy || N0->getOpcode() != ISD::VSELECT ||
15663 !ISD::isBuildVectorAllZeros(N0->getOperand(2).getNode()))
15664 return false;
15665 Mask = N0->getOperand(0);
15666 SDValue Mul = N0->getOperand(1);
15667 if (Mul->getOpcode() == ExtendCode &&
15668 Mul->getOperand(0).getScalarValueSizeInBits() * 2 >=
15669 ResVT.getScalarSizeInBits())
15670 Mul = Mul->getOperand(0);
15671 if (Mul->getOpcode() != ISD::MUL)
15672 return false;
15673 SDValue ExtA = Mul->getOperand(0);
15674 SDValue ExtB = Mul->getOperand(1);
15675 if (ExtA->getOpcode() != ExtendCode && ExtB->getOpcode() != ExtendCode)
15676 return false;
15677 A = ExtA->getOperand(0);
15678 B = ExtB->getOperand(0);
15679 if (A.getValueType() == B.getValueType() &&
15680 llvm::any_of(ExtTypes,
15681 [&A](MVT Ty) { return A.getValueType() == Ty; })) {
15682 A = ExtendIfNeeded(A, ExtendCode);
15683 B = ExtendIfNeeded(B, ExtendCode);
15684 return true;
15685 }
15686 return false;
15687 };
15688 auto Create64bitNode = [&](unsigned Opcode, ArrayRef<SDValue> Ops) {
15689 SDValue Node = DAG.getNode(Opcode, dl, {MVT::i32, MVT::i32}, Ops);
15690 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Node,
15691 SDValue(Node.getNode(), 1));
15692 };
15693
15694 if (SDValue A = IsVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}))
15695 return DAG.getNode(ARMISD::VADDVs, dl, ResVT, A);
15696 if (SDValue A = IsVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}))
15697 return DAG.getNode(ARMISD::VADDVu, dl, ResVT, A);
15698 if (SDValue A = IsVADDV(MVT::i64, ISD::SIGN_EXTEND,
15699 {MVT::v4i8, MVT::v4i16, MVT::v4i32}))
15700 return Create64bitNode(ARMISD::VADDLVs, {A});
15701 if (SDValue A = IsVADDV(MVT::i64, ISD::ZERO_EXTEND,
15702 {MVT::v4i8, MVT::v4i16, MVT::v4i32}))
15703 return Create64bitNode(ARMISD::VADDLVu, {A});
15704 if (SDValue A = IsVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}))
15705 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15706 DAG.getNode(ARMISD::VADDVs, dl, MVT::i32, A));
15707 if (SDValue A = IsVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}))
15708 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15709 DAG.getNode(ARMISD::VADDVu, dl, MVT::i32, A));
15710
15711 SDValue Mask;
15712 if (SDValue A = IsPredVADDV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
15713 return DAG.getNode(ARMISD::VADDVps, dl, ResVT, A, Mask);
15714 if (SDValue A = IsPredVADDV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, Mask))
15715 return DAG.getNode(ARMISD::VADDVpu, dl, ResVT, A, Mask);
15716 if (SDValue A = IsPredVADDV(MVT::i64, ISD::SIGN_EXTEND,
15717 {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask))
15718 return Create64bitNode(ARMISD::VADDLVps, {A, Mask});
15719 if (SDValue A = IsPredVADDV(MVT::i64, ISD::ZERO_EXTEND,
15720 {MVT::v4i8, MVT::v4i16, MVT::v4i32}, Mask))
15721 return Create64bitNode(ARMISD::VADDLVpu, {A, Mask});
15722 if (SDValue A = IsPredVADDV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, Mask))
15723 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15724 DAG.getNode(ARMISD::VADDVps, dl, MVT::i32, A, Mask));
15725 if (SDValue A = IsPredVADDV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, Mask))
15726 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15727 DAG.getNode(ARMISD::VADDVpu, dl, MVT::i32, A, Mask));
15728
15729 SDValue A, B;
15730 if (IsVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
15731 return DAG.getNode(ARMISD::VMLAVs, dl, ResVT, A, B);
15732 if (IsVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B))
15733 return DAG.getNode(ARMISD::VMLAVu, dl, ResVT, A, B);
15734 if (IsVMLAV(MVT::i64, ISD::SIGN_EXTEND,
15735 {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
15736 return Create64bitNode(ARMISD::VMLALVs, {A, B});
15737 if (IsVMLAV(MVT::i64, ISD::ZERO_EXTEND,
15738 {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A, B))
15739 return Create64bitNode(ARMISD::VMLALVu, {A, B});
15740 if (IsVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B))
15741 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15742 DAG.getNode(ARMISD::VMLAVs, dl, MVT::i32, A, B));
15743 if (IsVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B))
15744 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15745 DAG.getNode(ARMISD::VMLAVu, dl, MVT::i32, A, B));
15746
15747 if (IsPredVMLAV(MVT::i32, ISD::SIGN_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask))
15748 return DAG.getNode(ARMISD::VMLAVps, dl, ResVT, A, B, Mask);
15749 if (IsPredVMLAV(MVT::i32, ISD::ZERO_EXTEND, {MVT::v8i16, MVT::v16i8}, A, B, Mask))
15750 return DAG.getNode(ARMISD::VMLAVpu, dl, ResVT, A, B, Mask);
15751 if (IsPredVMLAV(MVT::i64, ISD::SIGN_EXTEND,
15752 {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A,
15753 B, Mask))
15754 return Create64bitNode(ARMISD::VMLALVps, {A, B, Mask});
15755 if (IsPredVMLAV(MVT::i64, ISD::ZERO_EXTEND,
15756 {MVT::v8i8, MVT::v8i16, MVT::v4i8, MVT::v4i16, MVT::v4i32}, A,
15757 B, Mask))
15758 return Create64bitNode(ARMISD::VMLALVpu, {A, B, Mask});
15759 if (IsPredVMLAV(MVT::i16, ISD::SIGN_EXTEND, {MVT::v16i8}, A, B, Mask))
15760 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15761 DAG.getNode(ARMISD::VMLAVps, dl, MVT::i32, A, B, Mask));
15762 if (IsPredVMLAV(MVT::i16, ISD::ZERO_EXTEND, {MVT::v16i8}, A, B, Mask))
15763 return DAG.getNode(ISD::TRUNCATE, dl, ResVT,
15764 DAG.getNode(ARMISD::VMLAVpu, dl, MVT::i32, A, B, Mask));
15765
15766 // Some complications. We can get a case where the two inputs of the mul are
15767 // the same, then the output sext will have been helpfully converted to a
15768 // zext. Turn it back.
15769 SDValue Op = N0;
15770 if (Op->getOpcode() == ISD::VSELECT)
15771 Op = Op->getOperand(1);
15772 if (Op->getOpcode() == ISD::ZERO_EXTEND &&
15773 Op->getOperand(0)->getOpcode() == ISD::MUL) {
15774 SDValue Mul = Op->getOperand(0);
15775 if (Mul->getOperand(0) == Mul->getOperand(1) &&
15776 Mul->getOperand(0)->getOpcode() == ISD::SIGN_EXTEND) {
15777 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, N0->getValueType(0), Mul);
15778 if (Op != N0)
15779 Ext = DAG.getNode(ISD::VSELECT, dl, N0->getValueType(0),
15780 N0->getOperand(0), Ext, N0->getOperand(2));
15781 return DAG.getNode(ISD::VECREDUCE_ADD, dl, ResVT, Ext);
15782 }
15783 }
15784
15785 return SDValue();
15786}
15787
15788static SDValue PerformVMOVNCombine(SDNode *N,
15789 TargetLowering::DAGCombinerInfo &DCI) {
15790 SDValue Op0 = N->getOperand(0);
15791 SDValue Op1 = N->getOperand(1);
15792 unsigned IsTop = N->getConstantOperandVal(2);
15793
15794 // VMOVNT a undef -> a
15795 // VMOVNB a undef -> a
15796 // VMOVNB undef a -> a
15797 if (Op1->isUndef())
15798 return Op0;
15799 if (Op0->isUndef() && !IsTop)
15800 return Op1;
15801
15802 // VMOVNt(c, VQMOVNb(a, b)) => VQMOVNt(c, b)
15803 // VMOVNb(c, VQMOVNb(a, b)) => VQMOVNb(c, b)
15804 if ((Op1->getOpcode() == ARMISD::VQMOVNs ||
15805 Op1->getOpcode() == ARMISD::VQMOVNu) &&
15806 Op1->getConstantOperandVal(2) == 0)
15807 return DCI.DAG.getNode(Op1->getOpcode(), SDLoc(Op1), N->getValueType(0),
15808 Op0, Op1->getOperand(1), N->getOperand(2));
15809
15810 // Only the bottom lanes from Qm (Op1) and either the top or bottom lanes from
15811 // Qd (Op0) are demanded from a VMOVN, depending on whether we are inserting
15812 // into the top or bottom lanes.
15813 unsigned NumElts = N->getValueType(0).getVectorNumElements();
15814 APInt Op1DemandedElts = APInt::getSplat(NumElts, APInt::getLowBitsSet(2, 1));
15815 APInt Op0DemandedElts =
15816 IsTop ? Op1DemandedElts
15817 : APInt::getSplat(NumElts, APInt::getHighBitsSet(2, 1));
15818
15819 APInt KnownUndef, KnownZero;
15820 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15821 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, KnownUndef,
15822 KnownZero, DCI))
15823 return SDValue(N, 0);
15824 if (TLI.SimplifyDemandedVectorElts(Op1, Op1DemandedElts, KnownUndef,
15825 KnownZero, DCI))
15826 return SDValue(N, 0);
15827
15828 return SDValue();
15829}
15830
15831static SDValue PerformVQMOVNCombine(SDNode *N,
15832 TargetLowering::DAGCombinerInfo &DCI) {
15833 SDValue Op0 = N->getOperand(0);
15834 unsigned IsTop = N->getConstantOperandVal(2);
15835
15836 unsigned NumElts = N->getValueType(0).getVectorNumElements();
15837 APInt Op0DemandedElts =
15838 APInt::getSplat(NumElts, IsTop ? APInt::getLowBitsSet(2, 1)
15839 : APInt::getHighBitsSet(2, 1));
15840
15841 APInt KnownUndef, KnownZero;
15842 const TargetLowering &TLI = DCI.DAG.getTargetLoweringInfo();
15843 if (TLI.SimplifyDemandedVectorElts(Op0, Op0DemandedElts, KnownUndef,
15844 KnownZero, DCI))
15845 return SDValue(N, 0);
15846 return SDValue();
15847}
15848
15849static SDValue PerformLongShiftCombine(SDNode *N, SelectionDAG &DAG) {
15850 SDLoc DL(N);
15851 SDValue Op0 = N->getOperand(0);
15852 SDValue Op1 = N->getOperand(1);
15853
15854 // Turn X << -C -> X >> C and viceversa. The negative shifts can come up from
15855 // uses of the intrinsics.
15856 if (auto C = dyn_cast<ConstantSDNode>(N->getOperand(2))) {
15857 int ShiftAmt = C->getSExtValue();
15858 if (ShiftAmt == 0) {
15859 SDValue Merge = DAG.getMergeValues({Op0, Op1}, DL);
15860 DAG.ReplaceAllUsesWith(N, Merge.getNode());
15861 return SDValue();
15862 }
15863
15864 if (ShiftAmt >= -32 && ShiftAmt < 0) {
15865 unsigned NewOpcode =
15866 N->getOpcode() == ARMISD::LSLL ? ARMISD::LSRL : ARMISD::LSLL;
15867 SDValue NewShift = DAG.getNode(NewOpcode, DL, N->getVTList(), Op0, Op1,
15868 DAG.getConstant(-ShiftAmt, DL, MVT::i32));
15869 DAG.ReplaceAllUsesWith(N, NewShift.getNode());
15870 return NewShift;
15871 }
15872 }
15873
15874 return SDValue();
15875}
15876
15877/// PerformIntrinsicCombine - ARM-specific DAG combining for intrinsics.
15878SDValue ARMTargetLowering::PerformIntrinsicCombine(SDNode *N,
15879 DAGCombinerInfo &DCI) const {
15880 SelectionDAG &DAG = DCI.DAG;
15881 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
15882 switch (IntNo) {
15883 default:
15884 // Don't do anything for most intrinsics.
15885 break;
15886
15887 // Vector shifts: check for immediate versions and lower them.
15888 // Note: This is done during DAG combining instead of DAG legalizing because
15889 // the build_vectors for 64-bit vector element shift counts are generally
15890 // not legal, and it is hard to see their values after they get legalized to
15891 // loads from a constant pool.
15892 case Intrinsic::arm_neon_vshifts:
15893 case Intrinsic::arm_neon_vshiftu:
15894 case Intrinsic::arm_neon_vrshifts:
15895 case Intrinsic::arm_neon_vrshiftu:
15896 case Intrinsic::arm_neon_vrshiftn:
15897 case Intrinsic::arm_neon_vqshifts:
15898 case Intrinsic::arm_neon_vqshiftu:
15899 case Intrinsic::arm_neon_vqshiftsu:
15900 case Intrinsic::arm_neon_vqshiftns:
15901 case Intrinsic::arm_neon_vqshiftnu:
15902 case Intrinsic::arm_neon_vqshiftnsu:
15903 case Intrinsic::arm_neon_vqrshiftns:
15904 case Intrinsic::arm_neon_vqrshiftnu:
15905 case Intrinsic::arm_neon_vqrshiftnsu: {
15906 EVT VT = N->getOperand(1).getValueType();
15907 int64_t Cnt;
15908 unsigned VShiftOpc = 0;
15909
15910 switch (IntNo) {
15911 case Intrinsic::arm_neon_vshifts:
15912 case Intrinsic::arm_neon_vshiftu:
15913 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt)) {
15914 VShiftOpc = ARMISD::VSHLIMM;
15915 break;
15916 }
15917 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt)) {
15918 VShiftOpc = (IntNo == Intrinsic::arm_neon_vshifts ? ARMISD::VSHRsIMM
15919 : ARMISD::VSHRuIMM);
15920 break;
15921 }
15922 return SDValue();
15923
15924 case Intrinsic::arm_neon_vrshifts:
15925 case Intrinsic::arm_neon_vrshiftu:
15926 if (isVShiftRImm(N->getOperand(2), VT, false, true, Cnt))
15927 break;
15928 return SDValue();
15929
15930 case Intrinsic::arm_neon_vqshifts:
15931 case Intrinsic::arm_neon_vqshiftu:
15932 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
15933 break;
15934 return SDValue();
15935
15936 case Intrinsic::arm_neon_vqshiftsu:
15937 if (isVShiftLImm(N->getOperand(2), VT, false, Cnt))
15938 break;
15939 llvm_unreachable("invalid shift count for vqshlu intrinsic")::llvm::llvm_unreachable_internal("invalid shift count for vqshlu intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15939)
;
15940
15941 case Intrinsic::arm_neon_vrshiftn:
15942 case Intrinsic::arm_neon_vqshiftns:
15943 case Intrinsic::arm_neon_vqshiftnu:
15944 case Intrinsic::arm_neon_vqshiftnsu:
15945 case Intrinsic::arm_neon_vqrshiftns:
15946 case Intrinsic::arm_neon_vqrshiftnu:
15947 case Intrinsic::arm_neon_vqrshiftnsu:
15948 // Narrowing shifts require an immediate right shift.
15949 if (isVShiftRImm(N->getOperand(2), VT, true, true, Cnt))
15950 break;
15951 llvm_unreachable("invalid shift count for narrowing vector shift "::llvm::llvm_unreachable_internal("invalid shift count for narrowing vector shift "
"intrinsic", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15952)
15952 "intrinsic")::llvm::llvm_unreachable_internal("invalid shift count for narrowing vector shift "
"intrinsic", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15952)
;
15953
15954 default:
15955 llvm_unreachable("unhandled vector shift")::llvm::llvm_unreachable_internal("unhandled vector shift", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 15955)
;
15956 }
15957
15958 switch (IntNo) {
15959 case Intrinsic::arm_neon_vshifts:
15960 case Intrinsic::arm_neon_vshiftu:
15961 // Opcode already set above.
15962 break;
15963 case Intrinsic::arm_neon_vrshifts:
15964 VShiftOpc = ARMISD::VRSHRsIMM;
15965 break;
15966 case Intrinsic::arm_neon_vrshiftu:
15967 VShiftOpc = ARMISD::VRSHRuIMM;
15968 break;
15969 case Intrinsic::arm_neon_vrshiftn:
15970 VShiftOpc = ARMISD::VRSHRNIMM;
15971 break;
15972 case Intrinsic::arm_neon_vqshifts:
15973 VShiftOpc = ARMISD::VQSHLsIMM;
15974 break;
15975 case Intrinsic::arm_neon_vqshiftu:
15976 VShiftOpc = ARMISD::VQSHLuIMM;
15977 break;
15978 case Intrinsic::arm_neon_vqshiftsu:
15979 VShiftOpc = ARMISD::VQSHLsuIMM;
15980 break;
15981 case Intrinsic::arm_neon_vqshiftns:
15982 VShiftOpc = ARMISD::VQSHRNsIMM;
15983 break;
15984 case Intrinsic::arm_neon_vqshiftnu:
15985 VShiftOpc = ARMISD::VQSHRNuIMM;
15986 break;
15987 case Intrinsic::arm_neon_vqshiftnsu:
15988 VShiftOpc = ARMISD::VQSHRNsuIMM;
15989 break;
15990 case Intrinsic::arm_neon_vqrshiftns:
15991 VShiftOpc = ARMISD::VQRSHRNsIMM;
15992 break;
15993 case Intrinsic::arm_neon_vqrshiftnu:
15994 VShiftOpc = ARMISD::VQRSHRNuIMM;
15995 break;
15996 case Intrinsic::arm_neon_vqrshiftnsu:
15997 VShiftOpc = ARMISD::VQRSHRNsuIMM;
15998 break;
15999 }
16000
16001 SDLoc dl(N);
16002 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
16003 N->getOperand(1), DAG.getConstant(Cnt, dl, MVT::i32));
16004 }
16005
16006 case Intrinsic::arm_neon_vshiftins: {
16007 EVT VT = N->getOperand(1).getValueType();
16008 int64_t Cnt;
16009 unsigned VShiftOpc = 0;
16010
16011 if (isVShiftLImm(N->getOperand(3), VT, false, Cnt))
16012 VShiftOpc = ARMISD::VSLIIMM;
16013 else if (isVShiftRImm(N->getOperand(3), VT, false, true, Cnt))
16014 VShiftOpc = ARMISD::VSRIIMM;
16015 else {
16016 llvm_unreachable("invalid shift count for vsli/vsri intrinsic")::llvm::llvm_unreachable_internal("invalid shift count for vsli/vsri intrinsic"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16016)
;
16017 }
16018
16019 SDLoc dl(N);
16020 return DAG.getNode(VShiftOpc, dl, N->getValueType(0),
16021 N->getOperand(1), N->getOperand(2),
16022 DAG.getConstant(Cnt, dl, MVT::i32));
16023 }
16024
16025 case Intrinsic::arm_neon_vqrshifts:
16026 case Intrinsic::arm_neon_vqrshiftu:
16027 // No immediate versions of these to check for.
16028 break;
16029
16030 case Intrinsic::arm_mve_vqdmlah:
16031 case Intrinsic::arm_mve_vqdmlash:
16032 case Intrinsic::arm_mve_vqrdmlah:
16033 case Intrinsic::arm_mve_vqrdmlash:
16034 case Intrinsic::arm_mve_vmla_n_predicated:
16035 case Intrinsic::arm_mve_vmlas_n_predicated:
16036 case Intrinsic::arm_mve_vqdmlah_predicated:
16037 case Intrinsic::arm_mve_vqdmlash_predicated:
16038 case Intrinsic::arm_mve_vqrdmlah_predicated:
16039 case Intrinsic::arm_mve_vqrdmlash_predicated: {
16040 // These intrinsics all take an i32 scalar operand which is narrowed to the
16041 // size of a single lane of the vector type they return. So we don't need
16042 // any bits of that operand above that point, which allows us to eliminate
16043 // uxth/sxth.
16044 unsigned BitWidth = N->getValueType(0).getScalarSizeInBits();
16045 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
16046 if (SimplifyDemandedBits(N->getOperand(3), DemandedMask, DCI))
16047 return SDValue();
16048 break;
16049 }
16050
16051 case Intrinsic::arm_mve_minv:
16052 case Intrinsic::arm_mve_maxv:
16053 case Intrinsic::arm_mve_minav:
16054 case Intrinsic::arm_mve_maxav:
16055 case Intrinsic::arm_mve_minv_predicated:
16056 case Intrinsic::arm_mve_maxv_predicated:
16057 case Intrinsic::arm_mve_minav_predicated:
16058 case Intrinsic::arm_mve_maxav_predicated: {
16059 // These intrinsics all take an i32 scalar operand which is narrowed to the
16060 // size of a single lane of the vector type they take as the other input.
16061 unsigned BitWidth = N->getOperand(2)->getValueType(0).getScalarSizeInBits();
16062 APInt DemandedMask = APInt::getLowBitsSet(32, BitWidth);
16063 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
16064 return SDValue();
16065 break;
16066 }
16067
16068 case Intrinsic::arm_mve_addv: {
16069 // Turn this intrinsic straight into the appropriate ARMISD::VADDV node,
16070 // which allow PerformADDVecReduce to turn it into VADDLV when possible.
16071 bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
16072 unsigned Opc = Unsigned ? ARMISD::VADDVu : ARMISD::VADDVs;
16073 return DAG.getNode(Opc, SDLoc(N), N->getVTList(), N->getOperand(1));
16074 }
16075
16076 case Intrinsic::arm_mve_addlv:
16077 case Intrinsic::arm_mve_addlv_predicated: {
16078 // Same for these, but ARMISD::VADDLV has to be followed by a BUILD_PAIR
16079 // which recombines the two outputs into an i64
16080 bool Unsigned = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
16081 unsigned Opc = IntNo == Intrinsic::arm_mve_addlv ?
16082 (Unsigned ? ARMISD::VADDLVu : ARMISD::VADDLVs) :
16083 (Unsigned ? ARMISD::VADDLVpu : ARMISD::VADDLVps);
16084
16085 SmallVector<SDValue, 4> Ops;
16086 for (unsigned i = 1, e = N->getNumOperands(); i < e; i++)
16087 if (i != 2) // skip the unsigned flag
16088 Ops.push_back(N->getOperand(i));
16089
16090 SDLoc dl(N);
16091 SDValue val = DAG.getNode(Opc, dl, {MVT::i32, MVT::i32}, Ops);
16092 return DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, val.getValue(0),
16093 val.getValue(1));
16094 }
16095 }
16096
16097 return SDValue();
16098}
16099
16100/// PerformShiftCombine - Checks for immediate versions of vector shifts and
16101/// lowers them. As with the vector shift intrinsics, this is done during DAG
16102/// combining instead of DAG legalizing because the build_vectors for 64-bit
16103/// vector element shift counts are generally not legal, and it is hard to see
16104/// their values after they get legalized to loads from a constant pool.
16105static SDValue PerformShiftCombine(SDNode *N,
16106 TargetLowering::DAGCombinerInfo &DCI,
16107 const ARMSubtarget *ST) {
16108 SelectionDAG &DAG = DCI.DAG;
16109 EVT VT = N->getValueType(0);
16110 if (N->getOpcode() == ISD::SRL && VT == MVT::i32 && ST->hasV6Ops()) {
16111 // Canonicalize (srl (bswap x), 16) to (rotr (bswap x), 16) if the high
16112 // 16-bits of x is zero. This optimizes rev + lsr 16 to rev16.
16113 SDValue N1 = N->getOperand(1);
16114 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N1)) {
16115 SDValue N0 = N->getOperand(0);
16116 if (C->getZExtValue() == 16 && N0.getOpcode() == ISD::BSWAP &&
16117 DAG.MaskedValueIsZero(N0.getOperand(0),
16118 APInt::getHighBitsSet(32, 16)))
16119 return DAG.getNode(ISD::ROTR, SDLoc(N), VT, N0, N1);
16120 }
16121 }
16122
16123 if (ST->isThumb1Only() && N->getOpcode() == ISD::SHL && VT == MVT::i32 &&
16124 N->getOperand(0)->getOpcode() == ISD::AND &&
16125 N->getOperand(0)->hasOneUse()) {
16126 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
16127 return SDValue();
16128 // Look for the pattern (shl (and x, AndMask), ShiftAmt). This doesn't
16129 // usually show up because instcombine prefers to canonicalize it to
16130 // (and (shl x, ShiftAmt) (shl AndMask, ShiftAmt)), but the shift can come
16131 // out of GEP lowering in some cases.
16132 SDValue N0 = N->getOperand(0);
16133 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
16134 if (!ShiftAmtNode)
16135 return SDValue();
16136 uint32_t ShiftAmt = static_cast<uint32_t>(ShiftAmtNode->getZExtValue());
16137 ConstantSDNode *AndMaskNode = dyn_cast<ConstantSDNode>(N0->getOperand(1));
16138 if (!AndMaskNode)
16139 return SDValue();
16140 uint32_t AndMask = static_cast<uint32_t>(AndMaskNode->getZExtValue());
16141 // Don't transform uxtb/uxth.
16142 if (AndMask == 255 || AndMask == 65535)
16143 return SDValue();
16144 if (isMask_32(AndMask)) {
16145 uint32_t MaskedBits = countLeadingZeros(AndMask);
16146 if (MaskedBits > ShiftAmt) {
16147 SDLoc DL(N);
16148 SDValue SHL = DAG.getNode(ISD::SHL, DL, MVT::i32, N0->getOperand(0),
16149 DAG.getConstant(MaskedBits, DL, MVT::i32));
16150 return DAG.getNode(
16151 ISD::SRL, DL, MVT::i32, SHL,
16152 DAG.getConstant(MaskedBits - ShiftAmt, DL, MVT::i32));
16153 }
16154 }
16155 }
16156
16157 // Nothing to be done for scalar shifts.
16158 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16159 if (!VT.isVector() || !TLI.isTypeLegal(VT))
16160 return SDValue();
16161 if (ST->hasMVEIntegerOps() && VT == MVT::v2i64)
16162 return SDValue();
16163
16164 int64_t Cnt;
16165
16166 switch (N->getOpcode()) {
16167 default: llvm_unreachable("unexpected shift opcode")::llvm::llvm_unreachable_internal("unexpected shift opcode", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16167)
;
16168
16169 case ISD::SHL:
16170 if (isVShiftLImm(N->getOperand(1), VT, false, Cnt)) {
16171 SDLoc dl(N);
16172 return DAG.getNode(ARMISD::VSHLIMM, dl, VT, N->getOperand(0),
16173 DAG.getConstant(Cnt, dl, MVT::i32));
16174 }
16175 break;
16176
16177 case ISD::SRA:
16178 case ISD::SRL:
16179 if (isVShiftRImm(N->getOperand(1), VT, false, false, Cnt)) {
16180 unsigned VShiftOpc =
16181 (N->getOpcode() == ISD::SRA ? ARMISD::VSHRsIMM : ARMISD::VSHRuIMM);
16182 SDLoc dl(N);
16183 return DAG.getNode(VShiftOpc, dl, VT, N->getOperand(0),
16184 DAG.getConstant(Cnt, dl, MVT::i32));
16185 }
16186 }
16187 return SDValue();
16188}
16189
16190// Look for a sign/zero/fpextend extend of a larger than legal load. This can be
16191// split into multiple extending loads, which are simpler to deal with than an
16192// arbitrary extend. For fp extends we use an integer extending load and a VCVTL
16193// to convert the type to an f32.
16194static SDValue PerformSplittingToWideningLoad(SDNode *N, SelectionDAG &DAG) {
16195 SDValue N0 = N->getOperand(0);
16196 if (N0.getOpcode() != ISD::LOAD)
16197 return SDValue();
16198 LoadSDNode *LD = cast<LoadSDNode>(N0.getNode());
16199 if (!LD->isSimple() || !N0.hasOneUse() || LD->isIndexed() ||
16200 LD->getExtensionType() != ISD::NON_EXTLOAD)
16201 return SDValue();
16202 EVT FromVT = LD->getValueType(0);
16203 EVT ToVT = N->getValueType(0);
16204 if (!ToVT.isVector())
16205 return SDValue();
16206 assert(FromVT.getVectorNumElements() == ToVT.getVectorNumElements())(static_cast <bool> (FromVT.getVectorNumElements() == ToVT
.getVectorNumElements()) ? void (0) : __assert_fail ("FromVT.getVectorNumElements() == ToVT.getVectorNumElements()"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16206, __extension__ __PRETTY_FUNCTION__))
;
16207 EVT ToEltVT = ToVT.getVectorElementType();
16208 EVT FromEltVT = FromVT.getVectorElementType();
16209
16210 unsigned NumElements = 0;
16211 if (ToEltVT == MVT::i32 && (FromEltVT == MVT::i16 || FromEltVT == MVT::i8))
16212 NumElements = 4;
16213 if (ToEltVT == MVT::i16 && FromEltVT == MVT::i8)
16214 NumElements = 8;
16215 if (ToEltVT == MVT::f32 && FromEltVT == MVT::f16)
16216 NumElements = 4;
16217 if (NumElements == 0 ||
16218 (FromEltVT != MVT::f16 && FromVT.getVectorNumElements() == NumElements) ||
16219 FromVT.getVectorNumElements() % NumElements != 0 ||
16220 !isPowerOf2_32(NumElements))
16221 return SDValue();
16222
16223 LLVMContext &C = *DAG.getContext();
16224 SDLoc DL(LD);
16225 // Details about the old load
16226 SDValue Ch = LD->getChain();
16227 SDValue BasePtr = LD->getBasePtr();
16228 Align Alignment = LD->getOriginalAlign();
16229 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
16230 AAMDNodes AAInfo = LD->getAAInfo();
16231
16232 ISD::LoadExtType NewExtType =
16233 N->getOpcode() == ISD::SIGN_EXTEND ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
16234 SDValue Offset = DAG.getUNDEF(BasePtr.getValueType());
16235 EVT NewFromVT = EVT::getVectorVT(
16236 C, EVT::getIntegerVT(C, FromEltVT.getScalarSizeInBits()), NumElements);
16237 EVT NewToVT = EVT::getVectorVT(
16238 C, EVT::getIntegerVT(C, ToEltVT.getScalarSizeInBits()), NumElements);
16239
16240 SmallVector<SDValue, 4> Loads;
16241 SmallVector<SDValue, 4> Chains;
16242 for (unsigned i = 0; i < FromVT.getVectorNumElements() / NumElements; i++) {
16243 unsigned NewOffset = (i * NewFromVT.getSizeInBits()) / 8;
16244 SDValue NewPtr =
16245 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::Fixed(NewOffset));
16246
16247 SDValue NewLoad =
16248 DAG.getLoad(ISD::UNINDEXED, NewExtType, NewToVT, DL, Ch, NewPtr, Offset,
16249 LD->getPointerInfo().getWithOffset(NewOffset), NewFromVT,
16250 Alignment, MMOFlags, AAInfo);
16251 Loads.push_back(NewLoad);
16252 Chains.push_back(SDValue(NewLoad.getNode(), 1));
16253 }
16254
16255 // Float truncs need to extended with VCVTB's into their floating point types.
16256 if (FromEltVT == MVT::f16) {
16257 SmallVector<SDValue, 4> Extends;
16258
16259 for (unsigned i = 0; i < Loads.size(); i++) {
16260 SDValue LoadBC =
16261 DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, MVT::v8f16, Loads[i]);
16262 SDValue FPExt = DAG.getNode(ARMISD::VCVTL, DL, MVT::v4f32, LoadBC,
16263 DAG.getConstant(0, DL, MVT::i32));
16264 Extends.push_back(FPExt);
16265 }
16266
16267 Loads = Extends;
16268 }
16269
16270 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
16271 DAG.ReplaceAllUsesOfValueWith(SDValue(LD, 1), NewChain);
16272 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ToVT, Loads);
16273}
16274
16275/// PerformExtendCombine - Target-specific DAG combining for ISD::SIGN_EXTEND,
16276/// ISD::ZERO_EXTEND, and ISD::ANY_EXTEND.
16277static SDValue PerformExtendCombine(SDNode *N, SelectionDAG &DAG,
16278 const ARMSubtarget *ST) {
16279 SDValue N0 = N->getOperand(0);
16280
16281 // Check for sign- and zero-extensions of vector extract operations of 8- and
16282 // 16-bit vector elements. NEON and MVE support these directly. They are
16283 // handled during DAG combining because type legalization will promote them
16284 // to 32-bit types and it is messy to recognize the operations after that.
16285 if ((ST->hasNEON() || ST->hasMVEIntegerOps()) &&
16286 N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
16287 SDValue Vec = N0.getOperand(0);
16288 SDValue Lane = N0.getOperand(1);
16289 EVT VT = N->getValueType(0);
16290 EVT EltVT = N0.getValueType();
16291 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16292
16293 if (VT == MVT::i32 &&
16294 (EltVT == MVT::i8 || EltVT == MVT::i16) &&
16295 TLI.isTypeLegal(Vec.getValueType()) &&
16296 isa<ConstantSDNode>(Lane)) {
16297
16298 unsigned Opc = 0;
16299 switch (N->getOpcode()) {
16300 default: llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16300)
;
16301 case ISD::SIGN_EXTEND:
16302 Opc = ARMISD::VGETLANEs;
16303 break;
16304 case ISD::ZERO_EXTEND:
16305 case ISD::ANY_EXTEND:
16306 Opc = ARMISD::VGETLANEu;
16307 break;
16308 }
16309 return DAG.getNode(Opc, SDLoc(N), VT, Vec, Lane);
16310 }
16311 }
16312
16313 if (ST->hasMVEIntegerOps())
16314 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
16315 return NewLoad;
16316
16317 return SDValue();
16318}
16319
16320static SDValue PerformFPExtendCombine(SDNode *N, SelectionDAG &DAG,
16321 const ARMSubtarget *ST) {
16322 if (ST->hasMVEFloatOps())
16323 if (SDValue NewLoad = PerformSplittingToWideningLoad(N, DAG))
16324 return NewLoad;
16325
16326 return SDValue();
16327}
16328
16329/// PerformMinMaxCombine - Target-specific DAG combining for creating truncating
16330/// saturates.
16331static SDValue PerformMinMaxCombine(SDNode *N, SelectionDAG &DAG,
16332 const ARMSubtarget *ST) {
16333 EVT VT = N->getValueType(0);
16334 SDValue N0 = N->getOperand(0);
16335 if (!ST->hasMVEIntegerOps())
16336 return SDValue();
16337
16338 if (SDValue V = PerformVQDMULHCombine(N, DAG))
16339 return V;
16340
16341 if (VT != MVT::v4i32 && VT != MVT::v8i16)
16342 return SDValue();
16343
16344 auto IsSignedSaturate = [&](SDNode *Min, SDNode *Max) {
16345 // Check one is a smin and the other is a smax
16346 if (Min->getOpcode() != ISD::SMIN)
16347 std::swap(Min, Max);
16348 if (Min->getOpcode() != ISD::SMIN || Max->getOpcode() != ISD::SMAX)
16349 return false;
16350
16351 APInt SaturateC;
16352 if (VT == MVT::v4i32)
16353 SaturateC = APInt(32, (1 << 15) - 1, true);
16354 else //if (VT == MVT::v8i16)
16355 SaturateC = APInt(16, (1 << 7) - 1, true);
16356
16357 APInt MinC, MaxC;
16358 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) ||
16359 MinC != SaturateC)
16360 return false;
16361 if (!ISD::isConstantSplatVector(Max->getOperand(1).getNode(), MaxC) ||
16362 MaxC != ~SaturateC)
16363 return false;
16364 return true;
16365 };
16366
16367 if (IsSignedSaturate(N, N0.getNode())) {
16368 SDLoc DL(N);
16369 MVT ExtVT, HalfVT;
16370 if (VT == MVT::v4i32) {
16371 HalfVT = MVT::v8i16;
16372 ExtVT = MVT::v4i16;
16373 } else { // if (VT == MVT::v8i16)
16374 HalfVT = MVT::v16i8;
16375 ExtVT = MVT::v8i8;
16376 }
16377
16378 // Create a VQMOVNB with undef top lanes, then signed extended into the top
16379 // half. That extend will hopefully be removed if only the bottom bits are
16380 // demanded (though a truncating store, for example).
16381 SDValue VQMOVN =
16382 DAG.getNode(ARMISD::VQMOVNs, DL, HalfVT, DAG.getUNDEF(HalfVT),
16383 N0->getOperand(0), DAG.getConstant(0, DL, MVT::i32));
16384 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
16385 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Bitcast,
16386 DAG.getValueType(ExtVT));
16387 }
16388
16389 auto IsUnsignedSaturate = [&](SDNode *Min) {
16390 // For unsigned, we just need to check for <= 0xffff
16391 if (Min->getOpcode() != ISD::UMIN)
16392 return false;
16393
16394 APInt SaturateC;
16395 if (VT == MVT::v4i32)
16396 SaturateC = APInt(32, (1 << 16) - 1, true);
16397 else //if (VT == MVT::v8i16)
16398 SaturateC = APInt(16, (1 << 8) - 1, true);
16399
16400 APInt MinC;
16401 if (!ISD::isConstantSplatVector(Min->getOperand(1).getNode(), MinC) ||
16402 MinC != SaturateC)
16403 return false;
16404 return true;
16405 };
16406
16407 if (IsUnsignedSaturate(N)) {
16408 SDLoc DL(N);
16409 MVT HalfVT;
16410 unsigned ExtConst;
16411 if (VT == MVT::v4i32) {
16412 HalfVT = MVT::v8i16;
16413 ExtConst = 0x0000FFFF;
16414 } else { //if (VT == MVT::v8i16)
16415 HalfVT = MVT::v16i8;
16416 ExtConst = 0x00FF;
16417 }
16418
16419 // Create a VQMOVNB with undef top lanes, then ZExt into the top half with
16420 // an AND. That extend will hopefully be removed if only the bottom bits are
16421 // demanded (though a truncating store, for example).
16422 SDValue VQMOVN =
16423 DAG.getNode(ARMISD::VQMOVNu, DL, HalfVT, DAG.getUNDEF(HalfVT), N0,
16424 DAG.getConstant(0, DL, MVT::i32));
16425 SDValue Bitcast = DAG.getNode(ARMISD::VECTOR_REG_CAST, DL, VT, VQMOVN);
16426 return DAG.getNode(ISD::AND, DL, VT, Bitcast,
16427 DAG.getConstant(ExtConst, DL, VT));
16428 }
16429
16430 return SDValue();
16431}
16432
16433static const APInt *isPowerOf2Constant(SDValue V) {
16434 ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
16435 if (!C)
16436 return nullptr;
16437 const APInt *CV = &C->getAPIntValue();
16438 return CV->isPowerOf2() ? CV : nullptr;
16439}
16440
16441SDValue ARMTargetLowering::PerformCMOVToBFICombine(SDNode *CMOV, SelectionDAG &DAG) const {
16442 // If we have a CMOV, OR and AND combination such as:
16443 // if (x & CN)
16444 // y |= CM;
16445 //
16446 // And:
16447 // * CN is a single bit;
16448 // * All bits covered by CM are known zero in y
16449 //
16450 // Then we can convert this into a sequence of BFI instructions. This will
16451 // always be a win if CM is a single bit, will always be no worse than the
16452 // TST&OR sequence if CM is two bits, and for thumb will be no worse if CM is
16453 // three bits (due to the extra IT instruction).
16454
16455 SDValue Op0 = CMOV->getOperand(0);
16456 SDValue Op1 = CMOV->getOperand(1);
16457 auto CCNode = cast<ConstantSDNode>(CMOV->getOperand(2));
16458 auto CC = CCNode->getAPIntValue().getLimitedValue();
16459 SDValue CmpZ = CMOV->getOperand(4);
16460
16461 // The compare must be against zero.
16462 if (!isNullConstant(CmpZ->getOperand(1)))
16463 return SDValue();
16464
16465 assert(CmpZ->getOpcode() == ARMISD::CMPZ)(static_cast <bool> (CmpZ->getOpcode() == ARMISD::CMPZ
) ? void (0) : __assert_fail ("CmpZ->getOpcode() == ARMISD::CMPZ"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16465, __extension__ __PRETTY_FUNCTION__))
;
16466 SDValue And = CmpZ->getOperand(0);
16467 if (And->getOpcode() != ISD::AND)
16468 return SDValue();
16469 const APInt *AndC = isPowerOf2Constant(And->getOperand(1));
16470 if (!AndC)
16471 return SDValue();
16472 SDValue X = And->getOperand(0);
16473
16474 if (CC == ARMCC::EQ) {
16475 // We're performing an "equal to zero" compare. Swap the operands so we
16476 // canonicalize on a "not equal to zero" compare.
16477 std::swap(Op0, Op1);
16478 } else {
16479 assert(CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?")(static_cast <bool> (CC == ARMCC::NE && "How can a CMPZ node not be EQ or NE?"
) ? void (0) : __assert_fail ("CC == ARMCC::NE && \"How can a CMPZ node not be EQ or NE?\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16479, __extension__ __PRETTY_FUNCTION__))
;
16480 }
16481
16482 if (Op1->getOpcode() != ISD::OR)
16483 return SDValue();
16484
16485 ConstantSDNode *OrC = dyn_cast<ConstantSDNode>(Op1->getOperand(1));
16486 if (!OrC)
16487 return SDValue();
16488 SDValue Y = Op1->getOperand(0);
16489
16490 if (Op0 != Y)
16491 return SDValue();
16492
16493 // Now, is it profitable to continue?
16494 APInt OrCI = OrC->getAPIntValue();
16495 unsigned Heuristic = Subtarget->isThumb() ? 3 : 2;
16496 if (OrCI.countPopulation() > Heuristic)
16497 return SDValue();
16498
16499 // Lastly, can we determine that the bits defined by OrCI
16500 // are zero in Y?
16501 KnownBits Known = DAG.computeKnownBits(Y);
16502 if ((OrCI & Known.Zero) != OrCI)
16503 return SDValue();
16504
16505 // OK, we can do the combine.
16506 SDValue V = Y;
16507 SDLoc dl(X);
16508 EVT VT = X.getValueType();
16509 unsigned BitInX = AndC->logBase2();
16510
16511 if (BitInX != 0) {
16512 // We must shift X first.
16513 X = DAG.getNode(ISD::SRL, dl, VT, X,
16514 DAG.getConstant(BitInX, dl, VT));
16515 }
16516
16517 for (unsigned BitInY = 0, NumActiveBits = OrCI.getActiveBits();
16518 BitInY < NumActiveBits; ++BitInY) {
16519 if (OrCI[BitInY] == 0)
16520 continue;
16521 APInt Mask(VT.getSizeInBits(), 0);
16522 Mask.setBit(BitInY);
16523 V = DAG.getNode(ARMISD::BFI, dl, VT, V, X,
16524 // Confusingly, the operand is an *inverted* mask.
16525 DAG.getConstant(~Mask, dl, VT));
16526 }
16527
16528 return V;
16529}
16530
16531// Given N, the value controlling the conditional branch, search for the loop
16532// intrinsic, returning it, along with how the value is used. We need to handle
16533// patterns such as the following:
16534// (brcond (xor (setcc (loop.decrement), 0, ne), 1), exit)
16535// (brcond (setcc (loop.decrement), 0, eq), exit)
16536// (brcond (setcc (loop.decrement), 0, ne), header)
16537static SDValue SearchLoopIntrinsic(SDValue N, ISD::CondCode &CC, int &Imm,
16538 bool &Negate) {
16539 switch (N->getOpcode()) {
16540 default:
16541 break;
16542 case ISD::XOR: {
16543 if (!isa<ConstantSDNode>(N.getOperand(1)))
16544 return SDValue();
16545 if (!cast<ConstantSDNode>(N.getOperand(1))->isOne())
16546 return SDValue();
16547 Negate = !Negate;
16548 return SearchLoopIntrinsic(N.getOperand(0), CC, Imm, Negate);
16549 }
16550 case ISD::SETCC: {
16551 auto *Const = dyn_cast<ConstantSDNode>(N.getOperand(1));
16552 if (!Const)
16553 return SDValue();
16554 if (Const->isNullValue())
16555 Imm = 0;
16556 else if (Const->isOne())
16557 Imm = 1;
16558 else
16559 return SDValue();
16560 CC = cast<CondCodeSDNode>(N.getOperand(2))->get();
16561 return SearchLoopIntrinsic(N->getOperand(0), CC, Imm, Negate);
16562 }
16563 case ISD::INTRINSIC_W_CHAIN: {
16564 unsigned IntOp = cast<ConstantSDNode>(N.getOperand(1))->getZExtValue();
16565 if (IntOp != Intrinsic::test_start_loop_iterations &&
16566 IntOp != Intrinsic::loop_decrement_reg)
16567 return SDValue();
16568 return N;
16569 }
16570 }
16571 return SDValue();
16572}
16573
16574static SDValue PerformHWLoopCombine(SDNode *N,
16575 TargetLowering::DAGCombinerInfo &DCI,
16576 const ARMSubtarget *ST) {
16577
16578 // The hwloop intrinsics that we're interested are used for control-flow,
16579 // either for entering or exiting the loop:
16580 // - test.start.loop.iterations will test whether its operand is zero. If it
16581 // is zero, the proceeding branch should not enter the loop.
16582 // - loop.decrement.reg also tests whether its operand is zero. If it is
16583 // zero, the proceeding branch should not branch back to the beginning of
16584 // the loop.
16585 // So here, we need to check that how the brcond is using the result of each
16586 // of the intrinsics to ensure that we're branching to the right place at the
16587 // right time.
16588
16589 ISD::CondCode CC;
16590 SDValue Cond;
16591 int Imm = 1;
16592 bool Negate = false;
16593 SDValue Chain = N->getOperand(0);
16594 SDValue Dest;
16595
16596 if (N->getOpcode() == ISD::BRCOND) {
16597 CC = ISD::SETEQ;
16598 Cond = N->getOperand(1);
16599 Dest = N->getOperand(2);
16600 } else {
16601 assert(N->getOpcode() == ISD::BR_CC && "Expected BRCOND or BR_CC!")(static_cast <bool> (N->getOpcode() == ISD::BR_CC &&
"Expected BRCOND or BR_CC!") ? void (0) : __assert_fail ("N->getOpcode() == ISD::BR_CC && \"Expected BRCOND or BR_CC!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16601, __extension__ __PRETTY_FUNCTION__))
;
16602 CC = cast<CondCodeSDNode>(N->getOperand(1))->get();
16603 Cond = N->getOperand(2);
16604 Dest = N->getOperand(4);
16605 if (auto *Const = dyn_cast<ConstantSDNode>(N->getOperand(3))) {
16606 if (!Const->isOne() && !Const->isNullValue())
16607 return SDValue();
16608 Imm = Const->getZExtValue();
16609 } else
16610 return SDValue();
16611 }
16612
16613 SDValue Int = SearchLoopIntrinsic(Cond, CC, Imm, Negate);
16614 if (!Int)
16615 return SDValue();
16616
16617 if (Negate)
16618 CC = ISD::getSetCCInverse(CC, /* Integer inverse */ MVT::i32);
16619
16620 auto IsTrueIfZero = [](ISD::CondCode CC, int Imm) {
16621 return (CC == ISD::SETEQ && Imm == 0) ||
16622 (CC == ISD::SETNE && Imm == 1) ||
16623 (CC == ISD::SETLT && Imm == 1) ||
16624 (CC == ISD::SETULT && Imm == 1);
16625 };
16626
16627 auto IsFalseIfZero = [](ISD::CondCode CC, int Imm) {
16628 return (CC == ISD::SETEQ && Imm == 1) ||
16629 (CC == ISD::SETNE && Imm == 0) ||
16630 (CC == ISD::SETGT && Imm == 0) ||
16631 (CC == ISD::SETUGT && Imm == 0) ||
16632 (CC == ISD::SETGE && Imm == 1) ||
16633 (CC == ISD::SETUGE && Imm == 1);
16634 };
16635
16636 assert((IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) &&(static_cast <bool> ((IsTrueIfZero(CC, Imm) || IsFalseIfZero
(CC, Imm)) && "unsupported condition") ? void (0) : __assert_fail
("(IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) && \"unsupported condition\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16637, __extension__ __PRETTY_FUNCTION__))
16637 "unsupported condition")(static_cast <bool> ((IsTrueIfZero(CC, Imm) || IsFalseIfZero
(CC, Imm)) && "unsupported condition") ? void (0) : __assert_fail
("(IsTrueIfZero(CC, Imm) || IsFalseIfZero(CC, Imm)) && \"unsupported condition\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16637, __extension__ __PRETTY_FUNCTION__))
;
16638
16639 SDLoc dl(Int);
16640 SelectionDAG &DAG = DCI.DAG;
16641 SDValue Elements = Int.getOperand(2);
16642 unsigned IntOp = cast<ConstantSDNode>(Int->getOperand(1))->getZExtValue();
16643 assert((N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR)(static_cast <bool> ((N->hasOneUse() && N->
use_begin()->getOpcode() == ISD::BR) && "expected single br user"
) ? void (0) : __assert_fail ("(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR) && \"expected single br user\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16644, __extension__ __PRETTY_FUNCTION__))
16644 && "expected single br user")(static_cast <bool> ((N->hasOneUse() && N->
use_begin()->getOpcode() == ISD::BR) && "expected single br user"
) ? void (0) : __assert_fail ("(N->hasOneUse() && N->use_begin()->getOpcode() == ISD::BR) && \"expected single br user\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 16644, __extension__ __PRETTY_FUNCTION__))
;
16645 SDNode *Br = *N->use_begin();
16646 SDValue OtherTarget = Br->getOperand(1);
16647
16648 // Update the unconditional branch to branch to the given Dest.
16649 auto UpdateUncondBr = [](SDNode *Br, SDValue Dest, SelectionDAG &DAG) {
16650 SDValue NewBrOps[] = { Br->getOperand(0), Dest };
16651 SDValue NewBr = DAG.getNode(ISD::BR, SDLoc(Br), MVT::Other, NewBrOps);
16652 DAG.ReplaceAllUsesOfValueWith(SDValue(Br, 0), NewBr);
16653 };
16654
16655 if (IntOp == Intrinsic::test_start_loop_iterations) {
16656 SDValue Res;
16657 SDValue Setup = DAG.getNode(ARMISD::WLSSETUP, dl, MVT::i32, Elements);
16658 // We expect this 'instruction' to branch when the counter is zero.
16659 if (IsTrueIfZero(CC, Imm)) {
16660 SDValue Ops[] = {Chain, Setup, Dest};
16661 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
16662 } else {
16663 // The logic is the reverse of what we need for WLS, so find the other
16664 // basic block target: the target of the proceeding br.
16665 UpdateUncondBr(Br, Dest, DAG);
16666
16667 SDValue Ops[] = {Chain, Setup, OtherTarget};
16668 Res = DAG.getNode(ARMISD::WLS, dl, MVT::Other, Ops);
16669 }
16670 // Update LR count to the new value
16671 DAG.ReplaceAllUsesOfValueWith(Int.getValue(0), Setup);
16672 // Update chain
16673 DAG.ReplaceAllUsesOfValueWith(Int.getValue(2), Int.getOperand(0));
16674 return Res;
16675 } else {
16676 SDValue Size = DAG.getTargetConstant(
16677 cast<ConstantSDNode>(Int.getOperand(3))->getZExtValue(), dl, MVT::i32);
16678 SDValue Args[] = { Int.getOperand(0), Elements, Size, };
16679 SDValue LoopDec = DAG.getNode(ARMISD::LOOP_DEC, dl,
16680 DAG.getVTList(MVT::i32, MVT::Other), Args);
16681 DAG.ReplaceAllUsesWith(Int.getNode(), LoopDec.getNode());
16682
16683 // We expect this instruction to branch when the count is not zero.
16684 SDValue Target = IsFalseIfZero(CC, Imm) ? Dest : OtherTarget;
16685
16686 // Update the unconditional branch to target the loop preheader if we've
16687 // found the condition has been reversed.
16688 if (Target == OtherTarget)
16689 UpdateUncondBr(Br, Dest, DAG);
16690
16691 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
16692 SDValue(LoopDec.getNode(), 1), Chain);
16693
16694 SDValue EndArgs[] = { Chain, SDValue(LoopDec.getNode(), 0), Target };
16695 return DAG.getNode(ARMISD::LE, dl, MVT::Other, EndArgs);
16696 }
16697 return SDValue();
16698}
16699
16700/// PerformBRCONDCombine - Target-specific DAG combining for ARMISD::BRCOND.
16701SDValue
16702ARMTargetLowering::PerformBRCONDCombine(SDNode *N, SelectionDAG &DAG) const {
16703 SDValue Cmp = N->getOperand(4);
16704 if (Cmp.getOpcode() != ARMISD::CMPZ)
16705 // Only looking at NE cases.
16706 return SDValue();
16707
16708 EVT VT = N->getValueType(0);
16709 SDLoc dl(N);
16710 SDValue LHS = Cmp.getOperand(0);
16711 SDValue RHS = Cmp.getOperand(1);
16712 SDValue Chain = N->getOperand(0);
16713 SDValue BB = N->getOperand(1);
16714 SDValue ARMcc = N->getOperand(2);
16715 ARMCC::CondCodes CC =
16716 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
16717
16718 // (brcond Chain BB ne CPSR (cmpz (and (cmov 0 1 CC CPSR Cmp) 1) 0))
16719 // -> (brcond Chain BB CC CPSR Cmp)
16720 if (CC == ARMCC::NE && LHS.getOpcode() == ISD::AND && LHS->hasOneUse() &&
16721 LHS->getOperand(0)->getOpcode() == ARMISD::CMOV &&
16722 LHS->getOperand(0)->hasOneUse()) {
16723 auto *LHS00C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(0));
16724 auto *LHS01C = dyn_cast<ConstantSDNode>(LHS->getOperand(0)->getOperand(1));
16725 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
16726 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
16727 if ((LHS00C && LHS00C->getZExtValue() == 0) &&
16728 (LHS01C && LHS01C->getZExtValue() == 1) &&
16729 (LHS1C && LHS1C->getZExtValue() == 1) &&
16730 (RHSC && RHSC->getZExtValue() == 0)) {
16731 return DAG.getNode(
16732 ARMISD::BRCOND, dl, VT, Chain, BB, LHS->getOperand(0)->getOperand(2),
16733 LHS->getOperand(0)->getOperand(3), LHS->getOperand(0)->getOperand(4));
16734 }
16735 }
16736
16737 return SDValue();
16738}
16739
16740/// PerformCMOVCombine - Target-specific DAG combining for ARMISD::CMOV.
16741SDValue
16742ARMTargetLowering::PerformCMOVCombine(SDNode *N, SelectionDAG &DAG) const {
16743 SDValue Cmp = N->getOperand(4);
16744 if (Cmp.getOpcode() != ARMISD::CMPZ)
16745 // Only looking at EQ and NE cases.
16746 return SDValue();
16747
16748 EVT VT = N->getValueType(0);
16749 SDLoc dl(N);
16750 SDValue LHS = Cmp.getOperand(0);
16751 SDValue RHS = Cmp.getOperand(1);
16752 SDValue FalseVal = N->getOperand(0);
16753 SDValue TrueVal = N->getOperand(1);
16754 SDValue ARMcc = N->getOperand(2);
16755 ARMCC::CondCodes CC =
16756 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
16757
16758 // BFI is only available on V6T2+.
16759 if (!Subtarget->isThumb1Only() && Subtarget->hasV6T2Ops()) {
16760 SDValue R = PerformCMOVToBFICombine(N, DAG);
16761 if (R)
16762 return R;
16763 }
16764
16765 // Simplify
16766 // mov r1, r0
16767 // cmp r1, x
16768 // mov r0, y
16769 // moveq r0, x
16770 // to
16771 // cmp r0, x
16772 // movne r0, y
16773 //
16774 // mov r1, r0
16775 // cmp r1, x
16776 // mov r0, x
16777 // movne r0, y
16778 // to
16779 // cmp r0, x
16780 // movne r0, y
16781 /// FIXME: Turn this into a target neutral optimization?
16782 SDValue Res;
16783 if (CC == ARMCC::NE && FalseVal == RHS && FalseVal != LHS) {
16784 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, TrueVal, ARMcc,
16785 N->getOperand(3), Cmp);
16786 } else if (CC == ARMCC::EQ && TrueVal == RHS) {
16787 SDValue ARMcc;
16788 SDValue NewCmp = getARMCmp(LHS, RHS, ISD::SETNE, ARMcc, DAG, dl);
16789 Res = DAG.getNode(ARMISD::CMOV, dl, VT, LHS, FalseVal, ARMcc,
16790 N->getOperand(3), NewCmp);
16791 }
16792
16793 // (cmov F T ne CPSR (cmpz (cmov 0 1 CC CPSR Cmp) 0))
16794 // -> (cmov F T CC CPSR Cmp)
16795 if (CC == ARMCC::NE && LHS.getOpcode() == ARMISD::CMOV && LHS->hasOneUse()) {
16796 auto *LHS0C = dyn_cast<ConstantSDNode>(LHS->getOperand(0));
16797 auto *LHS1C = dyn_cast<ConstantSDNode>(LHS->getOperand(1));
16798 auto *RHSC = dyn_cast<ConstantSDNode>(RHS);
16799 if ((LHS0C && LHS0C->getZExtValue() == 0) &&
16800 (LHS1C && LHS1C->getZExtValue() == 1) &&
16801 (RHSC && RHSC->getZExtValue() == 0)) {
16802 return DAG.getNode(ARMISD::CMOV, dl, VT, FalseVal, TrueVal,
16803 LHS->getOperand(2), LHS->getOperand(3),
16804 LHS->getOperand(4));
16805 }
16806 }
16807
16808 if (!VT.isInteger())
16809 return SDValue();
16810
16811 // Materialize a boolean comparison for integers so we can avoid branching.
16812 if (isNullConstant(FalseVal)) {
16813 if (CC == ARMCC::EQ && isOneConstant(TrueVal)) {
16814 if (!Subtarget->isThumb1Only() && Subtarget->hasV5TOps()) {
16815 // If x == y then x - y == 0 and ARM's CLZ will return 32, shifting it
16816 // right 5 bits will make that 32 be 1, otherwise it will be 0.
16817 // CMOV 0, 1, ==, (CMPZ x, y) -> SRL (CTLZ (SUB x, y)), 5
16818 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
16819 Res = DAG.getNode(ISD::SRL, dl, VT, DAG.getNode(ISD::CTLZ, dl, VT, Sub),
16820 DAG.getConstant(5, dl, MVT::i32));
16821 } else {
16822 // CMOV 0, 1, ==, (CMPZ x, y) ->
16823 // (ADDCARRY (SUB x, y), t:0, t:1)
16824 // where t = (SUBCARRY 0, (SUB x, y), 0)
16825 //
16826 // The SUBCARRY computes 0 - (x - y) and this will give a borrow when
16827 // x != y. In other words, a carry C == 1 when x == y, C == 0
16828 // otherwise.
16829 // The final ADDCARRY computes
16830 // x - y + (0 - (x - y)) + C == C
16831 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, LHS, RHS);
16832 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
16833 SDValue Neg = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, Sub);
16834 // ISD::SUBCARRY returns a borrow but we want the carry here
16835 // actually.
16836 SDValue Carry =
16837 DAG.getNode(ISD::SUB, dl, MVT::i32,
16838 DAG.getConstant(1, dl, MVT::i32), Neg.getValue(1));
16839 Res = DAG.getNode(ISD::ADDCARRY, dl, VTs, Sub, Neg, Carry);
16840 }
16841 } else if (CC == ARMCC::NE && !isNullConstant(RHS) &&
16842 (!Subtarget->isThumb1Only() || isPowerOf2Constant(TrueVal))) {
16843 // This seems pointless but will allow us to combine it further below.
16844 // CMOV 0, z, !=, (CMPZ x, y) -> CMOV (SUBS x, y), z, !=, (SUBS x, y):1
16845 SDValue Sub =
16846 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
16847 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
16848 Sub.getValue(1), SDValue());
16849 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, TrueVal, ARMcc,
16850 N->getOperand(3), CPSRGlue.getValue(1));
16851 FalseVal = Sub;
16852 }
16853 } else if (isNullConstant(TrueVal)) {
16854 if (CC == ARMCC::EQ && !isNullConstant(RHS) &&
16855 (!Subtarget->isThumb1Only() || isPowerOf2Constant(FalseVal))) {
16856 // This seems pointless but will allow us to combine it further below
16857 // Note that we change == for != as this is the dual for the case above.
16858 // CMOV z, 0, ==, (CMPZ x, y) -> CMOV (SUBS x, y), z, !=, (SUBS x, y):1
16859 SDValue Sub =
16860 DAG.getNode(ARMISD::SUBS, dl, DAG.getVTList(VT, MVT::i32), LHS, RHS);
16861 SDValue CPSRGlue = DAG.getCopyToReg(DAG.getEntryNode(), dl, ARM::CPSR,
16862 Sub.getValue(1), SDValue());
16863 Res = DAG.getNode(ARMISD::CMOV, dl, VT, Sub, FalseVal,
16864 DAG.getConstant(ARMCC::NE, dl, MVT::i32),
16865 N->getOperand(3), CPSRGlue.getValue(1));
16866 FalseVal = Sub;
16867 }
16868 }
16869
16870 // On Thumb1, the DAG above may be further combined if z is a power of 2
16871 // (z == 2 ^ K).
16872 // CMOV (SUBS x, y), z, !=, (SUBS x, y):1 ->
16873 // t1 = (USUBO (SUB x, y), 1)
16874 // t2 = (SUBCARRY (SUB x, y), t1:0, t1:1)
16875 // Result = if K != 0 then (SHL t2:0, K) else t2:0
16876 //
16877 // This also handles the special case of comparing against zero; it's
16878 // essentially, the same pattern, except there's no SUBS:
16879 // CMOV x, z, !=, (CMPZ x, 0) ->
16880 // t1 = (USUBO x, 1)
16881 // t2 = (SUBCARRY x, t1:0, t1:1)
16882 // Result = if K != 0 then (SHL t2:0, K) else t2:0
16883 const APInt *TrueConst;
16884 if (Subtarget->isThumb1Only() && CC == ARMCC::NE &&
16885 ((FalseVal.getOpcode() == ARMISD::SUBS &&
16886 FalseVal.getOperand(0) == LHS && FalseVal.getOperand(1) == RHS) ||
16887 (FalseVal == LHS && isNullConstant(RHS))) &&
16888 (TrueConst = isPowerOf2Constant(TrueVal))) {
16889 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
16890 unsigned ShiftAmount = TrueConst->logBase2();
16891 if (ShiftAmount)
16892 TrueVal = DAG.getConstant(1, dl, VT);
16893 SDValue Subc = DAG.getNode(ISD::USUBO, dl, VTs, FalseVal, TrueVal);
16894 Res = DAG.getNode(ISD::SUBCARRY, dl, VTs, FalseVal, Subc, Subc.getValue(1));
16895
16896 if (ShiftAmount)
16897 Res = DAG.getNode(ISD::SHL, dl, VT, Res,
16898 DAG.getConstant(ShiftAmount, dl, MVT::i32));
16899 }
16900
16901 if (Res.getNode()) {
16902 KnownBits Known = DAG.computeKnownBits(SDValue(N,0));
16903 // Capture demanded bits information that would be otherwise lost.
16904 if (Known.Zero == 0xfffffffe)
16905 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
16906 DAG.getValueType(MVT::i1));
16907 else if (Known.Zero == 0xffffff00)
16908 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
16909 DAG.getValueType(MVT::i8));
16910 else if (Known.Zero == 0xffff0000)
16911 Res = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Res,
16912 DAG.getValueType(MVT::i16));
16913 }
16914
16915 return Res;
16916}
16917
16918static SDValue PerformBITCASTCombine(SDNode *N,
16919 TargetLowering::DAGCombinerInfo &DCI,
16920 const ARMSubtarget *ST) {
16921 SelectionDAG &DAG = DCI.DAG;
16922 SDValue Src = N->getOperand(0);
16923 EVT DstVT = N->getValueType(0);
16924
16925 // Convert v4f32 bitcast (v4i32 vdup (i32)) -> v4f32 vdup (i32) under MVE.
16926 if (ST->hasMVEIntegerOps() && Src.getOpcode() == ARMISD::VDUP) {
16927 EVT SrcVT = Src.getValueType();
16928 if (SrcVT.getScalarSizeInBits() == DstVT.getScalarSizeInBits())
16929 return DAG.getNode(ARMISD::VDUP, SDLoc(N), DstVT, Src.getOperand(0));
16930 }
16931
16932 // We may have a bitcast of something that has already had this bitcast
16933 // combine performed on it, so skip past any VECTOR_REG_CASTs.
16934 while (Src.getOpcode() == ARMISD::VECTOR_REG_CAST)
16935 Src = Src.getOperand(0);
16936
16937 // Bitcast from element-wise VMOV or VMVN doesn't need VREV if the VREV that
16938 // would be generated is at least the width of the element type.
16939 EVT SrcVT = Src.getValueType();
16940 if ((Src.getOpcode() == ARMISD::VMOVIMM ||
16941 Src.getOpcode() == ARMISD::VMVNIMM ||
16942 Src.getOpcode() == ARMISD::VMOVFPIMM) &&
16943 SrcVT.getScalarSizeInBits() <= DstVT.getScalarSizeInBits() &&
16944 DAG.getDataLayout().isBigEndian())
16945 return DAG.getNode(ARMISD::VECTOR_REG_CAST, SDLoc(N), DstVT, Src);
16946
16947 // bitcast(extract(x, n)); bitcast(extract(x, n+1)) -> VMOVRRD x
16948 if (SDValue R = PerformExtractEltToVMOVRRD(N, DCI))
16949 return R;
16950
16951 return SDValue();
16952}
16953
16954SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
16955 DAGCombinerInfo &DCI) const {
16956 switch (N->getOpcode()) {
16957 default: break;
16958 case ISD::SELECT_CC:
16959 case ISD::SELECT: return PerformSELECTCombine(N, DCI, Subtarget);
16960 case ISD::VSELECT: return PerformVSELECTCombine(N, DCI, Subtarget);
16961 case ISD::ABS: return PerformABSCombine(N, DCI, Subtarget);
16962 case ARMISD::ADDE: return PerformADDECombine(N, DCI, Subtarget);
16963 case ARMISD::UMLAL: return PerformUMLALCombine(N, DCI.DAG, Subtarget);
16964 case ISD::ADD: return PerformADDCombine(N, DCI, Subtarget);
16965 case ISD::SUB: return PerformSUBCombine(N, DCI, Subtarget);
16966 case ISD::MUL: return PerformMULCombine(N, DCI, Subtarget);
16967 case ISD::OR: return PerformORCombine(N, DCI, Subtarget);
16968 case ISD::XOR: return PerformXORCombine(N, DCI, Subtarget);
16969 case ISD::AND: return PerformANDCombine(N, DCI, Subtarget);
16970 case ISD::BRCOND:
16971 case ISD::BR_CC: return PerformHWLoopCombine(N, DCI, Subtarget);
16972 case ARMISD::ADDC:
16973 case ARMISD::SUBC: return PerformAddcSubcCombine(N, DCI, Subtarget);
16974 case ARMISD::SUBE: return PerformAddeSubeCombine(N, DCI, Subtarget);
16975 case ARMISD::BFI: return PerformBFICombine(N, DCI);
16976 case ARMISD::VMOVRRD: return PerformVMOVRRDCombine(N, DCI, Subtarget);
16977 case ARMISD::VMOVDRR: return PerformVMOVDRRCombine(N, DCI.DAG);
16978 case ARMISD::VMOVhr: return PerformVMOVhrCombine(N, DCI);
16979 case ARMISD::VMOVrh: return PerformVMOVrhCombine(N, DCI);
16980 case ISD::STORE: return PerformSTORECombine(N, DCI, Subtarget);
16981 case ISD::BUILD_VECTOR: return PerformBUILD_VECTORCombine(N, DCI, Subtarget);
16982 case ISD::INSERT_VECTOR_ELT: return PerformInsertEltCombine(N, DCI);
16983 case ISD::EXTRACT_VECTOR_ELT:
16984 return PerformExtractEltCombine(N, DCI, Subtarget);
16985 case ISD::SIGN_EXTEND_INREG: return PerformSignExtendInregCombine(N, DCI.DAG);
16986 case ISD::VECTOR_SHUFFLE: return PerformVECTOR_SHUFFLECombine(N, DCI.DAG);
16987 case ARMISD::VDUPLANE: return PerformVDUPLANECombine(N, DCI, Subtarget);
16988 case ARMISD::VDUP: return PerformVDUPCombine(N, DCI, Subtarget);
16989 case ISD::FP_TO_SINT:
16990 case ISD::FP_TO_UINT:
16991 return PerformVCVTCombine(N, DCI.DAG, Subtarget);
16992 case ISD::FDIV:
16993 return PerformVDIVCombine(N, DCI.DAG, Subtarget);
16994 case ISD::INTRINSIC_WO_CHAIN:
16995 return PerformIntrinsicCombine(N, DCI);
16996 case ISD::SHL:
16997 case ISD::SRA:
16998 case ISD::SRL:
16999 return PerformShiftCombine(N, DCI, Subtarget);
17000 case ISD::SIGN_EXTEND:
17001 case ISD::ZERO_EXTEND:
17002 case ISD::ANY_EXTEND:
17003 return PerformExtendCombine(N, DCI.DAG, Subtarget);
17004 case ISD::FP_EXTEND:
17005 return PerformFPExtendCombine(N, DCI.DAG, Subtarget);
17006 case ISD::SMIN:
17007 case ISD::UMIN:
17008 case ISD::SMAX:
17009 case ISD::UMAX:
17010 return PerformMinMaxCombine(N, DCI.DAG, Subtarget);
17011 case ARMISD::CMOV: return PerformCMOVCombine(N, DCI.DAG);
17012 case ARMISD::BRCOND: return PerformBRCONDCombine(N, DCI.DAG);
17013 case ISD::LOAD: return PerformLOADCombine(N, DCI);
17014 case ARMISD::VLD1DUP:
17015 case ARMISD::VLD2DUP:
17016 case ARMISD::VLD3DUP:
17017 case ARMISD::VLD4DUP:
17018 return PerformVLDCombine(N, DCI);
17019 case ARMISD::BUILD_VECTOR:
17020 return PerformARMBUILD_VECTORCombine(N, DCI);
17021 case ISD::BITCAST:
17022 return PerformBITCASTCombine(N, DCI, Subtarget);
17023 case ARMISD::PREDICATE_CAST:
17024 return PerformPREDICATE_CASTCombine(N, DCI);
17025 case ARMISD::VECTOR_REG_CAST:
17026 return PerformVECTOR_REG_CASTCombine(N, DCI, Subtarget);
17027 case ARMISD::VCMP:
17028 return PerformVCMPCombine(N, DCI, Subtarget);
17029 case ISD::VECREDUCE_ADD:
17030 return PerformVECREDUCE_ADDCombine(N, DCI.DAG, Subtarget);
17031 case ARMISD::VMOVN:
17032 return PerformVMOVNCombine(N, DCI);
17033 case ARMISD::VQMOVNs:
17034 case ARMISD::VQMOVNu:
17035 return PerformVQMOVNCombine(N, DCI);
17036 case ARMISD::ASRL:
17037 case ARMISD::LSRL:
17038 case ARMISD::LSLL:
17039 return PerformLongShiftCombine(N, DCI.DAG);
17040 case ARMISD::SMULWB: {
17041 unsigned BitWidth = N->getValueType(0).getSizeInBits();
17042 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
17043 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
17044 return SDValue();
17045 break;
17046 }
17047 case ARMISD::SMULWT: {
17048 unsigned BitWidth = N->getValueType(0).getSizeInBits();
17049 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
17050 if (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI))
17051 return SDValue();
17052 break;
17053 }
17054 case ARMISD::SMLALBB:
17055 case ARMISD::QADD16b:
17056 case ARMISD::QSUB16b: {
17057 unsigned BitWidth = N->getValueType(0).getSizeInBits();
17058 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 16);
17059 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
17060 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
17061 return SDValue();
17062 break;
17063 }
17064 case ARMISD::SMLALBT: {
17065 unsigned LowWidth = N->getOperand(0).getValueType().getSizeInBits();
17066 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
17067 unsigned HighWidth = N->getOperand(1).getValueType().getSizeInBits();
17068 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
17069 if ((SimplifyDemandedBits(N->getOperand(0), LowMask, DCI)) ||
17070 (SimplifyDemandedBits(N->getOperand(1), HighMask, DCI)))
17071 return SDValue();
17072 break;
17073 }
17074 case ARMISD::SMLALTB: {
17075 unsigned HighWidth = N->getOperand(0).getValueType().getSizeInBits();
17076 APInt HighMask = APInt::getHighBitsSet(HighWidth, 16);
17077 unsigned LowWidth = N->getOperand(1).getValueType().getSizeInBits();
17078 APInt LowMask = APInt::getLowBitsSet(LowWidth, 16);
17079 if ((SimplifyDemandedBits(N->getOperand(0), HighMask, DCI)) ||
17080 (SimplifyDemandedBits(N->getOperand(1), LowMask, DCI)))
17081 return SDValue();
17082 break;
17083 }
17084 case ARMISD::SMLALTT: {
17085 unsigned BitWidth = N->getValueType(0).getSizeInBits();
17086 APInt DemandedMask = APInt::getHighBitsSet(BitWidth, 16);
17087 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
17088 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
17089 return SDValue();
17090 break;
17091 }
17092 case ARMISD::QADD8b:
17093 case ARMISD::QSUB8b: {
17094 unsigned BitWidth = N->getValueType(0).getSizeInBits();
17095 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, 8);
17096 if ((SimplifyDemandedBits(N->getOperand(0), DemandedMask, DCI)) ||
17097 (SimplifyDemandedBits(N->getOperand(1), DemandedMask, DCI)))
17098 return SDValue();
17099 break;
17100 }
17101 case ISD::INTRINSIC_VOID:
17102 case ISD::INTRINSIC_W_CHAIN:
17103 switch (cast<ConstantSDNode>(N->getOperand(1))->getZExtValue()) {
17104 case Intrinsic::arm_neon_vld1:
17105 case Intrinsic::arm_neon_vld1x2:
17106 case Intrinsic::arm_neon_vld1x3:
17107 case Intrinsic::arm_neon_vld1x4:
17108 case Intrinsic::arm_neon_vld2:
17109 case Intrinsic::arm_neon_vld3:
17110 case Intrinsic::arm_neon_vld4:
17111 case Intrinsic::arm_neon_vld2lane:
17112 case Intrinsic::arm_neon_vld3lane:
17113 case Intrinsic::arm_neon_vld4lane:
17114 case Intrinsic::arm_neon_vld2dup:
17115 case Intrinsic::arm_neon_vld3dup:
17116 case Intrinsic::arm_neon_vld4dup:
17117 case Intrinsic::arm_neon_vst1:
17118 case Intrinsic::arm_neon_vst1x2:
17119 case Intrinsic::arm_neon_vst1x3:
17120 case Intrinsic::arm_neon_vst1x4:
17121 case Intrinsic::arm_neon_vst2:
17122 case Intrinsic::arm_neon_vst3:
17123 case Intrinsic::arm_neon_vst4:
17124 case Intrinsic::arm_neon_vst2lane:
17125 case Intrinsic::arm_neon_vst3lane:
17126 case Intrinsic::arm_neon_vst4lane:
17127 return PerformVLDCombine(N, DCI);
17128 case Intrinsic::arm_mve_vld2q:
17129 case Intrinsic::arm_mve_vld4q:
17130 case Intrinsic::arm_mve_vst2q:
17131 case Intrinsic::arm_mve_vst4q:
17132 return PerformMVEVLDCombine(N, DCI);
17133 default: break;
17134 }
17135 break;
17136 }
17137 return SDValue();
17138}
17139
17140bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
17141 EVT VT) const {
17142 return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
17143}
17144
17145bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
17146 Align Alignment,
17147 MachineMemOperand::Flags,
17148 bool *Fast) const {
17149 // Depends what it gets converted into if the type is weird.
17150 if (!VT.isSimple())
17151 return false;
17152
17153 // The AllowsUnaligned flag models the SCTLR.A setting in ARM cpus
17154 bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
17155 auto Ty = VT.getSimpleVT().SimpleTy;
17156
17157 if (Ty == MVT::i8 || Ty == MVT::i16 || Ty == MVT::i32) {
17158 // Unaligned access can use (for example) LRDB, LRDH, LDR
17159 if (AllowsUnaligned) {
17160 if (Fast)
17161 *Fast = Subtarget->hasV7Ops();
17162 return true;
17163 }
17164 }
17165
17166 if (Ty == MVT::f64 || Ty == MVT::v2f64) {
17167 // For any little-endian targets with neon, we can support unaligned ld/st
17168 // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
17169 // A big-endian target may also explicitly support unaligned accesses
17170 if (Subtarget->hasNEON() && (AllowsUnaligned || Subtarget->isLittle())) {
17171 if (Fast)
17172 *Fast = true;
17173 return true;
17174 }
17175 }
17176
17177 if (!Subtarget->hasMVEIntegerOps())
17178 return false;
17179
17180 // These are for predicates
17181 if ((Ty == MVT::v16i1 || Ty == MVT::v8i1 || Ty == MVT::v4i1)) {
17182 if (Fast)
17183 *Fast = true;
17184 return true;
17185 }
17186
17187 // These are for truncated stores/narrowing loads. They are fine so long as
17188 // the alignment is at least the size of the item being loaded
17189 if ((Ty == MVT::v4i8 || Ty == MVT::v8i8 || Ty == MVT::v4i16) &&
17190 Alignment >= VT.getScalarSizeInBits() / 8) {
17191 if (Fast)
17192 *Fast = true;
17193 return true;
17194 }
17195
17196 // In little-endian MVE, the store instructions VSTRB.U8, VSTRH.U16 and
17197 // VSTRW.U32 all store the vector register in exactly the same format, and
17198 // differ only in the range of their immediate offset field and the required
17199 // alignment. So there is always a store that can be used, regardless of
17200 // actual type.
17201 //
17202 // For big endian, that is not the case. But can still emit a (VSTRB.U8;
17203 // VREV64.8) pair and get the same effect. This will likely be better than
17204 // aligning the vector through the stack.
17205 if (Ty == MVT::v16i8 || Ty == MVT::v8i16 || Ty == MVT::v8f16 ||
17206 Ty == MVT::v4i32 || Ty == MVT::v4f32 || Ty == MVT::v2i64 ||
17207 Ty == MVT::v2f64) {
17208 if (Fast)
17209 *Fast = true;
17210 return true;
17211 }
17212
17213 return false;
17214}
17215
17216
17217EVT ARMTargetLowering::getOptimalMemOpType(
17218 const MemOp &Op, const AttributeList &FuncAttributes) const {
17219 // See if we can use NEON instructions for this...
17220 if ((Op.isMemcpy() || Op.isZeroMemset()) && Subtarget->hasNEON() &&
17221 !FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
17222 bool Fast;
17223 if (Op.size() >= 16 &&
17224 (Op.isAligned(Align(16)) ||
17225 (allowsMisalignedMemoryAccesses(MVT::v2f64, 0, Align(1),
17226 MachineMemOperand::MONone, &Fast) &&
17227 Fast))) {
17228 return MVT::v2f64;
17229 } else if (Op.size() >= 8 &&
17230 (Op.isAligned(Align(8)) ||
17231 (allowsMisalignedMemoryAccesses(
17232 MVT::f64, 0, Align(1), MachineMemOperand::MONone, &Fast) &&
17233 Fast))) {
17234 return MVT::f64;
17235 }
17236 }
17237
17238 // Let the target-independent logic figure it out.
17239 return MVT::Other;
17240}
17241
17242// 64-bit integers are split into their high and low parts and held in two
17243// different registers, so the trunc is free since the low register can just
17244// be used.
17245bool ARMTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
17246 if (!SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
17247 return false;
17248 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
17249 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
17250 return (SrcBits == 64 && DestBits == 32);
17251}
17252
17253bool ARMTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
17254 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
17255 !DstVT.isInteger())
17256 return false;
17257 unsigned SrcBits = SrcVT.getSizeInBits();
17258 unsigned DestBits = DstVT.getSizeInBits();
17259 return (SrcBits == 64 && DestBits == 32);
17260}
17261
17262bool ARMTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
17263 if (Val.getOpcode() != ISD::LOAD)
17264 return false;
17265
17266 EVT VT1 = Val.getValueType();
17267 if (!VT1.isSimple() || !VT1.isInteger() ||
17268 !VT2.isSimple() || !VT2.isInteger())
17269 return false;
17270
17271 switch (VT1.getSimpleVT().SimpleTy) {
17272 default: break;
17273 case MVT::i1:
17274 case MVT::i8:
17275 case MVT::i16:
17276 // 8-bit and 16-bit loads implicitly zero-extend to 32-bits.
17277 return true;
17278 }
17279
17280 return false;
17281}
17282
17283bool ARMTargetLowering::isFNegFree(EVT VT) const {
17284 if (!VT.isSimple())
17285 return false;
17286
17287 // There are quite a few FP16 instructions (e.g. VNMLA, VNMLS, etc.) that
17288 // negate values directly (fneg is free). So, we don't want to let the DAG
17289 // combiner rewrite fneg into xors and some other instructions. For f16 and
17290 // FullFP16 argument passing, some bitcast nodes may be introduced,
17291 // triggering this DAG combine rewrite, so we are avoiding that with this.
17292 switch (VT.getSimpleVT().SimpleTy) {
17293 default: break;
17294 case MVT::f16:
17295 return Subtarget->hasFullFP16();
17296 }
17297
17298 return false;
17299}
17300
17301/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
17302/// of the vector elements.
17303static bool areExtractExts(Value *Ext1, Value *Ext2) {
17304 auto areExtDoubled = [](Instruction *Ext) {
17305 return Ext->getType()->getScalarSizeInBits() ==
17306 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
17307 };
17308
17309 if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
17310 !match(Ext2, m_ZExtOrSExt(m_Value())) ||
17311 !areExtDoubled(cast<Instruction>(Ext1)) ||
17312 !areExtDoubled(cast<Instruction>(Ext2)))
17313 return false;
17314
17315 return true;
17316}
17317
17318/// Check if sinking \p I's operands to I's basic block is profitable, because
17319/// the operands can be folded into a target instruction, e.g.
17320/// sext/zext can be folded into vsubl.
17321bool ARMTargetLowering::shouldSinkOperands(Instruction *I,
17322 SmallVectorImpl<Use *> &Ops) const {
17323 if (!I->getType()->isVectorTy())
17324 return false;
17325
17326 if (Subtarget->hasNEON()) {
17327 switch (I->getOpcode()) {
17328 case Instruction::Sub:
17329 case Instruction::Add: {
17330 if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
17331 return false;
17332 Ops.push_back(&I->getOperandUse(0));
17333 Ops.push_back(&I->getOperandUse(1));
17334 return true;
17335 }
17336 default:
17337 return false;
17338 }
17339 }
17340
17341 if (!Subtarget->hasMVEIntegerOps())
17342 return false;
17343
17344 auto IsFMSMul = [&](Instruction *I) {
17345 if (!I->hasOneUse())
17346 return false;
17347 auto *Sub = cast<Instruction>(*I->users().begin());
17348 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(1) == I;
17349 };
17350 auto IsFMS = [&](Instruction *I) {
17351 if (match(I->getOperand(0), m_FNeg(m_Value())) ||
17352 match(I->getOperand(1), m_FNeg(m_Value())))
17353 return true;
17354 return false;
17355 };
17356
17357 auto IsSinker = [&](Instruction *I, int Operand) {
17358 switch (I->getOpcode()) {
17359 case Instruction::Add:
17360 case Instruction::Mul:
17361 case Instruction::FAdd:
17362 case Instruction::ICmp:
17363 case Instruction::FCmp:
17364 return true;
17365 case Instruction::FMul:
17366 return !IsFMSMul(I);
17367 case Instruction::Sub:
17368 case Instruction::FSub:
17369 case Instruction::Shl:
17370 case Instruction::LShr:
17371 case Instruction::AShr:
17372 return Operand == 1;
17373 case Instruction::Call:
17374 if (auto *II = dyn_cast<IntrinsicInst>(I)) {
17375 switch (II->getIntrinsicID()) {
17376 case Intrinsic::fma:
17377 return !IsFMS(I);
17378 case Intrinsic::arm_mve_add_predicated:
17379 case Intrinsic::arm_mve_mul_predicated:
17380 case Intrinsic::arm_mve_qadd_predicated:
17381 case Intrinsic::arm_mve_hadd_predicated:
17382 case Intrinsic::arm_mve_vqdmull_predicated:
17383 case Intrinsic::arm_mve_qdmulh_predicated:
17384 case Intrinsic::arm_mve_qrdmulh_predicated:
17385 case Intrinsic::arm_mve_fma_predicated:
17386 return true;
17387 case Intrinsic::arm_mve_sub_predicated:
17388 case Intrinsic::arm_mve_qsub_predicated:
17389 case Intrinsic::arm_mve_hsub_predicated:
17390 return Operand == 1;
17391 default:
17392 return false;
17393 }
17394 }
17395 return false;
17396 default:
17397 return false;
17398 }
17399 };
17400
17401 for (auto OpIdx : enumerate(I->operands())) {
17402 Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
17403 // Make sure we are not already sinking this operand
17404 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
17405 continue;
17406
17407 Instruction *Shuffle = Op;
17408 if (Shuffle->getOpcode() == Instruction::BitCast)
17409 Shuffle = dyn_cast<Instruction>(Shuffle->getOperand(0));
17410 // We are looking for a splat that can be sunk.
17411 if (!Shuffle ||
17412 !match(Shuffle, m_Shuffle(
17413 m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
17414 m_Undef(), m_ZeroMask())))
17415 continue;
17416 if (!IsSinker(I, OpIdx.index()))
17417 continue;
17418
17419 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
17420 // and vector registers
17421 for (Use &U : Op->uses()) {
17422 Instruction *Insn = cast<Instruction>(U.getUser());
17423 if (!IsSinker(Insn, U.getOperandNo()))
17424 return false;
17425 }
17426
17427 Ops.push_back(&Shuffle->getOperandUse(0));
17428 if (Shuffle != Op)
17429 Ops.push_back(&Op->getOperandUse(0));
17430 Ops.push_back(&OpIdx.value());
17431 }
17432 return true;
17433}
17434
17435Type *ARMTargetLowering::shouldConvertSplatType(ShuffleVectorInst *SVI) const {
17436 if (!Subtarget->hasMVEIntegerOps())
17437 return nullptr;
17438 Type *SVIType = SVI->getType();
17439 Type *ScalarType = SVIType->getScalarType();
17440
17441 if (ScalarType->isFloatTy())
17442 return Type::getInt32Ty(SVIType->getContext());
17443 if (ScalarType->isHalfTy())
17444 return Type::getInt16Ty(SVIType->getContext());
17445 return nullptr;
17446}
17447
17448bool ARMTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
17449 EVT VT = ExtVal.getValueType();
17450
17451 if (!isTypeLegal(VT))
17452 return false;
17453
17454 if (auto *Ld = dyn_cast<MaskedLoadSDNode>(ExtVal.getOperand(0))) {
17455 if (Ld->isExpandingLoad())
17456 return false;
17457 }
17458
17459 if (Subtarget->hasMVEIntegerOps())
17460 return true;
17461
17462 // Don't create a loadext if we can fold the extension into a wide/long
17463 // instruction.
17464 // If there's more than one user instruction, the loadext is desirable no
17465 // matter what. There can be two uses by the same instruction.
17466 if (ExtVal->use_empty() ||
17467 !ExtVal->use_begin()->isOnlyUserOf(ExtVal.getNode()))
17468 return true;
17469
17470 SDNode *U = *ExtVal->use_begin();
17471 if ((U->getOpcode() == ISD::ADD || U->getOpcode() == ISD::SUB ||
17472 U->getOpcode() == ISD::SHL || U->getOpcode() == ARMISD::VSHLIMM))
17473 return false;
17474
17475 return true;
17476}
17477
17478bool ARMTargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
17479 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
17480 return false;
17481
17482 if (!isTypeLegal(EVT::getEVT(Ty1)))
17483 return false;
17484
17485 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop")(static_cast <bool> (Ty1->getPrimitiveSizeInBits() <=
64 && "i128 is probably not a noop") ? void (0) : __assert_fail
("Ty1->getPrimitiveSizeInBits() <= 64 && \"i128 is probably not a noop\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 17485, __extension__ __PRETTY_FUNCTION__))
;
17486
17487 // Assuming the caller doesn't have a zeroext or signext return parameter,
17488 // truncation all the way down to i1 is valid.
17489 return true;
17490}
17491
17492InstructionCost ARMTargetLowering::getScalingFactorCost(const DataLayout &DL,
17493 const AddrMode &AM,
17494 Type *Ty,
17495 unsigned AS) const {
17496 if (isLegalAddressingMode(DL, AM, Ty, AS)) {
17497 if (Subtarget->hasFPAO())
17498 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
17499 return 0;
17500 }
17501 return -1;
17502}
17503
17504/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
17505/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
17506/// expanded to FMAs when this method returns true, otherwise fmuladd is
17507/// expanded to fmul + fadd.
17508///
17509/// ARM supports both fused and unfused multiply-add operations; we already
17510/// lower a pair of fmul and fadd to the latter so it's not clear that there
17511/// would be a gain or that the gain would be worthwhile enough to risk
17512/// correctness bugs.
17513///
17514/// For MVE, we set this to true as it helps simplify the need for some
17515/// patterns (and we don't have the non-fused floating point instruction).
17516bool ARMTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
17517 EVT VT) const {
17518 if (!VT.isSimple())
17519 return false;
17520
17521 switch (VT.getSimpleVT().SimpleTy) {
17522 case MVT::v4f32:
17523 case MVT::v8f16:
17524 return Subtarget->hasMVEFloatOps();
17525 case MVT::f16:
17526 return Subtarget->useFPVFMx16();
17527 case MVT::f32:
17528 return Subtarget->useFPVFMx();
17529 case MVT::f64:
17530 return Subtarget->useFPVFMx64();
17531 default:
17532 break;
17533 }
17534
17535 return false;
17536}
17537
17538static bool isLegalT1AddressImmediate(int64_t V, EVT VT) {
17539 if (V < 0)
17540 return false;
17541
17542 unsigned Scale = 1;
17543 switch (VT.getSimpleVT().SimpleTy) {
17544 case MVT::i1:
17545 case MVT::i8:
17546 // Scale == 1;
17547 break;
17548 case MVT::i16:
17549 // Scale == 2;
17550 Scale = 2;
17551 break;
17552 default:
17553 // On thumb1 we load most things (i32, i64, floats, etc) with a LDR
17554 // Scale == 4;
17555 Scale = 4;
17556 break;
17557 }
17558
17559 if ((V & (Scale - 1)) != 0)
17560 return false;
17561 return isUInt<5>(V / Scale);
17562}
17563
17564static bool isLegalT2AddressImmediate(int64_t V, EVT VT,
17565 const ARMSubtarget *Subtarget) {
17566 if (!VT.isInteger() && !VT.isFloatingPoint())
17567 return false;
17568 if (VT.isVector() && Subtarget->hasNEON())
17569 return false;
17570 if (VT.isVector() && VT.isFloatingPoint() && Subtarget->hasMVEIntegerOps() &&
17571 !Subtarget->hasMVEFloatOps())
17572 return false;
17573
17574 bool IsNeg = false;
17575 if (V < 0) {
17576 IsNeg = true;
17577 V = -V;
17578 }
17579
17580 unsigned NumBytes = std::max((unsigned)VT.getSizeInBits() / 8, 1U);
17581
17582 // MVE: size * imm7
17583 if (VT.isVector() && Subtarget->hasMVEIntegerOps()) {
17584 switch (VT.getSimpleVT().getVectorElementType().SimpleTy) {
17585 case MVT::i32:
17586 case MVT::f32:
17587 return isShiftedUInt<7,2>(V);
17588 case MVT::i16:
17589 case MVT::f16:
17590 return isShiftedUInt<7,1>(V);
17591 case MVT::i8:
17592 return isUInt<7>(V);
17593 default:
17594 return false;
17595 }
17596 }
17597
17598 // half VLDR: 2 * imm8
17599 if (VT.isFloatingPoint() && NumBytes == 2 && Subtarget->hasFPRegs16())
17600 return isShiftedUInt<8, 1>(V);
17601 // VLDR and LDRD: 4 * imm8
17602 if ((VT.isFloatingPoint() && Subtarget->hasVFP2Base()) || NumBytes == 8)
17603 return isShiftedUInt<8, 2>(V);
17604
17605 if (NumBytes == 1 || NumBytes == 2 || NumBytes == 4) {
17606 // + imm12 or - imm8
17607 if (IsNeg)
17608 return isUInt<8>(V);
17609 return isUInt<12>(V);
17610 }
17611
17612 return false;
17613}
17614
17615/// isLegalAddressImmediate - Return true if the integer value can be used
17616/// as the offset of the target addressing mode for load / store of the
17617/// given type.
17618static bool isLegalAddressImmediate(int64_t V, EVT VT,
17619 const ARMSubtarget *Subtarget) {
17620 if (V == 0)
17621 return true;
17622
17623 if (!VT.isSimple())
17624 return false;
17625
17626 if (Subtarget->isThumb1Only())
17627 return isLegalT1AddressImmediate(V, VT);
17628 else if (Subtarget->isThumb2())
17629 return isLegalT2AddressImmediate(V, VT, Subtarget);
17630
17631 // ARM mode.
17632 if (V < 0)
17633 V = - V;
17634 switch (VT.getSimpleVT().SimpleTy) {
17635 default: return false;
17636 case MVT::i1:
17637 case MVT::i8:
17638 case MVT::i32:
17639 // +- imm12
17640 return isUInt<12>(V);
17641 case MVT::i16:
17642 // +- imm8
17643 return isUInt<8>(V);
17644 case MVT::f32:
17645 case MVT::f64:
17646 if (!Subtarget->hasVFP2Base()) // FIXME: NEON?
17647 return false;
17648 return isShiftedUInt<8, 2>(V);
17649 }
17650}
17651
17652bool ARMTargetLowering::isLegalT2ScaledAddressingMode(const AddrMode &AM,
17653 EVT VT) const {
17654 int Scale = AM.Scale;
17655 if (Scale < 0)
17656 return false;
17657
17658 switch (VT.getSimpleVT().SimpleTy) {
17659 default: return false;
17660 case MVT::i1:
17661 case MVT::i8:
17662 case MVT::i16:
17663 case MVT::i32:
17664 if (Scale == 1)
17665 return true;
17666 // r + r << imm
17667 Scale = Scale & ~1;
17668 return Scale == 2 || Scale == 4 || Scale == 8;
17669 case MVT::i64:
17670 // FIXME: What are we trying to model here? ldrd doesn't have an r + r
17671 // version in Thumb mode.
17672 // r + r
17673 if (Scale == 1)
17674 return true;
17675 // r * 2 (this can be lowered to r + r).
17676 if (!AM.HasBaseReg && Scale == 2)
17677 return true;
17678 return false;
17679 case MVT::isVoid:
17680 // Note, we allow "void" uses (basically, uses that aren't loads or
17681 // stores), because arm allows folding a scale into many arithmetic
17682 // operations. This should be made more precise and revisited later.
17683
17684 // Allow r << imm, but the imm has to be a multiple of two.
17685 if (Scale & 1) return false;
17686 return isPowerOf2_32(Scale);
17687 }
17688}
17689
17690bool ARMTargetLowering::isLegalT1ScaledAddressingMode(const AddrMode &AM,
17691 EVT VT) const {
17692 const int Scale = AM.Scale;
17693
17694 // Negative scales are not supported in Thumb1.
17695 if (Scale < 0)
17696 return false;
17697
17698 // Thumb1 addressing modes do not support register scaling excepting the
17699 // following cases:
17700 // 1. Scale == 1 means no scaling.
17701 // 2. Scale == 2 this can be lowered to r + r if there is no base register.
17702 return (Scale == 1) || (!AM.HasBaseReg && Scale == 2);
17703}
17704
17705/// isLegalAddressingMode - Return true if the addressing mode represented
17706/// by AM is legal for this target, for a load/store of the specified type.
17707bool ARMTargetLowering::isLegalAddressingMode(const DataLayout &DL,
17708 const AddrMode &AM, Type *Ty,
17709 unsigned AS, Instruction *I) const {
17710 EVT VT = getValueType(DL, Ty, true);
17711 if (!isLegalAddressImmediate(AM.BaseOffs, VT, Subtarget))
17712 return false;
17713
17714 // Can never fold addr of global into load/store.
17715 if (AM.BaseGV)
17716 return false;
17717
17718 switch (AM.Scale) {
17719 case 0: // no scale reg, must be "r+i" or "r", or "i".
17720 break;
17721 default:
17722 // ARM doesn't support any R+R*scale+imm addr modes.
17723 if (AM.BaseOffs)
17724 return false;
17725
17726 if (!VT.isSimple())
17727 return false;
17728
17729 if (Subtarget->isThumb1Only())
17730 return isLegalT1ScaledAddressingMode(AM, VT);
17731
17732 if (Subtarget->isThumb2())
17733 return isLegalT2ScaledAddressingMode(AM, VT);
17734
17735 int Scale = AM.Scale;
17736 switch (VT.getSimpleVT().SimpleTy) {
17737 default: return false;
17738 case MVT::i1:
17739 case MVT::i8:
17740 case MVT::i32:
17741 if (Scale < 0) Scale = -Scale;
17742 if (Scale == 1)
17743 return true;
17744 // r + r << imm
17745 return isPowerOf2_32(Scale & ~1);
17746 case MVT::i16:
17747 case MVT::i64:
17748 // r +/- r
17749 if (Scale == 1 || (AM.HasBaseReg && Scale == -1))
17750 return true;
17751 // r * 2 (this can be lowered to r + r).
17752 if (!AM.HasBaseReg && Scale == 2)
17753 return true;
17754 return false;
17755
17756 case MVT::isVoid:
17757 // Note, we allow "void" uses (basically, uses that aren't loads or
17758 // stores), because arm allows folding a scale into many arithmetic
17759 // operations. This should be made more precise and revisited later.
17760
17761 // Allow r << imm, but the imm has to be a multiple of two.
17762 if (Scale & 1) return false;
17763 return isPowerOf2_32(Scale);
17764 }
17765 }
17766 return true;
17767}
17768
17769/// isLegalICmpImmediate - Return true if the specified immediate is legal
17770/// icmp immediate, that is the target has icmp instructions which can compare
17771/// a register against the immediate without having to materialize the
17772/// immediate into a register.
17773bool ARMTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
17774 // Thumb2 and ARM modes can use cmn for negative immediates.
17775 if (!Subtarget->isThumb())
17776 return ARM_AM::getSOImmVal((uint32_t)Imm) != -1 ||
17777 ARM_AM::getSOImmVal(-(uint32_t)Imm) != -1;
17778 if (Subtarget->isThumb2())
17779 return ARM_AM::getT2SOImmVal((uint32_t)Imm) != -1 ||
17780 ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1;
17781 // Thumb1 doesn't have cmn, and only 8-bit immediates.
17782 return Imm >= 0 && Imm <= 255;
17783}
17784
17785/// isLegalAddImmediate - Return true if the specified immediate is a legal add
17786/// *or sub* immediate, that is the target has add or sub instructions which can
17787/// add a register with the immediate without having to materialize the
17788/// immediate into a register.
17789bool ARMTargetLowering::isLegalAddImmediate(int64_t Imm) const {
17790 // Same encoding for add/sub, just flip the sign.
17791 int64_t AbsImm = std::abs(Imm);
17792 if (!Subtarget->isThumb())
17793 return ARM_AM::getSOImmVal(AbsImm) != -1;
17794 if (Subtarget->isThumb2())
17795 return ARM_AM::getT2SOImmVal(AbsImm) != -1;
17796 // Thumb1 only has 8-bit unsigned immediate.
17797 return AbsImm >= 0 && AbsImm <= 255;
17798}
17799
17800static bool getARMIndexedAddressParts(SDNode *Ptr, EVT VT,
17801 bool isSEXTLoad, SDValue &Base,
17802 SDValue &Offset, bool &isInc,
17803 SelectionDAG &DAG) {
17804 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
17805 return false;
17806
17807 if (VT == MVT::i16 || ((VT == MVT::i8 || VT == MVT::i1) && isSEXTLoad)) {
17808 // AddressingMode 3
17809 Base = Ptr->getOperand(0);
17810 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
17811 int RHSC = (int)RHS->getZExtValue();
17812 if (RHSC < 0 && RHSC > -256) {
17813 assert(Ptr->getOpcode() == ISD::ADD)(static_cast <bool> (Ptr->getOpcode() == ISD::ADD) ?
void (0) : __assert_fail ("Ptr->getOpcode() == ISD::ADD",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 17813, __extension__ __PRETTY_FUNCTION__))
;
17814 isInc = false;
17815 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
17816 return true;
17817 }
17818 }
17819 isInc = (Ptr->getOpcode() == ISD::ADD);
17820 Offset = Ptr->getOperand(1);
17821 return true;
17822 } else if (VT == MVT::i32 || VT == MVT::i8 || VT == MVT::i1) {
17823 // AddressingMode 2
17824 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
17825 int RHSC = (int)RHS->getZExtValue();
17826 if (RHSC < 0 && RHSC > -0x1000) {
17827 assert(Ptr->getOpcode() == ISD::ADD)(static_cast <bool> (Ptr->getOpcode() == ISD::ADD) ?
void (0) : __assert_fail ("Ptr->getOpcode() == ISD::ADD",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 17827, __extension__ __PRETTY_FUNCTION__))
;
17828 isInc = false;
17829 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
17830 Base = Ptr->getOperand(0);
17831 return true;
17832 }
17833 }
17834
17835 if (Ptr->getOpcode() == ISD::ADD) {
17836 isInc = true;
17837 ARM_AM::ShiftOpc ShOpcVal=
17838 ARM_AM::getShiftOpcForNode(Ptr->getOperand(0).getOpcode());
17839 if (ShOpcVal != ARM_AM::no_shift) {
17840 Base = Ptr->getOperand(1);
17841 Offset = Ptr->getOperand(0);
17842 } else {
17843 Base = Ptr->getOperand(0);
17844 Offset = Ptr->getOperand(1);
17845 }
17846 return true;
17847 }
17848
17849 isInc = (Ptr->getOpcode() == ISD::ADD);
17850 Base = Ptr->getOperand(0);
17851 Offset = Ptr->getOperand(1);
17852 return true;
17853 }
17854
17855 // FIXME: Use VLDM / VSTM to emulate indexed FP load / store.
17856 return false;
17857}
17858
17859static bool getT2IndexedAddressParts(SDNode *Ptr, EVT VT,
17860 bool isSEXTLoad, SDValue &Base,
17861 SDValue &Offset, bool &isInc,
17862 SelectionDAG &DAG) {
17863 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
17864 return false;
17865
17866 Base = Ptr->getOperand(0);
17867 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ptr->getOperand(1))) {
17868 int RHSC = (int)RHS->getZExtValue();
17869 if (RHSC < 0 && RHSC > -0x100) { // 8 bits.
17870 assert(Ptr->getOpcode() == ISD::ADD)(static_cast <bool> (Ptr->getOpcode() == ISD::ADD) ?
void (0) : __assert_fail ("Ptr->getOpcode() == ISD::ADD",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 17870, __extension__ __PRETTY_FUNCTION__))
;
17871 isInc = false;
17872 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
17873 return true;
17874 } else if (RHSC > 0 && RHSC < 0x100) { // 8 bit, no zero.
17875 isInc = Ptr->getOpcode() == ISD::ADD;
17876 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
17877 return true;
17878 }
17879 }
17880
17881 return false;
17882}
17883
17884static bool getMVEIndexedAddressParts(SDNode *Ptr, EVT VT, Align Alignment,
17885 bool isSEXTLoad, bool IsMasked, bool isLE,
17886 SDValue &Base, SDValue &Offset,
17887 bool &isInc, SelectionDAG &DAG) {
17888 if (Ptr->getOpcode() != ISD::ADD && Ptr->getOpcode() != ISD::SUB)
17889 return false;
17890 if (!isa<ConstantSDNode>(Ptr->getOperand(1)))
17891 return false;
17892
17893 // We allow LE non-masked loads to change the type (for example use a vldrb.8
17894 // as opposed to a vldrw.32). This can allow extra addressing modes or
17895 // alignments for what is otherwise an equivalent instruction.
17896 bool CanChangeType = isLE && !IsMasked;
17897
17898 ConstantSDNode *RHS = cast<ConstantSDNode>(Ptr->getOperand(1));
17899 int RHSC = (int)RHS->getZExtValue();
17900
17901 auto IsInRange = [&](int RHSC, int Limit, int Scale) {
17902 if (RHSC < 0 && RHSC > -Limit * Scale && RHSC % Scale == 0) {
17903 assert(Ptr->getOpcode() == ISD::ADD)(static_cast <bool> (Ptr->getOpcode() == ISD::ADD) ?
void (0) : __assert_fail ("Ptr->getOpcode() == ISD::ADD",
"/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 17903, __extension__ __PRETTY_FUNCTION__))
;
17904 isInc = false;
17905 Offset = DAG.getConstant(-RHSC, SDLoc(Ptr), RHS->getValueType(0));
17906 return true;
17907 } else if (RHSC > 0 && RHSC < Limit * Scale && RHSC % Scale == 0) {
17908 isInc = Ptr->getOpcode() == ISD::ADD;
17909 Offset = DAG.getConstant(RHSC, SDLoc(Ptr), RHS->getValueType(0));
17910 return true;
17911 }
17912 return false;
17913 };
17914
17915 // Try to find a matching instruction based on s/zext, Alignment, Offset and
17916 // (in BE/masked) type.
17917 Base = Ptr->getOperand(0);
17918 if (VT == MVT::v4i16) {
17919 if (Alignment >= 2 && IsInRange(RHSC, 0x80, 2))
17920 return true;
17921 } else if (VT == MVT::v4i8 || VT == MVT::v8i8) {
17922 if (IsInRange(RHSC, 0x80, 1))
17923 return true;
17924 } else if (Alignment >= 4 &&
17925 (CanChangeType || VT == MVT::v4i32 || VT == MVT::v4f32) &&
17926 IsInRange(RHSC, 0x80, 4))
17927 return true;
17928 else if (Alignment >= 2 &&
17929 (CanChangeType || VT == MVT::v8i16 || VT == MVT::v8f16) &&
17930 IsInRange(RHSC, 0x80, 2))
17931 return true;
17932 else if ((CanChangeType || VT == MVT::v16i8) && IsInRange(RHSC, 0x80, 1))
17933 return true;
17934 return false;
17935}
17936
17937/// getPreIndexedAddressParts - returns true by value, base pointer and
17938/// offset pointer and addressing mode by reference if the node's address
17939/// can be legally represented as pre-indexed load / store address.
17940bool
17941ARMTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
17942 SDValue &Offset,
17943 ISD::MemIndexedMode &AM,
17944 SelectionDAG &DAG) const {
17945 if (Subtarget->isThumb1Only())
17946 return false;
17947
17948 EVT VT;
17949 SDValue Ptr;
17950 Align Alignment;
17951 bool isSEXTLoad = false;
17952 bool IsMasked = false;
17953 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
17954 Ptr = LD->getBasePtr();
17955 VT = LD->getMemoryVT();
17956 Alignment = LD->getAlign();
17957 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
17958 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
17959 Ptr = ST->getBasePtr();
17960 VT = ST->getMemoryVT();
17961 Alignment = ST->getAlign();
17962 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
17963 Ptr = LD->getBasePtr();
17964 VT = LD->getMemoryVT();
17965 Alignment = LD->getAlign();
17966 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
17967 IsMasked = true;
17968 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
17969 Ptr = ST->getBasePtr();
17970 VT = ST->getMemoryVT();
17971 Alignment = ST->getAlign();
17972 IsMasked = true;
17973 } else
17974 return false;
17975
17976 bool isInc;
17977 bool isLegal = false;
17978 if (VT.isVector())
17979 isLegal = Subtarget->hasMVEIntegerOps() &&
17980 getMVEIndexedAddressParts(
17981 Ptr.getNode(), VT, Alignment, isSEXTLoad, IsMasked,
17982 Subtarget->isLittle(), Base, Offset, isInc, DAG);
17983 else {
17984 if (Subtarget->isThumb2())
17985 isLegal = getT2IndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
17986 Offset, isInc, DAG);
17987 else
17988 isLegal = getARMIndexedAddressParts(Ptr.getNode(), VT, isSEXTLoad, Base,
17989 Offset, isInc, DAG);
17990 }
17991 if (!isLegal)
17992 return false;
17993
17994 AM = isInc ? ISD::PRE_INC : ISD::PRE_DEC;
17995 return true;
17996}
17997
17998/// getPostIndexedAddressParts - returns true by value, base pointer and
17999/// offset pointer and addressing mode by reference if this node can be
18000/// combined with a load / store to form a post-indexed load / store.
18001bool ARMTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
18002 SDValue &Base,
18003 SDValue &Offset,
18004 ISD::MemIndexedMode &AM,
18005 SelectionDAG &DAG) const {
18006 EVT VT;
18007 SDValue Ptr;
18008 Align Alignment;
18009 bool isSEXTLoad = false, isNonExt;
18010 bool IsMasked = false;
18011 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
18012 VT = LD->getMemoryVT();
18013 Ptr = LD->getBasePtr();
18014 Alignment = LD->getAlign();
18015 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
18016 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
18017 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(N)) {
18018 VT = ST->getMemoryVT();
18019 Ptr = ST->getBasePtr();
18020 Alignment = ST->getAlign();
18021 isNonExt = !ST->isTruncatingStore();
18022 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
18023 VT = LD->getMemoryVT();
18024 Ptr = LD->getBasePtr();
18025 Alignment = LD->getAlign();
18026 isSEXTLoad = LD->getExtensionType() == ISD::SEXTLOAD;
18027 isNonExt = LD->getExtensionType() == ISD::NON_EXTLOAD;
18028 IsMasked = true;
18029 } else if (MaskedStoreSDNode *ST = dyn_cast<MaskedStoreSDNode>(N)) {
18030 VT = ST->getMemoryVT();
18031 Ptr = ST->getBasePtr();
18032 Alignment = ST->getAlign();
18033 isNonExt = !ST->isTruncatingStore();
18034 IsMasked = true;
18035 } else
18036 return false;
18037
18038 if (Subtarget->isThumb1Only()) {
18039 // Thumb-1 can do a limited post-inc load or store as an updating LDM. It
18040 // must be non-extending/truncating, i32, with an offset of 4.
18041 assert(Op->getValueType(0) == MVT::i32 && "Non-i32 post-inc op?!")(static_cast <bool> (Op->getValueType(0) == MVT::i32
&& "Non-i32 post-inc op?!") ? void (0) : __assert_fail
("Op->getValueType(0) == MVT::i32 && \"Non-i32 post-inc op?!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18041, __extension__ __PRETTY_FUNCTION__))
;
18042 if (Op->getOpcode() != ISD::ADD || !isNonExt)
18043 return false;
18044 auto *RHS = dyn_cast<ConstantSDNode>(Op->getOperand(1));
18045 if (!RHS || RHS->getZExtValue() != 4)
18046 return false;
18047
18048 Offset = Op->getOperand(1);
18049 Base = Op->getOperand(0);
18050 AM = ISD::POST_INC;
18051 return true;
18052 }
18053
18054 bool isInc;
18055 bool isLegal = false;
18056 if (VT.isVector())
18057 isLegal = Subtarget->hasMVEIntegerOps() &&
18058 getMVEIndexedAddressParts(Op, VT, Alignment, isSEXTLoad, IsMasked,
18059 Subtarget->isLittle(), Base, Offset,
18060 isInc, DAG);
18061 else {
18062 if (Subtarget->isThumb2())
18063 isLegal = getT2IndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
18064 isInc, DAG);
18065 else
18066 isLegal = getARMIndexedAddressParts(Op, VT, isSEXTLoad, Base, Offset,
18067 isInc, DAG);
18068 }
18069 if (!isLegal)
18070 return false;
18071
18072 if (Ptr != Base) {
18073 // Swap base ptr and offset to catch more post-index load / store when
18074 // it's legal. In Thumb2 mode, offset must be an immediate.
18075 if (Ptr == Offset && Op->getOpcode() == ISD::ADD &&
18076 !Subtarget->isThumb2())
18077 std::swap(Base, Offset);
18078
18079 // Post-indexed load / store update the base pointer.
18080 if (Ptr != Base)
18081 return false;
18082 }
18083
18084 AM = isInc ? ISD::POST_INC : ISD::POST_DEC;
18085 return true;
18086}
18087
18088void ARMTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18089 KnownBits &Known,
18090 const APInt &DemandedElts,
18091 const SelectionDAG &DAG,
18092 unsigned Depth) const {
18093 unsigned BitWidth = Known.getBitWidth();
18094 Known.resetAll();
18095 switch (Op.getOpcode()) {
18096 default: break;
18097 case ARMISD::ADDC:
18098 case ARMISD::ADDE:
18099 case ARMISD::SUBC:
18100 case ARMISD::SUBE:
18101 // Special cases when we convert a carry to a boolean.
18102 if (Op.getResNo() == 0) {
18103 SDValue LHS = Op.getOperand(0);
18104 SDValue RHS = Op.getOperand(1);
18105 // (ADDE 0, 0, C) will give us a single bit.
18106 if (Op->getOpcode() == ARMISD::ADDE && isNullConstant(LHS) &&
18107 isNullConstant(RHS)) {
18108 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - 1);
18109 return;
18110 }
18111 }
18112 break;
18113 case ARMISD::CMOV: {
18114 // Bits are known zero/one if known on the LHS and RHS.
18115 Known = DAG.computeKnownBits(Op.getOperand(0), Depth+1);
18116 if (Known.isUnknown())
18117 return;
18118
18119 KnownBits KnownRHS = DAG.computeKnownBits(Op.getOperand(1), Depth+1);
18120 Known = KnownBits::commonBits(Known, KnownRHS);
18121 return;
18122 }
18123 case ISD::INTRINSIC_W_CHAIN: {
18124 ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
18125 Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
18126 switch (IntID) {
18127 default: return;
18128 case Intrinsic::arm_ldaex:
18129 case Intrinsic::arm_ldrex: {
18130 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
18131 unsigned MemBits = VT.getScalarSizeInBits();
18132 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
18133 return;
18134 }
18135 }
18136 }
18137 case ARMISD::BFI: {
18138 // Conservatively, we can recurse down the first operand
18139 // and just mask out all affected bits.
18140 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
18141
18142 // The operand to BFI is already a mask suitable for removing the bits it
18143 // sets.
18144 ConstantSDNode *CI = cast<ConstantSDNode>(Op.getOperand(2));
18145 const APInt &Mask = CI->getAPIntValue();
18146 Known.Zero &= Mask;
18147 Known.One &= Mask;
18148 return;
18149 }
18150 case ARMISD::VGETLANEs:
18151 case ARMISD::VGETLANEu: {
18152 const SDValue &SrcSV = Op.getOperand(0);
18153 EVT VecVT = SrcSV.getValueType();
18154 assert(VecVT.isVector() && "VGETLANE expected a vector type")(static_cast <bool> (VecVT.isVector() && "VGETLANE expected a vector type"
) ? void (0) : __assert_fail ("VecVT.isVector() && \"VGETLANE expected a vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18154, __extension__ __PRETTY_FUNCTION__))
;
18155 const unsigned NumSrcElts = VecVT.getVectorNumElements();
18156 ConstantSDNode *Pos = cast<ConstantSDNode>(Op.getOperand(1).getNode());
18157 assert(Pos->getAPIntValue().ult(NumSrcElts) &&(static_cast <bool> (Pos->getAPIntValue().ult(NumSrcElts
) && "VGETLANE index out of bounds") ? void (0) : __assert_fail
("Pos->getAPIntValue().ult(NumSrcElts) && \"VGETLANE index out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18158, __extension__ __PRETTY_FUNCTION__))
18158 "VGETLANE index out of bounds")(static_cast <bool> (Pos->getAPIntValue().ult(NumSrcElts
) && "VGETLANE index out of bounds") ? void (0) : __assert_fail
("Pos->getAPIntValue().ult(NumSrcElts) && \"VGETLANE index out of bounds\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18158, __extension__ __PRETTY_FUNCTION__))
;
18159 unsigned Idx = Pos->getZExtValue();
18160 APInt DemandedElt = APInt::getOneBitSet(NumSrcElts, Idx);
18161 Known = DAG.computeKnownBits(SrcSV, DemandedElt, Depth + 1);
18162
18163 EVT VT = Op.getValueType();
18164 const unsigned DstSz = VT.getScalarSizeInBits();
18165 const unsigned SrcSz = VecVT.getVectorElementType().getSizeInBits();
18166 (void)SrcSz;
18167 assert(SrcSz == Known.getBitWidth())(static_cast <bool> (SrcSz == Known.getBitWidth()) ? void
(0) : __assert_fail ("SrcSz == Known.getBitWidth()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18167, __extension__ __PRETTY_FUNCTION__))
;
18168 assert(DstSz > SrcSz)(static_cast <bool> (DstSz > SrcSz) ? void (0) : __assert_fail
("DstSz > SrcSz", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18168, __extension__ __PRETTY_FUNCTION__))
;
18169 if (Op.getOpcode() == ARMISD::VGETLANEs)
18170 Known = Known.sext(DstSz);
18171 else {
18172 Known = Known.zext(DstSz);
18173 }
18174 assert(DstSz == Known.getBitWidth())(static_cast <bool> (DstSz == Known.getBitWidth()) ? void
(0) : __assert_fail ("DstSz == Known.getBitWidth()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18174, __extension__ __PRETTY_FUNCTION__))
;
18175 break;
18176 }
18177 case ARMISD::VMOVrh: {
18178 KnownBits KnownOp = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
18179 assert(KnownOp.getBitWidth() == 16)(static_cast <bool> (KnownOp.getBitWidth() == 16) ? void
(0) : __assert_fail ("KnownOp.getBitWidth() == 16", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18179, __extension__ __PRETTY_FUNCTION__))
;
18180 Known = KnownOp.zext(32);
18181 break;
18182 }
18183 case ARMISD::CSINC:
18184 case ARMISD::CSINV:
18185 case ARMISD::CSNEG: {
18186 KnownBits KnownOp0 = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
18187 KnownBits KnownOp1 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
18188
18189 // The result is either:
18190 // CSINC: KnownOp0 or KnownOp1 + 1
18191 // CSINV: KnownOp0 or ~KnownOp1
18192 // CSNEG: KnownOp0 or KnownOp1 * -1
18193 if (Op.getOpcode() == ARMISD::CSINC)
18194 KnownOp1 = KnownBits::computeForAddSub(
18195 true, false, KnownOp1, KnownBits::makeConstant(APInt(32, 1)));
18196 else if (Op.getOpcode() == ARMISD::CSINV)
18197 std::swap(KnownOp1.Zero, KnownOp1.One);
18198 else if (Op.getOpcode() == ARMISD::CSNEG)
18199 KnownOp1 = KnownBits::mul(
18200 KnownOp1, KnownBits::makeConstant(APInt(32, -1)));
18201
18202 Known = KnownBits::commonBits(KnownOp0, KnownOp1);
18203 break;
18204 }
18205 }
18206}
18207
18208bool ARMTargetLowering::targetShrinkDemandedConstant(
18209 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
18210 TargetLoweringOpt &TLO) const {
18211 // Delay optimization, so we don't have to deal with illegal types, or block
18212 // optimizations.
18213 if (!TLO.LegalOps)
18214 return false;
18215
18216 // Only optimize AND for now.
18217 if (Op.getOpcode() != ISD::AND)
18218 return false;
18219
18220 EVT VT = Op.getValueType();
18221
18222 // Ignore vectors.
18223 if (VT.isVector())
18224 return false;
18225
18226 assert(VT == MVT::i32 && "Unexpected integer type")(static_cast <bool> (VT == MVT::i32 && "Unexpected integer type"
) ? void (0) : __assert_fail ("VT == MVT::i32 && \"Unexpected integer type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18226, __extension__ __PRETTY_FUNCTION__))
;
18227
18228 // Make sure the RHS really is a constant.
18229 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
18230 if (!C)
18231 return false;
18232
18233 unsigned Mask = C->getZExtValue();
18234
18235 unsigned Demanded = DemandedBits.getZExtValue();
18236 unsigned ShrunkMask = Mask & Demanded;
18237 unsigned ExpandedMask = Mask | ~Demanded;
18238
18239 // If the mask is all zeros, let the target-independent code replace the
18240 // result with zero.
18241 if (ShrunkMask == 0)
18242 return false;
18243
18244 // If the mask is all ones, erase the AND. (Currently, the target-independent
18245 // code won't do this, so we have to do it explicitly to avoid an infinite
18246 // loop in obscure cases.)
18247 if (ExpandedMask == ~0U)
18248 return TLO.CombineTo(Op, Op.getOperand(0));
18249
18250 auto IsLegalMask = [ShrunkMask, ExpandedMask](unsigned Mask) -> bool {
18251 return (ShrunkMask & Mask) == ShrunkMask && (~ExpandedMask & Mask) == 0;
18252 };
18253 auto UseMask = [Mask, Op, VT, &TLO](unsigned NewMask) -> bool {
18254 if (NewMask == Mask)
18255 return true;
18256 SDLoc DL(Op);
18257 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
18258 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
18259 return TLO.CombineTo(Op, NewOp);
18260 };
18261
18262 // Prefer uxtb mask.
18263 if (IsLegalMask(0xFF))
18264 return UseMask(0xFF);
18265
18266 // Prefer uxth mask.
18267 if (IsLegalMask(0xFFFF))
18268 return UseMask(0xFFFF);
18269
18270 // [1, 255] is Thumb1 movs+ands, legal immediate for ARM/Thumb2.
18271 // FIXME: Prefer a contiguous sequence of bits for other optimizations.
18272 if (ShrunkMask < 256)
18273 return UseMask(ShrunkMask);
18274
18275 // [-256, -2] is Thumb1 movs+bics, legal immediate for ARM/Thumb2.
18276 // FIXME: Prefer a contiguous sequence of bits for other optimizations.
18277 if ((int)ExpandedMask <= -2 && (int)ExpandedMask >= -256)
18278 return UseMask(ExpandedMask);
18279
18280 // Potential improvements:
18281 //
18282 // We could try to recognize lsls+lsrs or lsrs+lsls pairs here.
18283 // We could try to prefer Thumb1 immediates which can be lowered to a
18284 // two-instruction sequence.
18285 // We could try to recognize more legal ARM/Thumb2 immediates here.
18286
18287 return false;
18288}
18289
18290bool ARMTargetLowering::SimplifyDemandedBitsForTargetNode(
18291 SDValue Op, const APInt &OriginalDemandedBits,
18292 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
18293 unsigned Depth) const {
18294 unsigned Opc = Op.getOpcode();
18295
18296 switch (Opc) {
18297 case ARMISD::ASRL:
18298 case ARMISD::LSRL: {
18299 // If this is result 0 and the other result is unused, see if the demand
18300 // bits allow us to shrink this long shift into a standard small shift in
18301 // the opposite direction.
18302 if (Op.getResNo() == 0 && !Op->hasAnyUseOfValue(1) &&
18303 isa<ConstantSDNode>(Op->getOperand(2))) {
18304 unsigned ShAmt = Op->getConstantOperandVal(2);
18305 if (ShAmt < 32 && OriginalDemandedBits.isSubsetOf(
18306 APInt::getAllOnesValue(32) << (32 - ShAmt)))
18307 return TLO.CombineTo(
18308 Op, TLO.DAG.getNode(
18309 ISD::SHL, SDLoc(Op), MVT::i32, Op.getOperand(1),
18310 TLO.DAG.getConstant(32 - ShAmt, SDLoc(Op), MVT::i32)));
18311 }
18312 break;
18313 }
18314 }
18315
18316 return TargetLowering::SimplifyDemandedBitsForTargetNode(
18317 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
18318}
18319
18320//===----------------------------------------------------------------------===//
18321// ARM Inline Assembly Support
18322//===----------------------------------------------------------------------===//
18323
18324bool ARMTargetLowering::ExpandInlineAsm(CallInst *CI) const {
18325 // Looking for "rev" which is V6+.
18326 if (!Subtarget->hasV6Ops())
18327 return false;
18328
18329 InlineAsm *IA = cast<InlineAsm>(CI->getCalledOperand());
18330 std::string AsmStr = IA->getAsmString();
18331 SmallVector<StringRef, 4> AsmPieces;
18332 SplitString(AsmStr, AsmPieces, ";\n");
18333
18334 switch (AsmPieces.size()) {
18335 default: return false;
18336 case 1:
18337 AsmStr = std::string(AsmPieces[0]);
18338 AsmPieces.clear();
18339 SplitString(AsmStr, AsmPieces, " \t,");
18340
18341 // rev $0, $1
18342 if (AsmPieces.size() == 3 &&
18343 AsmPieces[0] == "rev" && AsmPieces[1] == "$0" && AsmPieces[2] == "$1" &&
18344 IA->getConstraintString().compare(0, 4, "=l,l") == 0) {
18345 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
18346 if (Ty && Ty->getBitWidth() == 32)
18347 return IntrinsicLowering::LowerToByteSwap(CI);
18348 }
18349 break;
18350 }
18351
18352 return false;
18353}
18354
18355const char *ARMTargetLowering::LowerXConstraint(EVT ConstraintVT) const {
18356 // At this point, we have to lower this constraint to something else, so we
18357 // lower it to an "r" or "w". However, by doing this we will force the result
18358 // to be in register, while the X constraint is much more permissive.
18359 //
18360 // Although we are correct (we are free to emit anything, without
18361 // constraints), we might break use cases that would expect us to be more
18362 // efficient and emit something else.
18363 if (!Subtarget->hasVFP2Base())
18364 return "r";
18365 if (ConstraintVT.isFloatingPoint())
18366 return "w";
18367 if (ConstraintVT.isVector() && Subtarget->hasNEON() &&
18368 (ConstraintVT.getSizeInBits() == 64 ||
18369 ConstraintVT.getSizeInBits() == 128))
18370 return "w";
18371
18372 return "r";
18373}
18374
18375/// getConstraintType - Given a constraint letter, return the type of
18376/// constraint it is for this target.
18377ARMTargetLowering::ConstraintType
18378ARMTargetLowering::getConstraintType(StringRef Constraint) const {
18379 unsigned S = Constraint.size();
18380 if (S == 1) {
18381 switch (Constraint[0]) {
18382 default: break;
18383 case 'l': return C_RegisterClass;
18384 case 'w': return C_RegisterClass;
18385 case 'h': return C_RegisterClass;
18386 case 'x': return C_RegisterClass;
18387 case 't': return C_RegisterClass;
18388 case 'j': return C_Immediate; // Constant for movw.
18389 // An address with a single base register. Due to the way we
18390 // currently handle addresses it is the same as an 'r' memory constraint.
18391 case 'Q': return C_Memory;
18392 }
18393 } else if (S == 2) {
18394 switch (Constraint[0]) {
18395 default: break;
18396 case 'T': return C_RegisterClass;
18397 // All 'U+' constraints are addresses.
18398 case 'U': return C_Memory;
18399 }
18400 }
18401 return TargetLowering::getConstraintType(Constraint);
18402}
18403
18404/// Examine constraint type and operand type and determine a weight value.
18405/// This object must already have been set up with the operand type
18406/// and the current alternative constraint selected.
18407TargetLowering::ConstraintWeight
18408ARMTargetLowering::getSingleConstraintMatchWeight(
18409 AsmOperandInfo &info, const char *constraint) const {
18410 ConstraintWeight weight = CW_Invalid;
18411 Value *CallOperandVal = info.CallOperandVal;
18412 // If we don't have a value, we can't do a match,
18413 // but allow it at the lowest weight.
18414 if (!CallOperandVal)
18415 return CW_Default;
18416 Type *type = CallOperandVal->getType();
18417 // Look at the constraint type.
18418 switch (*constraint) {
18419 default:
18420 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
18421 break;
18422 case 'l':
18423 if (type->isIntegerTy()) {
18424 if (Subtarget->isThumb())
18425 weight = CW_SpecificReg;
18426 else
18427 weight = CW_Register;
18428 }
18429 break;
18430 case 'w':
18431 if (type->isFloatingPointTy())
18432 weight = CW_Register;
18433 break;
18434 }
18435 return weight;
18436}
18437
18438using RCPair = std::pair<unsigned, const TargetRegisterClass *>;
18439
18440RCPair ARMTargetLowering::getRegForInlineAsmConstraint(
18441 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
18442 switch (Constraint.size()) {
18443 case 1:
18444 // GCC ARM Constraint Letters
18445 switch (Constraint[0]) {
18446 case 'l': // Low regs or general regs.
18447 if (Subtarget->isThumb())
18448 return RCPair(0U, &ARM::tGPRRegClass);
18449 return RCPair(0U, &ARM::GPRRegClass);
18450 case 'h': // High regs or no regs.
18451 if (Subtarget->isThumb())
18452 return RCPair(0U, &ARM::hGPRRegClass);
18453 break;
18454 case 'r':
18455 if (Subtarget->isThumb1Only())
18456 return RCPair(0U, &ARM::tGPRRegClass);
18457 return RCPair(0U, &ARM::GPRRegClass);
18458 case 'w':
18459 if (VT == MVT::Other)
18460 break;
18461 if (VT == MVT::f32)
18462 return RCPair(0U, &ARM::SPRRegClass);
18463 if (VT.getSizeInBits() == 64)
18464 return RCPair(0U, &ARM::DPRRegClass);
18465 if (VT.getSizeInBits() == 128)
18466 return RCPair(0U, &ARM::QPRRegClass);
18467 break;
18468 case 'x':
18469 if (VT == MVT::Other)
18470 break;
18471 if (VT == MVT::f32)
18472 return RCPair(0U, &ARM::SPR_8RegClass);
18473 if (VT.getSizeInBits() == 64)
18474 return RCPair(0U, &ARM::DPR_8RegClass);
18475 if (VT.getSizeInBits() == 128)
18476 return RCPair(0U, &ARM::QPR_8RegClass);
18477 break;
18478 case 't':
18479 if (VT == MVT::Other)
18480 break;
18481 if (VT == MVT::f32 || VT == MVT::i32)
18482 return RCPair(0U, &ARM::SPRRegClass);
18483 if (VT.getSizeInBits() == 64)
18484 return RCPair(0U, &ARM::DPR_VFP2RegClass);
18485 if (VT.getSizeInBits() == 128)
18486 return RCPair(0U, &ARM::QPR_VFP2RegClass);
18487 break;
18488 }
18489 break;
18490
18491 case 2:
18492 if (Constraint[0] == 'T') {
18493 switch (Constraint[1]) {
18494 default:
18495 break;
18496 case 'e':
18497 return RCPair(0U, &ARM::tGPREvenRegClass);
18498 case 'o':
18499 return RCPair(0U, &ARM::tGPROddRegClass);
18500 }
18501 }
18502 break;
18503
18504 default:
18505 break;
18506 }
18507
18508 if (StringRef("{cc}").equals_lower(Constraint))
18509 return std::make_pair(unsigned(ARM::CPSR), &ARM::CCRRegClass);
18510
18511 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
18512}
18513
18514/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
18515/// vector. If it is invalid, don't add anything to Ops.
18516void ARMTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
18517 std::string &Constraint,
18518 std::vector<SDValue>&Ops,
18519 SelectionDAG &DAG) const {
18520 SDValue Result;
18521
18522 // Currently only support length 1 constraints.
18523 if (Constraint.length() != 1) return;
18524
18525 char ConstraintLetter = Constraint[0];
18526 switch (ConstraintLetter) {
18527 default: break;
18528 case 'j':
18529 case 'I': case 'J': case 'K': case 'L':
18530 case 'M': case 'N': case 'O':
18531 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
18532 if (!C)
18533 return;
18534
18535 int64_t CVal64 = C->getSExtValue();
18536 int CVal = (int) CVal64;
18537 // None of these constraints allow values larger than 32 bits. Check
18538 // that the value fits in an int.
18539 if (CVal != CVal64)
18540 return;
18541
18542 switch (ConstraintLetter) {
18543 case 'j':
18544 // Constant suitable for movw, must be between 0 and
18545 // 65535.
18546 if (Subtarget->hasV6T2Ops() || (Subtarget->hasV8MBaselineOps()))
18547 if (CVal >= 0 && CVal <= 65535)
18548 break;
18549 return;
18550 case 'I':
18551 if (Subtarget->isThumb1Only()) {
18552 // This must be a constant between 0 and 255, for ADD
18553 // immediates.
18554 if (CVal >= 0 && CVal <= 255)
18555 break;
18556 } else if (Subtarget->isThumb2()) {
18557 // A constant that can be used as an immediate value in a
18558 // data-processing instruction.
18559 if (ARM_AM::getT2SOImmVal(CVal) != -1)
18560 break;
18561 } else {
18562 // A constant that can be used as an immediate value in a
18563 // data-processing instruction.
18564 if (ARM_AM::getSOImmVal(CVal) != -1)
18565 break;
18566 }
18567 return;
18568
18569 case 'J':
18570 if (Subtarget->isThumb1Only()) {
18571 // This must be a constant between -255 and -1, for negated ADD
18572 // immediates. This can be used in GCC with an "n" modifier that
18573 // prints the negated value, for use with SUB instructions. It is
18574 // not useful otherwise but is implemented for compatibility.
18575 if (CVal >= -255 && CVal <= -1)
18576 break;
18577 } else {
18578 // This must be a constant between -4095 and 4095. It is not clear
18579 // what this constraint is intended for. Implemented for
18580 // compatibility with GCC.
18581 if (CVal >= -4095 && CVal <= 4095)
18582 break;
18583 }
18584 return;
18585
18586 case 'K':
18587 if (Subtarget->isThumb1Only()) {
18588 // A 32-bit value where only one byte has a nonzero value. Exclude
18589 // zero to match GCC. This constraint is used by GCC internally for
18590 // constants that can be loaded with a move/shift combination.
18591 // It is not useful otherwise but is implemented for compatibility.
18592 if (CVal != 0 && ARM_AM::isThumbImmShiftedVal(CVal))
18593 break;
18594 } else if (Subtarget->isThumb2()) {
18595 // A constant whose bitwise inverse can be used as an immediate
18596 // value in a data-processing instruction. This can be used in GCC
18597 // with a "B" modifier that prints the inverted value, for use with
18598 // BIC and MVN instructions. It is not useful otherwise but is
18599 // implemented for compatibility.
18600 if (ARM_AM::getT2SOImmVal(~CVal) != -1)
18601 break;
18602 } else {
18603 // A constant whose bitwise inverse can be used as an immediate
18604 // value in a data-processing instruction. This can be used in GCC
18605 // with a "B" modifier that prints the inverted value, for use with
18606 // BIC and MVN instructions. It is not useful otherwise but is
18607 // implemented for compatibility.
18608 if (ARM_AM::getSOImmVal(~CVal) != -1)
18609 break;
18610 }
18611 return;
18612
18613 case 'L':
18614 if (Subtarget->isThumb1Only()) {
18615 // This must be a constant between -7 and 7,
18616 // for 3-operand ADD/SUB immediate instructions.
18617 if (CVal >= -7 && CVal < 7)
18618 break;
18619 } else if (Subtarget->isThumb2()) {
18620 // A constant whose negation can be used as an immediate value in a
18621 // data-processing instruction. This can be used in GCC with an "n"
18622 // modifier that prints the negated value, for use with SUB
18623 // instructions. It is not useful otherwise but is implemented for
18624 // compatibility.
18625 if (ARM_AM::getT2SOImmVal(-CVal) != -1)
18626 break;
18627 } else {
18628 // A constant whose negation can be used as an immediate value in a
18629 // data-processing instruction. This can be used in GCC with an "n"
18630 // modifier that prints the negated value, for use with SUB
18631 // instructions. It is not useful otherwise but is implemented for
18632 // compatibility.
18633 if (ARM_AM::getSOImmVal(-CVal) != -1)
18634 break;
18635 }
18636 return;
18637
18638 case 'M':
18639 if (Subtarget->isThumb1Only()) {
18640 // This must be a multiple of 4 between 0 and 1020, for
18641 // ADD sp + immediate.
18642 if ((CVal >= 0 && CVal <= 1020) && ((CVal & 3) == 0))
18643 break;
18644 } else {
18645 // A power of two or a constant between 0 and 32. This is used in
18646 // GCC for the shift amount on shifted register operands, but it is
18647 // useful in general for any shift amounts.
18648 if ((CVal >= 0 && CVal <= 32) || ((CVal & (CVal - 1)) == 0))
18649 break;
18650 }
18651 return;
18652
18653 case 'N':
18654 if (Subtarget->isThumb1Only()) {
18655 // This must be a constant between 0 and 31, for shift amounts.
18656 if (CVal >= 0 && CVal <= 31)
18657 break;
18658 }
18659 return;
18660
18661 case 'O':
18662 if (Subtarget->isThumb1Only()) {
18663 // This must be a multiple of 4 between -508 and 508, for
18664 // ADD/SUB sp = sp + immediate.
18665 if ((CVal >= -508 && CVal <= 508) && ((CVal & 3) == 0))
18666 break;
18667 }
18668 return;
18669 }
18670 Result = DAG.getTargetConstant(CVal, SDLoc(Op), Op.getValueType());
18671 break;
18672 }
18673
18674 if (Result.getNode()) {
18675 Ops.push_back(Result);
18676 return;
18677 }
18678 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
18679}
18680
18681static RTLIB::Libcall getDivRemLibcall(
18682 const SDNode *N, MVT::SimpleValueType SVT) {
18683 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemLibcall"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemLibcall\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18685, __extension__ __PRETTY_FUNCTION__))
18684 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemLibcall"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemLibcall\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18685, __extension__ __PRETTY_FUNCTION__))
18685 "Unhandled Opcode in getDivRemLibcall")(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemLibcall"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemLibcall\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18685, __extension__ __PRETTY_FUNCTION__))
;
18686 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
18687 N->getOpcode() == ISD::SREM;
18688 RTLIB::Libcall LC;
18689 switch (SVT) {
18690 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18690)
;
18691 case MVT::i8: LC = isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
18692 case MVT::i16: LC = isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
18693 case MVT::i32: LC = isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
18694 case MVT::i64: LC = isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
18695 }
18696 return LC;
18697}
18698
18699static TargetLowering::ArgListTy getDivRemArgList(
18700 const SDNode *N, LLVMContext *Context, const ARMSubtarget *Subtarget) {
18701 assert((N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM ||(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemArgList"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemArgList\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18703, __extension__ __PRETTY_FUNCTION__))
18702 N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) &&(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemArgList"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemArgList\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18703, __extension__ __PRETTY_FUNCTION__))
18703 "Unhandled Opcode in getDivRemArgList")(static_cast <bool> ((N->getOpcode() == ISD::SDIVREM
|| N->getOpcode() == ISD::UDIVREM || N->getOpcode() ==
ISD::SREM || N->getOpcode() == ISD::UREM) && "Unhandled Opcode in getDivRemArgList"
) ? void (0) : __assert_fail ("(N->getOpcode() == ISD::SDIVREM || N->getOpcode() == ISD::UDIVREM || N->getOpcode() == ISD::SREM || N->getOpcode() == ISD::UREM) && \"Unhandled Opcode in getDivRemArgList\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18703, __extension__ __PRETTY_FUNCTION__))
;
18704 bool isSigned = N->getOpcode() == ISD::SDIVREM ||
18705 N->getOpcode() == ISD::SREM;
18706 TargetLowering::ArgListTy Args;
18707 TargetLowering::ArgListEntry Entry;
18708 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
18709 EVT ArgVT = N->getOperand(i).getValueType();
18710 Type *ArgTy = ArgVT.getTypeForEVT(*Context);
18711 Entry.Node = N->getOperand(i);
18712 Entry.Ty = ArgTy;
18713 Entry.IsSExt = isSigned;
18714 Entry.IsZExt = !isSigned;
18715 Args.push_back(Entry);
18716 }
18717 if (Subtarget->isTargetWindows() && Args.size() >= 2)
18718 std::swap(Args[0], Args[1]);
18719 return Args;
18720}
18721
18722SDValue ARMTargetLowering::LowerDivRem(SDValue Op, SelectionDAG &DAG) const {
18723 assert((Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() ||(static_cast <bool> ((Subtarget->isTargetAEABI() || Subtarget
->isTargetAndroid() || Subtarget->isTargetGNUAEABI() ||
Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows
()) && "Register-based DivRem lowering only") ? void (
0) : __assert_fail ("(Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows()) && \"Register-based DivRem lowering only\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18726, __extension__ __PRETTY_FUNCTION__))
18724 Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() ||(static_cast <bool> ((Subtarget->isTargetAEABI() || Subtarget
->isTargetAndroid() || Subtarget->isTargetGNUAEABI() ||
Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows
()) && "Register-based DivRem lowering only") ? void (
0) : __assert_fail ("(Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows()) && \"Register-based DivRem lowering only\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18726, __extension__ __PRETTY_FUNCTION__))
18725 Subtarget->isTargetWindows()) &&(static_cast <bool> ((Subtarget->isTargetAEABI() || Subtarget
->isTargetAndroid() || Subtarget->isTargetGNUAEABI() ||
Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows
()) && "Register-based DivRem lowering only") ? void (
0) : __assert_fail ("(Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows()) && \"Register-based DivRem lowering only\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18726, __extension__ __PRETTY_FUNCTION__))
18726 "Register-based DivRem lowering only")(static_cast <bool> ((Subtarget->isTargetAEABI() || Subtarget
->isTargetAndroid() || Subtarget->isTargetGNUAEABI() ||
Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows
()) && "Register-based DivRem lowering only") ? void (
0) : __assert_fail ("(Subtarget->isTargetAEABI() || Subtarget->isTargetAndroid() || Subtarget->isTargetGNUAEABI() || Subtarget->isTargetMuslAEABI() || Subtarget->isTargetWindows()) && \"Register-based DivRem lowering only\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18726, __extension__ __PRETTY_FUNCTION__))
;
18727 unsigned Opcode = Op->getOpcode();
18728 assert((Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) &&(static_cast <bool> ((Opcode == ISD::SDIVREM || Opcode ==
ISD::UDIVREM) && "Invalid opcode for Div/Rem lowering"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) && \"Invalid opcode for Div/Rem lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18729, __extension__ __PRETTY_FUNCTION__))
18729 "Invalid opcode for Div/Rem lowering")(static_cast <bool> ((Opcode == ISD::SDIVREM || Opcode ==
ISD::UDIVREM) && "Invalid opcode for Div/Rem lowering"
) ? void (0) : __assert_fail ("(Opcode == ISD::SDIVREM || Opcode == ISD::UDIVREM) && \"Invalid opcode for Div/Rem lowering\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18729, __extension__ __PRETTY_FUNCTION__))
;
18730 bool isSigned = (Opcode == ISD::SDIVREM);
18731 EVT VT = Op->getValueType(0);
18732 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
18733 SDLoc dl(Op);
18734
18735 // If the target has hardware divide, use divide + multiply + subtract:
18736 // div = a / b
18737 // rem = a - b * div
18738 // return {div, rem}
18739 // This should be lowered into UDIV/SDIV + MLS later on.
18740 bool hasDivide = Subtarget->isThumb() ? Subtarget->hasDivideInThumbMode()
18741 : Subtarget->hasDivideInARMMode();
18742 if (hasDivide && Op->getValueType(0).isSimple() &&
18743 Op->getSimpleValueType(0) == MVT::i32) {
18744 unsigned DivOpcode = isSigned ? ISD::SDIV : ISD::UDIV;
18745 const SDValue Dividend = Op->getOperand(0);
18746 const SDValue Divisor = Op->getOperand(1);
18747 SDValue Div = DAG.getNode(DivOpcode, dl, VT, Dividend, Divisor);
18748 SDValue Mul = DAG.getNode(ISD::MUL, dl, VT, Div, Divisor);
18749 SDValue Rem = DAG.getNode(ISD::SUB, dl, VT, Dividend, Mul);
18750
18751 SDValue Values[2] = {Div, Rem};
18752 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(VT, VT), Values);
18753 }
18754
18755 RTLIB::Libcall LC = getDivRemLibcall(Op.getNode(),
18756 VT.getSimpleVT().SimpleTy);
18757 SDValue InChain = DAG.getEntryNode();
18758
18759 TargetLowering::ArgListTy Args = getDivRemArgList(Op.getNode(),
18760 DAG.getContext(),
18761 Subtarget);
18762
18763 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
18764 getPointerTy(DAG.getDataLayout()));
18765
18766 Type *RetTy = StructType::get(Ty, Ty);
18767
18768 if (Subtarget->isTargetWindows())
18769 InChain = WinDBZCheckDenominator(DAG, Op.getNode(), InChain);
18770
18771 TargetLowering::CallLoweringInfo CLI(DAG);
18772 CLI.setDebugLoc(dl).setChain(InChain)
18773 .setCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args))
18774 .setInRegister().setSExtResult(isSigned).setZExtResult(!isSigned);
18775
18776 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
18777 return CallInfo.first;
18778}
18779
18780// Lowers REM using divmod helpers
18781// see RTABI section 4.2/4.3
18782SDValue ARMTargetLowering::LowerREM(SDNode *N, SelectionDAG &DAG) const {
18783 // Build return types (div and rem)
18784 std::vector<Type*> RetTyParams;
18785 Type *RetTyElement;
18786
18787 switch (N->getValueType(0).getSimpleVT().SimpleTy) {
18788 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18788)
;
18789 case MVT::i8: RetTyElement = Type::getInt8Ty(*DAG.getContext()); break;
18790 case MVT::i16: RetTyElement = Type::getInt16Ty(*DAG.getContext()); break;
18791 case MVT::i32: RetTyElement = Type::getInt32Ty(*DAG.getContext()); break;
18792 case MVT::i64: RetTyElement = Type::getInt64Ty(*DAG.getContext()); break;
18793 }
18794
18795 RetTyParams.push_back(RetTyElement);
18796 RetTyParams.push_back(RetTyElement);
18797 ArrayRef<Type*> ret = ArrayRef<Type*>(RetTyParams);
18798 Type *RetTy = StructType::get(*DAG.getContext(), ret);
18799
18800 RTLIB::Libcall LC = getDivRemLibcall(N, N->getValueType(0).getSimpleVT().
18801 SimpleTy);
18802 SDValue InChain = DAG.getEntryNode();
18803 TargetLowering::ArgListTy Args = getDivRemArgList(N, DAG.getContext(),
18804 Subtarget);
18805 bool isSigned = N->getOpcode() == ISD::SREM;
18806 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
18807 getPointerTy(DAG.getDataLayout()));
18808
18809 if (Subtarget->isTargetWindows())
18810 InChain = WinDBZCheckDenominator(DAG, N, InChain);
18811
18812 // Lower call
18813 CallLoweringInfo CLI(DAG);
18814 CLI.setChain(InChain)
18815 .setCallee(CallingConv::ARM_AAPCS, RetTy, Callee, std::move(Args))
18816 .setSExtResult(isSigned).setZExtResult(!isSigned).setDebugLoc(SDLoc(N));
18817 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
18818
18819 // Return second (rem) result operand (first contains div)
18820 SDNode *ResNode = CallResult.first.getNode();
18821 assert(ResNode->getNumOperands() == 2 && "divmod should return two operands")(static_cast <bool> (ResNode->getNumOperands() == 2 &&
"divmod should return two operands") ? void (0) : __assert_fail
("ResNode->getNumOperands() == 2 && \"divmod should return two operands\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18821, __extension__ __PRETTY_FUNCTION__))
;
18822 return ResNode->getOperand(1);
18823}
18824
18825SDValue
18826ARMTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const {
18827 assert(Subtarget->isTargetWindows() && "unsupported target platform")(static_cast <bool> (Subtarget->isTargetWindows() &&
"unsupported target platform") ? void (0) : __assert_fail ("Subtarget->isTargetWindows() && \"unsupported target platform\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18827, __extension__ __PRETTY_FUNCTION__))
;
18828 SDLoc DL(Op);
18829
18830 // Get the inputs.
18831 SDValue Chain = Op.getOperand(0);
18832 SDValue Size = Op.getOperand(1);
18833
18834 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
18835 "no-stack-arg-probe")) {
18836 MaybeAlign Align =
18837 cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
18838 SDValue SP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
18839 Chain = SP.getValue(1);
18840 SP = DAG.getNode(ISD::SUB, DL, MVT::i32, SP, Size);
18841 if (Align)
18842 SP =
18843 DAG.getNode(ISD::AND, DL, MVT::i32, SP.getValue(0),
18844 DAG.getConstant(-(uint64_t)Align->value(), DL, MVT::i32));
18845 Chain = DAG.getCopyToReg(Chain, DL, ARM::SP, SP);
18846 SDValue Ops[2] = { SP, Chain };
18847 return DAG.getMergeValues(Ops, DL);
18848 }
18849
18850 SDValue Words = DAG.getNode(ISD::SRL, DL, MVT::i32, Size,
18851 DAG.getConstant(2, DL, MVT::i32));
18852
18853 SDValue Flag;
18854 Chain = DAG.getCopyToReg(Chain, DL, ARM::R4, Words, Flag);
18855 Flag = Chain.getValue(1);
18856
18857 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18858 Chain = DAG.getNode(ARMISD::WIN__CHKSTK, DL, NodeTys, Chain, Flag);
18859
18860 SDValue NewSP = DAG.getCopyFromReg(Chain, DL, ARM::SP, MVT::i32);
18861 Chain = NewSP.getValue(1);
18862
18863 SDValue Ops[2] = { NewSP, Chain };
18864 return DAG.getMergeValues(Ops, DL);
18865}
18866
18867SDValue ARMTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
18868 bool IsStrict = Op->isStrictFPOpcode();
18869 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
18870 const unsigned DstSz = Op.getValueType().getSizeInBits();
18871 const unsigned SrcSz = SrcVal.getValueType().getSizeInBits();
18872 assert(DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 &&(static_cast <bool> (DstSz > SrcSz && DstSz <=
64 && SrcSz >= 16 && "Unexpected type for custom-lowering FP_EXTEND"
) ? void (0) : __assert_fail ("DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 && \"Unexpected type for custom-lowering FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18873, __extension__ __PRETTY_FUNCTION__))
18873 "Unexpected type for custom-lowering FP_EXTEND")(static_cast <bool> (DstSz > SrcSz && DstSz <=
64 && SrcSz >= 16 && "Unexpected type for custom-lowering FP_EXTEND"
) ? void (0) : __assert_fail ("DstSz > SrcSz && DstSz <= 64 && SrcSz >= 16 && \"Unexpected type for custom-lowering FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18873, __extension__ __PRETTY_FUNCTION__))
;
18874
18875 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&(static_cast <bool> ((!Subtarget->hasFP64() || !Subtarget
->hasFPARMv8Base()) && "With both FP DP and 16, any FP conversion is legal!"
) ? void (0) : __assert_fail ("(!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && \"With both FP DP and 16, any FP conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18876, __extension__ __PRETTY_FUNCTION__))
18876 "With both FP DP and 16, any FP conversion is legal!")(static_cast <bool> ((!Subtarget->hasFP64() || !Subtarget
->hasFPARMv8Base()) && "With both FP DP and 16, any FP conversion is legal!"
) ? void (0) : __assert_fail ("(!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && \"With both FP DP and 16, any FP conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18876, __extension__ __PRETTY_FUNCTION__))
;
18877
18878 assert(!(DstSz == 32 && Subtarget->hasFP16()) &&(static_cast <bool> (!(DstSz == 32 && Subtarget
->hasFP16()) && "With FP16, 16 to 32 conversion is legal!"
) ? void (0) : __assert_fail ("!(DstSz == 32 && Subtarget->hasFP16()) && \"With FP16, 16 to 32 conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18879, __extension__ __PRETTY_FUNCTION__))
18879 "With FP16, 16 to 32 conversion is legal!")(static_cast <bool> (!(DstSz == 32 && Subtarget
->hasFP16()) && "With FP16, 16 to 32 conversion is legal!"
) ? void (0) : __assert_fail ("!(DstSz == 32 && Subtarget->hasFP16()) && \"With FP16, 16 to 32 conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18879, __extension__ __PRETTY_FUNCTION__))
;
18880
18881 // Converting from 32 -> 64 is valid if we have FP64.
18882 if (SrcSz == 32 && DstSz == 64 && Subtarget->hasFP64()) {
18883 // FIXME: Remove this when we have strict fp instruction selection patterns
18884 if (IsStrict) {
18885 SDLoc Loc(Op);
18886 SDValue Result = DAG.getNode(ISD::FP_EXTEND,
18887 Loc, Op.getValueType(), SrcVal);
18888 return DAG.getMergeValues({Result, Op.getOperand(0)}, Loc);
18889 }
18890 return Op;
18891 }
18892
18893 // Either we are converting from 16 -> 64, without FP16 and/or
18894 // FP.double-precision or without Armv8-fp. So we must do it in two
18895 // steps.
18896 // Or we are converting from 32 -> 64 without fp.double-precision or 16 -> 32
18897 // without FP16. So we must do a function call.
18898 SDLoc Loc(Op);
18899 RTLIB::Libcall LC;
18900 MakeLibCallOptions CallOptions;
18901 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
18902 for (unsigned Sz = SrcSz; Sz <= 32 && Sz < DstSz; Sz *= 2) {
18903 bool Supported = (Sz == 16 ? Subtarget->hasFP16() : Subtarget->hasFP64());
18904 MVT SrcVT = (Sz == 16 ? MVT::f16 : MVT::f32);
18905 MVT DstVT = (Sz == 16 ? MVT::f32 : MVT::f64);
18906 if (Supported) {
18907 if (IsStrict) {
18908 SrcVal = DAG.getNode(ISD::STRICT_FP_EXTEND, Loc,
18909 {DstVT, MVT::Other}, {Chain, SrcVal});
18910 Chain = SrcVal.getValue(1);
18911 } else {
18912 SrcVal = DAG.getNode(ISD::FP_EXTEND, Loc, DstVT, SrcVal);
18913 }
18914 } else {
18915 LC = RTLIB::getFPEXT(SrcVT, DstVT);
18916 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected type for custom-lowering FP_EXTEND") ? void (0) :
__assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected type for custom-lowering FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18917, __extension__ __PRETTY_FUNCTION__))
18917 "Unexpected type for custom-lowering FP_EXTEND")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected type for custom-lowering FP_EXTEND") ? void (0) :
__assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected type for custom-lowering FP_EXTEND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18917, __extension__ __PRETTY_FUNCTION__))
;
18918 std::tie(SrcVal, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
18919 Loc, Chain);
18920 }
18921 }
18922
18923 return IsStrict ? DAG.getMergeValues({SrcVal, Chain}, Loc) : SrcVal;
18924}
18925
18926SDValue ARMTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
18927 bool IsStrict = Op->isStrictFPOpcode();
18928
18929 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
18930 EVT SrcVT = SrcVal.getValueType();
18931 EVT DstVT = Op.getValueType();
18932 const unsigned DstSz = Op.getValueType().getSizeInBits();
18933 const unsigned SrcSz = SrcVT.getSizeInBits();
18934 (void)DstSz;
18935 assert(DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 &&(static_cast <bool> (DstSz < SrcSz && SrcSz <=
64 && DstSz >= 16 && "Unexpected type for custom-lowering FP_ROUND"
) ? void (0) : __assert_fail ("DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 && \"Unexpected type for custom-lowering FP_ROUND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18936, __extension__ __PRETTY_FUNCTION__))
18936 "Unexpected type for custom-lowering FP_ROUND")(static_cast <bool> (DstSz < SrcSz && SrcSz <=
64 && DstSz >= 16 && "Unexpected type for custom-lowering FP_ROUND"
) ? void (0) : __assert_fail ("DstSz < SrcSz && SrcSz <= 64 && DstSz >= 16 && \"Unexpected type for custom-lowering FP_ROUND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18936, __extension__ __PRETTY_FUNCTION__))
;
18937
18938 assert((!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) &&(static_cast <bool> ((!Subtarget->hasFP64() || !Subtarget
->hasFPARMv8Base()) && "With both FP DP and 16, any FP conversion is legal!"
) ? void (0) : __assert_fail ("(!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && \"With both FP DP and 16, any FP conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18939, __extension__ __PRETTY_FUNCTION__))
18939 "With both FP DP and 16, any FP conversion is legal!")(static_cast <bool> ((!Subtarget->hasFP64() || !Subtarget
->hasFPARMv8Base()) && "With both FP DP and 16, any FP conversion is legal!"
) ? void (0) : __assert_fail ("(!Subtarget->hasFP64() || !Subtarget->hasFPARMv8Base()) && \"With both FP DP and 16, any FP conversion is legal!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18939, __extension__ __PRETTY_FUNCTION__))
;
18940
18941 SDLoc Loc(Op);
18942
18943 // Instruction from 32 -> 16 if hasFP16 is valid
18944 if (SrcSz == 32 && Subtarget->hasFP16())
18945 return Op;
18946
18947 // Lib call from 32 -> 16 / 64 -> [32, 16]
18948 RTLIB::Libcall LC = RTLIB::getFPROUND(SrcVT, DstVT);
18949 assert(LC != RTLIB::UNKNOWN_LIBCALL &&(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected type for custom-lowering FP_ROUND") ? void (0) :
__assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected type for custom-lowering FP_ROUND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18950, __extension__ __PRETTY_FUNCTION__))
18950 "Unexpected type for custom-lowering FP_ROUND")(static_cast <bool> (LC != RTLIB::UNKNOWN_LIBCALL &&
"Unexpected type for custom-lowering FP_ROUND") ? void (0) :
__assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected type for custom-lowering FP_ROUND\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18950, __extension__ __PRETTY_FUNCTION__))
;
18951 MakeLibCallOptions CallOptions;
18952 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
18953 SDValue Result;
18954 std::tie(Result, Chain) = makeLibCall(DAG, LC, DstVT, SrcVal, CallOptions,
18955 Loc, Chain);
18956 return IsStrict ? DAG.getMergeValues({Result, Chain}, Loc) : Result;
18957}
18958
18959void ARMTargetLowering::lowerABS(SDNode *N, SmallVectorImpl<SDValue> &Results,
18960 SelectionDAG &DAG) const {
18961 assert(N->getValueType(0) == MVT::i64 && "Unexpected type (!= i64) on ABS.")(static_cast <bool> (N->getValueType(0) == MVT::i64 &&
"Unexpected type (!= i64) on ABS.") ? void (0) : __assert_fail
("N->getValueType(0) == MVT::i64 && \"Unexpected type (!= i64) on ABS.\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 18961, __extension__ __PRETTY_FUNCTION__))
;
18962 MVT HalfT = MVT::i32;
18963 SDLoc dl(N);
18964 SDValue Hi, Lo, Tmp;
18965
18966 if (!isOperationLegalOrCustom(ISD::ADDCARRY, HalfT) ||
18967 !isOperationLegalOrCustom(ISD::UADDO, HalfT))
18968 return ;
18969
18970 unsigned OpTypeBits = HalfT.getScalarSizeInBits();
18971 SDVTList VTList = DAG.getVTList(HalfT, MVT::i1);
18972
18973 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
18974 DAG.getConstant(0, dl, HalfT));
18975 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
18976 DAG.getConstant(1, dl, HalfT));
18977
18978 Tmp = DAG.getNode(ISD::SRA, dl, HalfT, Hi,
18979 DAG.getConstant(OpTypeBits - 1, dl,
18980 getShiftAmountTy(HalfT, DAG.getDataLayout())));
18981 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Tmp, Lo);
18982 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Tmp, Hi,
18983 SDValue(Lo.getNode(), 1));
18984 Hi = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Hi);
18985 Lo = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Lo);
18986
18987 Results.push_back(Lo);
18988 Results.push_back(Hi);
18989}
18990
18991bool
18992ARMTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
18993 // The ARM target isn't yet aware of offsets.
18994 return false;
18995}
18996
18997bool ARM::isBitFieldInvertedMask(unsigned v) {
18998 if (v == 0xffffffff)
18999 return false;
19000
19001 // there can be 1's on either or both "outsides", all the "inside"
19002 // bits must be 0's
19003 return isShiftedMask_32(~v);
19004}
19005
19006/// isFPImmLegal - Returns true if the target can instruction select the
19007/// specified FP immediate natively. If false, the legalizer will
19008/// materialize the FP immediate as a load from a constant pool.
19009bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
19010 bool ForCodeSize) const {
19011 if (!Subtarget->hasVFP3Base())
19012 return false;
19013 if (VT == MVT::f16 && Subtarget->hasFullFP16())
19014 return ARM_AM::getFP16Imm(Imm) != -1;
19015 if (VT == MVT::f32 && Subtarget->hasFullFP16() &&
19016 ARM_AM::getFP32FP16Imm(Imm) != -1)
19017 return true;
19018 if (VT == MVT::f32)
19019 return ARM_AM::getFP32Imm(Imm) != -1;
19020 if (VT == MVT::f64 && Subtarget->hasFP64())
19021 return ARM_AM::getFP64Imm(Imm) != -1;
19022 return false;
19023}
19024
19025/// getTgtMemIntrinsic - Represent NEON load and store intrinsics as
19026/// MemIntrinsicNodes. The associated MachineMemOperands record the alignment
19027/// specified in the intrinsic calls.
19028bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
19029 const CallInst &I,
19030 MachineFunction &MF,
19031 unsigned Intrinsic) const {
19032 switch (Intrinsic) {
19033 case Intrinsic::arm_neon_vld1:
19034 case Intrinsic::arm_neon_vld2:
19035 case Intrinsic::arm_neon_vld3:
19036 case Intrinsic::arm_neon_vld4:
19037 case Intrinsic::arm_neon_vld2lane:
19038 case Intrinsic::arm_neon_vld3lane:
19039 case Intrinsic::arm_neon_vld4lane:
19040 case Intrinsic::arm_neon_vld2dup:
19041 case Intrinsic::arm_neon_vld3dup:
19042 case Intrinsic::arm_neon_vld4dup: {
19043 Info.opc = ISD::INTRINSIC_W_CHAIN;
19044 // Conservatively set memVT to the entire set of vectors loaded.
19045 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19046 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
19047 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
19048 Info.ptrVal = I.getArgOperand(0);
19049 Info.offset = 0;
19050 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
19051 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
19052 // volatile loads with NEON intrinsics not supported
19053 Info.flags = MachineMemOperand::MOLoad;
19054 return true;
19055 }
19056 case Intrinsic::arm_neon_vld1x2:
19057 case Intrinsic::arm_neon_vld1x3:
19058 case Intrinsic::arm_neon_vld1x4: {
19059 Info.opc = ISD::INTRINSIC_W_CHAIN;
19060 // Conservatively set memVT to the entire set of vectors loaded.
19061 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19062 uint64_t NumElts = DL.getTypeSizeInBits(I.getType()) / 64;
19063 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
19064 Info.ptrVal = I.getArgOperand(I.getNumArgOperands() - 1);
19065 Info.offset = 0;
19066 Info.align.reset();
19067 // volatile loads with NEON intrinsics not supported
19068 Info.flags = MachineMemOperand::MOLoad;
19069 return true;
19070 }
19071 case Intrinsic::arm_neon_vst1:
19072 case Intrinsic::arm_neon_vst2:
19073 case Intrinsic::arm_neon_vst3:
19074 case Intrinsic::arm_neon_vst4:
19075 case Intrinsic::arm_neon_vst2lane:
19076 case Intrinsic::arm_neon_vst3lane:
19077 case Intrinsic::arm_neon_vst4lane: {
19078 Info.opc = ISD::INTRINSIC_VOID;
19079 // Conservatively set memVT to the entire set of vectors stored.
19080 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19081 unsigned NumElts = 0;
19082 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
19083 Type *ArgTy = I.getArgOperand(ArgI)->getType();
19084 if (!ArgTy->isVectorTy())
19085 break;
19086 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
19087 }
19088 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
19089 Info.ptrVal = I.getArgOperand(0);
19090 Info.offset = 0;
19091 Value *AlignArg = I.getArgOperand(I.getNumArgOperands() - 1);
19092 Info.align = cast<ConstantInt>(AlignArg)->getMaybeAlignValue();
19093 // volatile stores with NEON intrinsics not supported
19094 Info.flags = MachineMemOperand::MOStore;
19095 return true;
19096 }
19097 case Intrinsic::arm_neon_vst1x2:
19098 case Intrinsic::arm_neon_vst1x3:
19099 case Intrinsic::arm_neon_vst1x4: {
19100 Info.opc = ISD::INTRINSIC_VOID;
19101 // Conservatively set memVT to the entire set of vectors stored.
19102 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19103 unsigned NumElts = 0;
19104 for (unsigned ArgI = 1, ArgE = I.getNumArgOperands(); ArgI < ArgE; ++ArgI) {
19105 Type *ArgTy = I.getArgOperand(ArgI)->getType();
19106 if (!ArgTy->isVectorTy())
19107 break;
19108 NumElts += DL.getTypeSizeInBits(ArgTy) / 64;
19109 }
19110 Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
19111 Info.ptrVal = I.getArgOperand(0);
19112 Info.offset = 0;
19113 Info.align.reset();
19114 // volatile stores with NEON intrinsics not supported
19115 Info.flags = MachineMemOperand::MOStore;
19116 return true;
19117 }
19118 case Intrinsic::arm_mve_vld2q:
19119 case Intrinsic::arm_mve_vld4q: {
19120 Info.opc = ISD::INTRINSIC_W_CHAIN;
19121 // Conservatively set memVT to the entire set of vectors loaded.
19122 Type *VecTy = cast<StructType>(I.getType())->getElementType(1);
19123 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vld2q ? 2 : 4;
19124 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
19125 Info.ptrVal = I.getArgOperand(0);
19126 Info.offset = 0;
19127 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
19128 // volatile loads with MVE intrinsics not supported
19129 Info.flags = MachineMemOperand::MOLoad;
19130 return true;
19131 }
19132 case Intrinsic::arm_mve_vst2q:
19133 case Intrinsic::arm_mve_vst4q: {
19134 Info.opc = ISD::INTRINSIC_VOID;
19135 // Conservatively set memVT to the entire set of vectors stored.
19136 Type *VecTy = I.getArgOperand(1)->getType();
19137 unsigned Factor = Intrinsic == Intrinsic::arm_mve_vst2q ? 2 : 4;
19138 Info.memVT = EVT::getVectorVT(VecTy->getContext(), MVT::i64, Factor * 2);
19139 Info.ptrVal = I.getArgOperand(0);
19140 Info.offset = 0;
19141 Info.align = Align(VecTy->getScalarSizeInBits() / 8);
19142 // volatile stores with MVE intrinsics not supported
19143 Info.flags = MachineMemOperand::MOStore;
19144 return true;
19145 }
19146 case Intrinsic::arm_mve_vldr_gather_base:
19147 case Intrinsic::arm_mve_vldr_gather_base_predicated: {
19148 Info.opc = ISD::INTRINSIC_W_CHAIN;
19149 Info.ptrVal = nullptr;
19150 Info.memVT = MVT::getVT(I.getType());
19151 Info.align = Align(1);
19152 Info.flags |= MachineMemOperand::MOLoad;
19153 return true;
19154 }
19155 case Intrinsic::arm_mve_vldr_gather_base_wb:
19156 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
19157 Info.opc = ISD::INTRINSIC_W_CHAIN;
19158 Info.ptrVal = nullptr;
19159 Info.memVT = MVT::getVT(I.getType()->getContainedType(0));
19160 Info.align = Align(1);
19161 Info.flags |= MachineMemOperand::MOLoad;
19162 return true;
19163 }
19164 case Intrinsic::arm_mve_vldr_gather_offset:
19165 case Intrinsic::arm_mve_vldr_gather_offset_predicated: {
19166 Info.opc = ISD::INTRINSIC_W_CHAIN;
19167 Info.ptrVal = nullptr;
19168 MVT DataVT = MVT::getVT(I.getType());
19169 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
19170 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
19171 DataVT.getVectorNumElements());
19172 Info.align = Align(1);
19173 Info.flags |= MachineMemOperand::MOLoad;
19174 return true;
19175 }
19176 case Intrinsic::arm_mve_vstr_scatter_base:
19177 case Intrinsic::arm_mve_vstr_scatter_base_predicated: {
19178 Info.opc = ISD::INTRINSIC_VOID;
19179 Info.ptrVal = nullptr;
19180 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
19181 Info.align = Align(1);
19182 Info.flags |= MachineMemOperand::MOStore;
19183 return true;
19184 }
19185 case Intrinsic::arm_mve_vstr_scatter_base_wb:
19186 case Intrinsic::arm_mve_vstr_scatter_base_wb_predicated: {
19187 Info.opc = ISD::INTRINSIC_W_CHAIN;
19188 Info.ptrVal = nullptr;
19189 Info.memVT = MVT::getVT(I.getArgOperand(2)->getType());
19190 Info.align = Align(1);
19191 Info.flags |= MachineMemOperand::MOStore;
19192 return true;
19193 }
19194 case Intrinsic::arm_mve_vstr_scatter_offset:
19195 case Intrinsic::arm_mve_vstr_scatter_offset_predicated: {
19196 Info.opc = ISD::INTRINSIC_VOID;
19197 Info.ptrVal = nullptr;
19198 MVT DataVT = MVT::getVT(I.getArgOperand(2)->getType());
19199 unsigned MemSize = cast<ConstantInt>(I.getArgOperand(3))->getZExtValue();
19200 Info.memVT = MVT::getVectorVT(MVT::getIntegerVT(MemSize),
19201 DataVT.getVectorNumElements());
19202 Info.align = Align(1);
19203 Info.flags |= MachineMemOperand::MOStore;
19204 return true;
19205 }
19206 case Intrinsic::arm_ldaex:
19207 case Intrinsic::arm_ldrex: {
19208 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19209 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
19210 Info.opc = ISD::INTRINSIC_W_CHAIN;
19211 Info.memVT = MVT::getVT(PtrTy->getElementType());
19212 Info.ptrVal = I.getArgOperand(0);
19213 Info.offset = 0;
19214 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
19215 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19216 return true;
19217 }
19218 case Intrinsic::arm_stlex:
19219 case Intrinsic::arm_strex: {
19220 auto &DL = I.getCalledFunction()->getParent()->getDataLayout();
19221 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(1)->getType());
19222 Info.opc = ISD::INTRINSIC_W_CHAIN;
19223 Info.memVT = MVT::getVT(PtrTy->getElementType());
19224 Info.ptrVal = I.getArgOperand(1);
19225 Info.offset = 0;
19226 Info.align = DL.getABITypeAlign(PtrTy->getElementType());
19227 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19228 return true;
19229 }
19230 case Intrinsic::arm_stlexd:
19231 case Intrinsic::arm_strexd:
19232 Info.opc = ISD::INTRINSIC_W_CHAIN;
19233 Info.memVT = MVT::i64;
19234 Info.ptrVal = I.getArgOperand(2);
19235 Info.offset = 0;
19236 Info.align = Align(8);
19237 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19238 return true;
19239
19240 case Intrinsic::arm_ldaexd:
19241 case Intrinsic::arm_ldrexd:
19242 Info.opc = ISD::INTRINSIC_W_CHAIN;
19243 Info.memVT = MVT::i64;
19244 Info.ptrVal = I.getArgOperand(0);
19245 Info.offset = 0;
19246 Info.align = Align(8);
19247 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19248 return true;
19249
19250 default:
19251 break;
19252 }
19253
19254 return false;
19255}
19256
19257/// Returns true if it is beneficial to convert a load of a constant
19258/// to just the constant itself.
19259bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
19260 Type *Ty) const {
19261 assert(Ty->isIntegerTy())(static_cast <bool> (Ty->isIntegerTy()) ? void (0) :
__assert_fail ("Ty->isIntegerTy()", "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19261, __extension__ __PRETTY_FUNCTION__))
;
19262
19263 unsigned Bits = Ty->getPrimitiveSizeInBits();
19264 if (Bits == 0 || Bits > 32)
19265 return false;
19266 return true;
19267}
19268
19269bool ARMTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
19270 unsigned Index) const {
19271 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
19272 return false;
19273
19274 return (Index == 0 || Index == ResVT.getVectorNumElements());
19275}
19276
19277Instruction* ARMTargetLowering::makeDMB(IRBuilder<> &Builder,
19278 ARM_MB::MemBOpt Domain) const {
19279 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19280
19281 // First, if the target has no DMB, see what fallback we can use.
19282 if (!Subtarget->hasDataBarrier()) {
19283 // Some ARMv6 cpus can support data barriers with an mcr instruction.
19284 // Thumb1 and pre-v6 ARM mode use a libcall instead and should never get
19285 // here.
19286 if (Subtarget->hasV6Ops() && !Subtarget->isThumb()) {
19287 Function *MCR = Intrinsic::getDeclaration(M, Intrinsic::arm_mcr);
19288 Value* args[6] = {Builder.getInt32(15), Builder.getInt32(0),
19289 Builder.getInt32(0), Builder.getInt32(7),
19290 Builder.getInt32(10), Builder.getInt32(5)};
19291 return Builder.CreateCall(MCR, args);
19292 } else {
19293 // Instead of using barriers, atomic accesses on these subtargets use
19294 // libcalls.
19295 llvm_unreachable("makeDMB on a target so old that it has no barriers")::llvm::llvm_unreachable_internal("makeDMB on a target so old that it has no barriers"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19295)
;
19296 }
19297 } else {
19298 Function *DMB = Intrinsic::getDeclaration(M, Intrinsic::arm_dmb);
19299 // Only a full system barrier exists in the M-class architectures.
19300 Domain = Subtarget->isMClass() ? ARM_MB::SY : Domain;
19301 Constant *CDomain = Builder.getInt32(Domain);
19302 return Builder.CreateCall(DMB, CDomain);
19303 }
19304}
19305
19306// Based on http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
19307Instruction *ARMTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
19308 Instruction *Inst,
19309 AtomicOrdering Ord) const {
19310 switch (Ord) {
19311 case AtomicOrdering::NotAtomic:
19312 case AtomicOrdering::Unordered:
19313 llvm_unreachable("Invalid fence: unordered/non-atomic")::llvm::llvm_unreachable_internal("Invalid fence: unordered/non-atomic"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19313)
;
19314 case AtomicOrdering::Monotonic:
19315 case AtomicOrdering::Acquire:
19316 return nullptr; // Nothing to do
19317 case AtomicOrdering::SequentiallyConsistent:
19318 if (!Inst->hasAtomicStore())
19319 return nullptr; // Nothing to do
19320 LLVM_FALLTHROUGH[[gnu::fallthrough]];
19321 case AtomicOrdering::Release:
19322 case AtomicOrdering::AcquireRelease:
19323 if (Subtarget->preferISHSTBarriers())
19324 return makeDMB(Builder, ARM_MB::ISHST);
19325 // FIXME: add a comment with a link to documentation justifying this.
19326 else
19327 return makeDMB(Builder, ARM_MB::ISH);
19328 }
19329 llvm_unreachable("Unknown fence ordering in emitLeadingFence")::llvm::llvm_unreachable_internal("Unknown fence ordering in emitLeadingFence"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19329)
;
19330}
19331
19332Instruction *ARMTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
19333 Instruction *Inst,
19334 AtomicOrdering Ord) const {
19335 switch (Ord) {
19336 case AtomicOrdering::NotAtomic:
19337 case AtomicOrdering::Unordered:
19338 llvm_unreachable("Invalid fence: unordered/not-atomic")::llvm::llvm_unreachable_internal("Invalid fence: unordered/not-atomic"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19338)
;
19339 case AtomicOrdering::Monotonic:
19340 case AtomicOrdering::Release:
19341 return nullptr; // Nothing to do
19342 case AtomicOrdering::Acquire:
19343 case AtomicOrdering::AcquireRelease:
19344 case AtomicOrdering::SequentiallyConsistent:
19345 return makeDMB(Builder, ARM_MB::ISH);
19346 }
19347 llvm_unreachable("Unknown fence ordering in emitTrailingFence")::llvm::llvm_unreachable_internal("Unknown fence ordering in emitTrailingFence"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19347)
;
19348}
19349
19350// Loads and stores less than 64-bits are already atomic; ones above that
19351// are doomed anyway, so defer to the default libcall and blame the OS when
19352// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
19353// anything for those.
19354bool ARMTargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
19355 unsigned Size = SI->getValueOperand()->getType()->getPrimitiveSizeInBits();
19356 return (Size == 64) && !Subtarget->isMClass();
19357}
19358
19359// Loads and stores less than 64-bits are already atomic; ones above that
19360// are doomed anyway, so defer to the default libcall and blame the OS when
19361// things go wrong. Cortex M doesn't have ldrexd/strexd though, so don't emit
19362// anything for those.
19363// FIXME: ldrd and strd are atomic if the CPU has LPAE (e.g. A15 has that
19364// guarantee, see DDI0406C ARM architecture reference manual,
19365// sections A8.8.72-74 LDRD)
19366TargetLowering::AtomicExpansionKind
19367ARMTargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
19368 unsigned Size = LI->getType()->getPrimitiveSizeInBits();
19369 return ((Size == 64) && !Subtarget->isMClass()) ? AtomicExpansionKind::LLOnly
19370 : AtomicExpansionKind::None;
19371}
19372
19373// For the real atomic operations, we have ldrex/strex up to 32 bits,
19374// and up to 64 bits on the non-M profiles
19375TargetLowering::AtomicExpansionKind
19376ARMTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
19377 if (AI->isFloatingPointOperation())
19378 return AtomicExpansionKind::CmpXChg;
19379
19380 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
19381 // implement atomicrmw without spilling. If the target address is also on the
19382 // stack and close enough to the spill slot, this can lead to a situation
19383 // where the monitor always gets cleared and the atomic operation can never
19384 // succeed. So at -O0 lower this operation to a CAS loop.
19385 if (getTargetMachine().getOptLevel() == CodeGenOpt::None)
19386 return AtomicExpansionKind::CmpXChg;
19387
19388 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
19389 bool hasAtomicRMW = !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps();
19390 return (Size <= (Subtarget->isMClass() ? 32U : 64U) && hasAtomicRMW)
19391 ? AtomicExpansionKind::LLSC
19392 : AtomicExpansionKind::None;
19393}
19394
19395// Similar to shouldExpandAtomicRMWInIR, ldrex/strex can be used up to 32
19396// bits, and up to 64 bits on the non-M profiles.
19397TargetLowering::AtomicExpansionKind
19398ARMTargetLowering::shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
19399 // At -O0, fast-regalloc cannot cope with the live vregs necessary to
19400 // implement cmpxchg without spilling. If the address being exchanged is also
19401 // on the stack and close enough to the spill slot, this can lead to a
19402 // situation where the monitor always gets cleared and the atomic operation
19403 // can never succeed. So at -O0 we need a late-expanded pseudo-inst instead.
19404 unsigned Size = AI->getOperand(1)->getType()->getPrimitiveSizeInBits();
19405 bool HasAtomicCmpXchg =
19406 !Subtarget->isThumb() || Subtarget->hasV8MBaselineOps();
19407 if (getTargetMachine().getOptLevel() != 0 && HasAtomicCmpXchg &&
19408 Size <= (Subtarget->isMClass() ? 32U : 64U))
19409 return AtomicExpansionKind::LLSC;
19410 return AtomicExpansionKind::None;
19411}
19412
19413bool ARMTargetLowering::shouldInsertFencesForAtomic(
19414 const Instruction *I) const {
19415 return InsertFencesForAtomic;
19416}
19417
19418// This has so far only been implemented for MachO.
19419bool ARMTargetLowering::useLoadStackGuardNode() const {
19420 return Subtarget->isTargetMachO();
19421}
19422
19423void ARMTargetLowering::insertSSPDeclarations(Module &M) const {
19424 if (!Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
19425 return TargetLowering::insertSSPDeclarations(M);
19426
19427 // MSVC CRT has a global variable holding security cookie.
19428 M.getOrInsertGlobal("__security_cookie",
19429 Type::getInt8PtrTy(M.getContext()));
19430
19431 // MSVC CRT has a function to validate security cookie.
19432 FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
19433 "__security_check_cookie", Type::getVoidTy(M.getContext()),
19434 Type::getInt8PtrTy(M.getContext()));
19435 if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee()))
19436 F->addAttribute(1, Attribute::AttrKind::InReg);
19437}
19438
19439Value *ARMTargetLowering::getSDagStackGuard(const Module &M) const {
19440 // MSVC CRT has a global variable holding security cookie.
19441 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
19442 return M.getGlobalVariable("__security_cookie");
19443 return TargetLowering::getSDagStackGuard(M);
19444}
19445
19446Function *ARMTargetLowering::getSSPStackGuardCheck(const Module &M) const {
19447 // MSVC CRT has a function to validate security cookie.
19448 if (Subtarget->getTargetTriple().isWindowsMSVCEnvironment())
19449 return M.getFunction("__security_check_cookie");
19450 return TargetLowering::getSSPStackGuardCheck(M);
19451}
19452
19453bool ARMTargetLowering::canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
19454 unsigned &Cost) const {
19455 // If we do not have NEON, vector types are not natively supported.
19456 if (!Subtarget->hasNEON())
19457 return false;
19458
19459 // Floating point values and vector values map to the same register file.
19460 // Therefore, although we could do a store extract of a vector type, this is
19461 // better to leave at float as we have more freedom in the addressing mode for
19462 // those.
19463 if (VectorTy->isFPOrFPVectorTy())
19464 return false;
19465
19466 // If the index is unknown at compile time, this is very expensive to lower
19467 // and it is not possible to combine the store with the extract.
19468 if (!isa<ConstantInt>(Idx))
19469 return false;
19470
19471 assert(VectorTy->isVectorTy() && "VectorTy is not a vector type")(static_cast <bool> (VectorTy->isVectorTy() &&
"VectorTy is not a vector type") ? void (0) : __assert_fail (
"VectorTy->isVectorTy() && \"VectorTy is not a vector type\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19471, __extension__ __PRETTY_FUNCTION__))
;
19472 unsigned BitWidth = VectorTy->getPrimitiveSizeInBits().getFixedSize();
19473 // We can do a store + vector extract on any vector that fits perfectly in a D
19474 // or Q register.
19475 if (BitWidth == 64 || BitWidth == 128) {
19476 Cost = 0;
19477 return true;
19478 }
19479 return false;
19480}
19481
19482bool ARMTargetLowering::isCheapToSpeculateCttz() const {
19483 return Subtarget->hasV6T2Ops();
19484}
19485
19486bool ARMTargetLowering::isCheapToSpeculateCtlz() const {
19487 return Subtarget->hasV6T2Ops();
19488}
19489
19490bool ARMTargetLowering::shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
19491 return !Subtarget->hasMinSize() || Subtarget->isTargetWindows();
19492}
19493
19494Value *ARMTargetLowering::emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
19495 AtomicOrdering Ord) const {
19496 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19497 Type *ValTy = cast<PointerType>(Addr->getType())->getElementType();
19498 bool IsAcquire = isAcquireOrStronger(Ord);
19499
19500 // Since i64 isn't legal and intrinsics don't get type-lowered, the ldrexd
19501 // intrinsic must return {i32, i32} and we have to recombine them into a
19502 // single i64 here.
19503 if (ValTy->getPrimitiveSizeInBits() == 64) {
19504 Intrinsic::ID Int =
19505 IsAcquire ? Intrinsic::arm_ldaexd : Intrinsic::arm_ldrexd;
19506 Function *Ldrex = Intrinsic::getDeclaration(M, Int);
19507
19508 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
19509 Value *LoHi = Builder.CreateCall(Ldrex, Addr, "lohi");
19510
19511 Value *Lo = Builder.CreateExtractValue(LoHi, 0, "lo");
19512 Value *Hi = Builder.CreateExtractValue(LoHi, 1, "hi");
19513 if (!Subtarget->isLittle())
19514 std::swap (Lo, Hi);
19515 Lo = Builder.CreateZExt(Lo, ValTy, "lo64");
19516 Hi = Builder.CreateZExt(Hi, ValTy, "hi64");
19517 return Builder.CreateOr(
19518 Lo, Builder.CreateShl(Hi, ConstantInt::get(ValTy, 32)), "val64");
19519 }
19520
19521 Type *Tys[] = { Addr->getType() };
19522 Intrinsic::ID Int = IsAcquire ? Intrinsic::arm_ldaex : Intrinsic::arm_ldrex;
19523 Function *Ldrex = Intrinsic::getDeclaration(M, Int, Tys);
19524
19525 return Builder.CreateTruncOrBitCast(
19526 Builder.CreateCall(Ldrex, Addr),
19527 cast<PointerType>(Addr->getType())->getElementType());
19528}
19529
19530void ARMTargetLowering::emitAtomicCmpXchgNoStoreLLBalance(
19531 IRBuilder<> &Builder) const {
19532 if (!Subtarget->hasV7Ops())
19533 return;
19534 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19535 Builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::arm_clrex));
19536}
19537
19538Value *ARMTargetLowering::emitStoreConditional(IRBuilder<> &Builder, Value *Val,
19539 Value *Addr,
19540 AtomicOrdering Ord) const {
19541 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
19542 bool IsRelease = isReleaseOrStronger(Ord);
19543
19544 // Since the intrinsics must have legal type, the i64 intrinsics take two
19545 // parameters: "i32, i32". We must marshal Val into the appropriate form
19546 // before the call.
19547 if (Val->getType()->getPrimitiveSizeInBits() == 64) {
19548 Intrinsic::ID Int =
19549 IsRelease ? Intrinsic::arm_stlexd : Intrinsic::arm_strexd;
19550 Function *Strex = Intrinsic::getDeclaration(M, Int);
19551 Type *Int32Ty = Type::getInt32Ty(M->getContext());
19552
19553 Value *Lo = Builder.CreateTrunc(Val, Int32Ty, "lo");
19554 Value *Hi = Builder.CreateTrunc(Builder.CreateLShr(Val, 32), Int32Ty, "hi");
19555 if (!Subtarget->isLittle())
19556 std::swap(Lo, Hi);
19557 Addr = Builder.CreateBitCast(Addr, Type::getInt8PtrTy(M->getContext()));
19558 return Builder.CreateCall(Strex, {Lo, Hi, Addr});
19559 }
19560
19561 Intrinsic::ID Int = IsRelease ? Intrinsic::arm_stlex : Intrinsic::arm_strex;
19562 Type *Tys[] = { Addr->getType() };
19563 Function *Strex = Intrinsic::getDeclaration(M, Int, Tys);
19564
19565 return Builder.CreateCall(
19566 Strex, {Builder.CreateZExtOrBitCast(
19567 Val, Strex->getFunctionType()->getParamType(0)),
19568 Addr});
19569}
19570
19571
19572bool ARMTargetLowering::alignLoopsWithOptSize() const {
19573 return Subtarget->isMClass();
19574}
19575
19576/// A helper function for determining the number of interleaved accesses we
19577/// will generate when lowering accesses of the given type.
19578unsigned
19579ARMTargetLowering::getNumInterleavedAccesses(VectorType *VecTy,
19580 const DataLayout &DL) const {
19581 return (DL.getTypeSizeInBits(VecTy) + 127) / 128;
19582}
19583
19584bool ARMTargetLowering::isLegalInterleavedAccessType(
19585 unsigned Factor, FixedVectorType *VecTy, Align Alignment,
19586 const DataLayout &DL) const {
19587
19588 unsigned VecSize = DL.getTypeSizeInBits(VecTy);
19589 unsigned ElSize = DL.getTypeSizeInBits(VecTy->getElementType());
19590
19591 if (!Subtarget->hasNEON() && !Subtarget->hasMVEIntegerOps())
19592 return false;
19593
19594 // Ensure the vector doesn't have f16 elements. Even though we could do an
19595 // i16 vldN, we can't hold the f16 vectors and will end up converting via
19596 // f32.
19597 if (Subtarget->hasNEON() && VecTy->getElementType()->isHalfTy())
19598 return false;
19599 if (Subtarget->hasMVEIntegerOps() && Factor == 3)
19600 return false;
19601
19602 // Ensure the number of vector elements is greater than 1.
19603 if (VecTy->getNumElements() < 2)
19604 return false;
19605
19606 // Ensure the element type is legal.
19607 if (ElSize != 8 && ElSize != 16 && ElSize != 32)
19608 return false;
19609 // And the alignment if high enough under MVE.
19610 if (Subtarget->hasMVEIntegerOps() && Alignment < ElSize / 8)
19611 return false;
19612
19613 // Ensure the total vector size is 64 or a multiple of 128. Types larger than
19614 // 128 will be split into multiple interleaved accesses.
19615 if (Subtarget->hasNEON() && VecSize == 64)
19616 return true;
19617 return VecSize % 128 == 0;
19618}
19619
19620unsigned ARMTargetLowering::getMaxSupportedInterleaveFactor() const {
19621 if (Subtarget->hasNEON())
19622 return 4;
19623 if (Subtarget->hasMVEIntegerOps())
19624 return MVEMaxSupportedInterleaveFactor;
19625 return TargetLoweringBase::getMaxSupportedInterleaveFactor();
19626}
19627
19628/// Lower an interleaved load into a vldN intrinsic.
19629///
19630/// E.g. Lower an interleaved load (Factor = 2):
19631/// %wide.vec = load <8 x i32>, <8 x i32>* %ptr, align 4
19632/// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
19633/// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
19634///
19635/// Into:
19636/// %vld2 = { <4 x i32>, <4 x i32> } call llvm.arm.neon.vld2(%ptr, 4)
19637/// %vec0 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 0
19638/// %vec1 = extractelement { <4 x i32>, <4 x i32> } %vld2, i32 1
19639bool ARMTargetLowering::lowerInterleavedLoad(
19640 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
19641 ArrayRef<unsigned> Indices, unsigned Factor) const {
19642 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&(static_cast <bool> (Factor >= 2 && Factor <=
getMaxSupportedInterleaveFactor() && "Invalid interleave factor"
) ? void (0) : __assert_fail ("Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && \"Invalid interleave factor\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19643, __extension__ __PRETTY_FUNCTION__))
19643 "Invalid interleave factor")(static_cast <bool> (Factor >= 2 && Factor <=
getMaxSupportedInterleaveFactor() && "Invalid interleave factor"
) ? void (0) : __assert_fail ("Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && \"Invalid interleave factor\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19643, __extension__ __PRETTY_FUNCTION__))
;
19644 assert(!Shuffles.empty() && "Empty shufflevector input")(static_cast <bool> (!Shuffles.empty() && "Empty shufflevector input"
) ? void (0) : __assert_fail ("!Shuffles.empty() && \"Empty shufflevector input\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19644, __extension__ __PRETTY_FUNCTION__))
;
19645 assert(Shuffles.size() == Indices.size() &&(static_cast <bool> (Shuffles.size() == Indices.size() &&
"Unmatched number of shufflevectors and indices") ? void (0)
: __assert_fail ("Shuffles.size() == Indices.size() && \"Unmatched number of shufflevectors and indices\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19646, __extension__ __PRETTY_FUNCTION__))
19646 "Unmatched number of shufflevectors and indices")(static_cast <bool> (Shuffles.size() == Indices.size() &&
"Unmatched number of shufflevectors and indices") ? void (0)
: __assert_fail ("Shuffles.size() == Indices.size() && \"Unmatched number of shufflevectors and indices\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19646, __extension__ __PRETTY_FUNCTION__))
;
19647
19648 auto *VecTy = cast<FixedVectorType>(Shuffles[0]->getType());
19649 Type *EltTy = VecTy->getElementType();
19650
19651 const DataLayout &DL = LI->getModule()->getDataLayout();
19652 Align Alignment = LI->getAlign();
19653
19654 // Skip if we do not have NEON and skip illegal vector types. We can
19655 // "legalize" wide vector types into multiple interleaved accesses as long as
19656 // the vector types are divisible by 128.
19657 if (!isLegalInterleavedAccessType(Factor, VecTy, Alignment, DL))
19658 return false;
19659
19660 unsigned NumLoads = getNumInterleavedAccesses(VecTy, DL);
19661
19662 // A pointer vector can not be the return type of the ldN intrinsics. Need to
19663 // load integer vectors first and then convert to pointer vectors.
19664 if (EltTy->isPointerTy())
19665 VecTy = FixedVectorType::get(DL.getIntPtrType(EltTy), VecTy);
19666
19667 IRBuilder<> Builder(LI);
19668
19669 // The base address of the load.
19670 Value *BaseAddr = LI->getPointerOperand();
19671
19672 if (NumLoads > 1) {
19673 // If we're going to generate more than one load, reset the sub-vector type
19674 // to something legal.
19675 VecTy = FixedVectorType::get(VecTy->getElementType(),
19676 VecTy->getNumElements() / NumLoads);
19677
19678 // We will compute the pointer operand of each load from the original base
19679 // address using GEPs. Cast the base address to a pointer to the scalar
19680 // element type.
19681 BaseAddr = Builder.CreateBitCast(
19682 BaseAddr,
19683 VecTy->getElementType()->getPointerTo(LI->getPointerAddressSpace()));
19684 }
19685
19686 assert(isTypeLegal(EVT::getEVT(VecTy)) && "Illegal vldN vector type!")(static_cast <bool> (isTypeLegal(EVT::getEVT(VecTy)) &&
"Illegal vldN vector type!") ? void (0) : __assert_fail ("isTypeLegal(EVT::getEVT(VecTy)) && \"Illegal vldN vector type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19686, __extension__ __PRETTY_FUNCTION__))
;
19687
19688 auto createLoadIntrinsic = [&](Value *BaseAddr) {
19689 if (Subtarget->hasNEON()) {
19690 Type *Int8Ptr = Builder.getInt8PtrTy(LI->getPointerAddressSpace());
19691 Type *Tys[] = {VecTy, Int8Ptr};
19692 static const Intrinsic::ID LoadInts[3] = {Intrinsic::arm_neon_vld2,
19693 Intrinsic::arm_neon_vld3,
19694 Intrinsic::arm_neon_vld4};
19695 Function *VldnFunc =
19696 Intrinsic::getDeclaration(LI->getModule(), LoadInts[Factor - 2], Tys);
19697
19698 SmallVector<Value *, 2> Ops;
19699 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
19700 Ops.push_back(Builder.getInt32(LI->getAlignment()));
19701
19702 return Builder.CreateCall(VldnFunc, Ops, "vldN");
19703 } else {
19704 assert((Factor == 2 || Factor == 4) &&(static_cast <bool> ((Factor == 2 || Factor == 4) &&
"expected interleave factor of 2 or 4 for MVE") ? void (0) :
__assert_fail ("(Factor == 2 || Factor == 4) && \"expected interleave factor of 2 or 4 for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19705, __extension__ __PRETTY_FUNCTION__))
19705 "expected interleave factor of 2 or 4 for MVE")(static_cast <bool> ((Factor == 2 || Factor == 4) &&
"expected interleave factor of 2 or 4 for MVE") ? void (0) :
__assert_fail ("(Factor == 2 || Factor == 4) && \"expected interleave factor of 2 or 4 for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19705, __extension__ __PRETTY_FUNCTION__))
;
19706 Intrinsic::ID LoadInts =
19707 Factor == 2 ? Intrinsic::arm_mve_vld2q : Intrinsic::arm_mve_vld4q;
19708 Type *VecEltTy =
19709 VecTy->getElementType()->getPointerTo(LI->getPointerAddressSpace());
19710 Type *Tys[] = {VecTy, VecEltTy};
19711 Function *VldnFunc =
19712 Intrinsic::getDeclaration(LI->getModule(), LoadInts, Tys);
19713
19714 SmallVector<Value *, 2> Ops;
19715 Ops.push_back(Builder.CreateBitCast(BaseAddr, VecEltTy));
19716 return Builder.CreateCall(VldnFunc, Ops, "vldN");
19717 }
19718 };
19719
19720 // Holds sub-vectors extracted from the load intrinsic return values. The
19721 // sub-vectors are associated with the shufflevector instructions they will
19722 // replace.
19723 DenseMap<ShuffleVectorInst *, SmallVector<Value *, 4>> SubVecs;
19724
19725 for (unsigned LoadCount = 0; LoadCount < NumLoads; ++LoadCount) {
19726 // If we're generating more than one load, compute the base address of
19727 // subsequent loads as an offset from the previous.
19728 if (LoadCount > 0)
19729 BaseAddr = Builder.CreateConstGEP1_32(VecTy->getElementType(), BaseAddr,
19730 VecTy->getNumElements() * Factor);
19731
19732 CallInst *VldN = createLoadIntrinsic(BaseAddr);
19733
19734 // Replace uses of each shufflevector with the corresponding vector loaded
19735 // by ldN.
19736 for (unsigned i = 0; i < Shuffles.size(); i++) {
19737 ShuffleVectorInst *SV = Shuffles[i];
19738 unsigned Index = Indices[i];
19739
19740 Value *SubVec = Builder.CreateExtractValue(VldN, Index);
19741
19742 // Convert the integer vector to pointer vector if the element is pointer.
19743 if (EltTy->isPointerTy())
19744 SubVec = Builder.CreateIntToPtr(
19745 SubVec,
19746 FixedVectorType::get(SV->getType()->getElementType(), VecTy));
19747
19748 SubVecs[SV].push_back(SubVec);
19749 }
19750 }
19751
19752 // Replace uses of the shufflevector instructions with the sub-vectors
19753 // returned by the load intrinsic. If a shufflevector instruction is
19754 // associated with more than one sub-vector, those sub-vectors will be
19755 // concatenated into a single wide vector.
19756 for (ShuffleVectorInst *SVI : Shuffles) {
19757 auto &SubVec = SubVecs[SVI];
19758 auto *WideVec =
19759 SubVec.size() > 1 ? concatenateVectors(Builder, SubVec) : SubVec[0];
19760 SVI->replaceAllUsesWith(WideVec);
19761 }
19762
19763 return true;
19764}
19765
19766/// Lower an interleaved store into a vstN intrinsic.
19767///
19768/// E.g. Lower an interleaved store (Factor = 3):
19769/// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
19770/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
19771/// store <12 x i32> %i.vec, <12 x i32>* %ptr, align 4
19772///
19773/// Into:
19774/// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
19775/// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
19776/// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
19777/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
19778///
19779/// Note that the new shufflevectors will be removed and we'll only generate one
19780/// vst3 instruction in CodeGen.
19781///
19782/// Example for a more general valid mask (Factor 3). Lower:
19783/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1,
19784/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
19785/// store <12 x i32> %i.vec, <12 x i32>* %ptr
19786///
19787/// Into:
19788/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7>
19789/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35>
19790/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19>
19791/// call void llvm.arm.neon.vst3(%ptr, %sub.v0, %sub.v1, %sub.v2, 4)
19792bool ARMTargetLowering::lowerInterleavedStore(StoreInst *SI,
19793 ShuffleVectorInst *SVI,
19794 unsigned Factor) const {
19795 assert(Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() &&(static_cast <bool> (Factor >= 2 && Factor <=
getMaxSupportedInterleaveFactor() && "Invalid interleave factor"
) ? void (0) : __assert_fail ("Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && \"Invalid interleave factor\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19796, __extension__ __PRETTY_FUNCTION__))
19796 "Invalid interleave factor")(static_cast <bool> (Factor >= 2 && Factor <=
getMaxSupportedInterleaveFactor() && "Invalid interleave factor"
) ? void (0) : __assert_fail ("Factor >= 2 && Factor <= getMaxSupportedInterleaveFactor() && \"Invalid interleave factor\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19796, __extension__ __PRETTY_FUNCTION__))
;
19797
19798 auto *VecTy = cast<FixedVectorType>(SVI->getType());
19799 assert(VecTy->getNumElements() % Factor == 0 && "Invalid interleaved store")(static_cast <bool> (VecTy->getNumElements() % Factor
== 0 && "Invalid interleaved store") ? void (0) : __assert_fail
("VecTy->getNumElements() % Factor == 0 && \"Invalid interleaved store\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19799, __extension__ __PRETTY_FUNCTION__))
;
19800
19801 unsigned LaneLen = VecTy->getNumElements() / Factor;
19802 Type *EltTy = VecTy->getElementType();
19803 auto *SubVecTy = FixedVectorType::get(EltTy, LaneLen);
19804
19805 const DataLayout &DL = SI->getModule()->getDataLayout();
19806 Align Alignment = SI->getAlign();
19807
19808 // Skip if we do not have NEON and skip illegal vector types. We can
19809 // "legalize" wide vector types into multiple interleaved accesses as long as
19810 // the vector types are divisible by 128.
19811 if (!isLegalInterleavedAccessType(Factor, SubVecTy, Alignment, DL))
19812 return false;
19813
19814 unsigned NumStores = getNumInterleavedAccesses(SubVecTy, DL);
19815
19816 Value *Op0 = SVI->getOperand(0);
19817 Value *Op1 = SVI->getOperand(1);
19818 IRBuilder<> Builder(SI);
19819
19820 // StN intrinsics don't support pointer vectors as arguments. Convert pointer
19821 // vectors to integer vectors.
19822 if (EltTy->isPointerTy()) {
19823 Type *IntTy = DL.getIntPtrType(EltTy);
19824
19825 // Convert to the corresponding integer vector.
19826 auto *IntVecTy =
19827 FixedVectorType::get(IntTy, cast<FixedVectorType>(Op0->getType()));
19828 Op0 = Builder.CreatePtrToInt(Op0, IntVecTy);
19829 Op1 = Builder.CreatePtrToInt(Op1, IntVecTy);
19830
19831 SubVecTy = FixedVectorType::get(IntTy, LaneLen);
19832 }
19833
19834 // The base address of the store.
19835 Value *BaseAddr = SI->getPointerOperand();
19836
19837 if (NumStores > 1) {
19838 // If we're going to generate more than one store, reset the lane length
19839 // and sub-vector type to something legal.
19840 LaneLen /= NumStores;
19841 SubVecTy = FixedVectorType::get(SubVecTy->getElementType(), LaneLen);
19842
19843 // We will compute the pointer operand of each store from the original base
19844 // address using GEPs. Cast the base address to a pointer to the scalar
19845 // element type.
19846 BaseAddr = Builder.CreateBitCast(
19847 BaseAddr,
19848 SubVecTy->getElementType()->getPointerTo(SI->getPointerAddressSpace()));
19849 }
19850
19851 assert(isTypeLegal(EVT::getEVT(SubVecTy)) && "Illegal vstN vector type!")(static_cast <bool> (isTypeLegal(EVT::getEVT(SubVecTy))
&& "Illegal vstN vector type!") ? void (0) : __assert_fail
("isTypeLegal(EVT::getEVT(SubVecTy)) && \"Illegal vstN vector type!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19851, __extension__ __PRETTY_FUNCTION__))
;
19852
19853 auto Mask = SVI->getShuffleMask();
19854
19855 auto createStoreIntrinsic = [&](Value *BaseAddr,
19856 SmallVectorImpl<Value *> &Shuffles) {
19857 if (Subtarget->hasNEON()) {
19858 static const Intrinsic::ID StoreInts[3] = {Intrinsic::arm_neon_vst2,
19859 Intrinsic::arm_neon_vst3,
19860 Intrinsic::arm_neon_vst4};
19861 Type *Int8Ptr = Builder.getInt8PtrTy(SI->getPointerAddressSpace());
19862 Type *Tys[] = {Int8Ptr, SubVecTy};
19863
19864 Function *VstNFunc = Intrinsic::getDeclaration(
19865 SI->getModule(), StoreInts[Factor - 2], Tys);
19866
19867 SmallVector<Value *, 6> Ops;
19868 Ops.push_back(Builder.CreateBitCast(BaseAddr, Int8Ptr));
19869 append_range(Ops, Shuffles);
19870 Ops.push_back(Builder.getInt32(SI->getAlignment()));
19871 Builder.CreateCall(VstNFunc, Ops);
19872 } else {
19873 assert((Factor == 2 || Factor == 4) &&(static_cast <bool> ((Factor == 2 || Factor == 4) &&
"expected interleave factor of 2 or 4 for MVE") ? void (0) :
__assert_fail ("(Factor == 2 || Factor == 4) && \"expected interleave factor of 2 or 4 for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19874, __extension__ __PRETTY_FUNCTION__))
19874 "expected interleave factor of 2 or 4 for MVE")(static_cast <bool> ((Factor == 2 || Factor == 4) &&
"expected interleave factor of 2 or 4 for MVE") ? void (0) :
__assert_fail ("(Factor == 2 || Factor == 4) && \"expected interleave factor of 2 or 4 for MVE\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 19874, __extension__ __PRETTY_FUNCTION__))
;
19875 Intrinsic::ID StoreInts =
19876 Factor == 2 ? Intrinsic::arm_mve_vst2q : Intrinsic::arm_mve_vst4q;
19877 Type *EltPtrTy = SubVecTy->getElementType()->getPointerTo(
19878 SI->getPointerAddressSpace());
19879 Type *Tys[] = {EltPtrTy, SubVecTy};
19880 Function *VstNFunc =
19881 Intrinsic::getDeclaration(SI->getModule(), StoreInts, Tys);
19882
19883 SmallVector<Value *, 6> Ops;
19884 Ops.push_back(Builder.CreateBitCast(BaseAddr, EltPtrTy));
19885 append_range(Ops, Shuffles);
19886 for (unsigned F = 0; F < Factor; F++) {
19887 Ops.push_back(Builder.getInt32(F));
19888 Builder.CreateCall(VstNFunc, Ops);
19889 Ops.pop_back();
19890 }
19891 }
19892 };
19893
19894 for (unsigned StoreCount = 0; StoreCount < NumStores; ++StoreCount) {
19895 // If we generating more than one store, we compute the base address of
19896 // subsequent stores as an offset from the previous.
19897 if (StoreCount > 0)
19898 BaseAddr = Builder.CreateConstGEP1_32(SubVecTy->getElementType(),
19899 BaseAddr, LaneLen * Factor);
19900
19901 SmallVector<Value *, 4> Shuffles;
19902
19903 // Split the shufflevector operands into sub vectors for the new vstN call.
19904 for (unsigned i = 0; i < Factor; i++) {
19905 unsigned IdxI = StoreCount * LaneLen * Factor + i;
19906 if (Mask[IdxI] >= 0) {
19907 Shuffles.push_back(Builder.CreateShuffleVector(
19908 Op0, Op1, createSequentialMask(Mask[IdxI], LaneLen, 0)));
19909 } else {
19910 unsigned StartMask = 0;
19911 for (unsigned j = 1; j < LaneLen; j++) {
19912 unsigned IdxJ = StoreCount * LaneLen * Factor + j;
19913 if (Mask[IdxJ * Factor + IdxI] >= 0) {
19914 StartMask = Mask[IdxJ * Factor + IdxI] - IdxJ;
19915 break;
19916 }
19917 }
19918 // Note: If all elements in a chunk are undefs, StartMask=0!
19919 // Note: Filling undef gaps with random elements is ok, since
19920 // those elements were being written anyway (with undefs).
19921 // In the case of all undefs we're defaulting to using elems from 0
19922 // Note: StartMask cannot be negative, it's checked in
19923 // isReInterleaveMask
19924 Shuffles.push_back(Builder.CreateShuffleVector(
19925 Op0, Op1, createSequentialMask(StartMask, LaneLen, 0)));
19926 }
19927 }
19928
19929 createStoreIntrinsic(BaseAddr, Shuffles);
19930 }
19931 return true;
19932}
19933
19934enum HABaseType {
19935 HA_UNKNOWN = 0,
19936 HA_FLOAT,
19937 HA_DOUBLE,
19938 HA_VECT64,
19939 HA_VECT128
19940};
19941
19942static bool isHomogeneousAggregate(Type *Ty, HABaseType &Base,
19943 uint64_t &Members) {
19944 if (auto *ST = dyn_cast<StructType>(Ty)) {
19945 for (unsigned i = 0; i < ST->getNumElements(); ++i) {
19946 uint64_t SubMembers = 0;
19947 if (!isHomogeneousAggregate(ST->getElementType(i), Base, SubMembers))
19948 return false;
19949 Members += SubMembers;
19950 }
19951 } else if (auto *AT = dyn_cast<ArrayType>(Ty)) {
19952 uint64_t SubMembers = 0;
19953 if (!isHomogeneousAggregate(AT->getElementType(), Base, SubMembers))
19954 return false;
19955 Members += SubMembers * AT->getNumElements();
19956 } else if (Ty->isFloatTy()) {
19957 if (Base != HA_UNKNOWN && Base != HA_FLOAT)
19958 return false;
19959 Members = 1;
19960 Base = HA_FLOAT;
19961 } else if (Ty->isDoubleTy()) {
19962 if (Base != HA_UNKNOWN && Base != HA_DOUBLE)
19963 return false;
19964 Members = 1;
19965 Base = HA_DOUBLE;
19966 } else if (auto *VT = dyn_cast<VectorType>(Ty)) {
19967 Members = 1;
19968 switch (Base) {
19969 case HA_FLOAT:
19970 case HA_DOUBLE:
19971 return false;
19972 case HA_VECT64:
19973 return VT->getPrimitiveSizeInBits().getFixedSize() == 64;
19974 case HA_VECT128:
19975 return VT->getPrimitiveSizeInBits().getFixedSize() == 128;
19976 case HA_UNKNOWN:
19977 switch (VT->getPrimitiveSizeInBits().getFixedSize()) {
19978 case 64:
19979 Base = HA_VECT64;
19980 return true;
19981 case 128:
19982 Base = HA_VECT128;
19983 return true;
19984 default:
19985 return false;
19986 }
19987 }
19988 }
19989
19990 return (Members > 0 && Members <= 4);
19991}
19992
19993/// Return the correct alignment for the current calling convention.
19994Align ARMTargetLowering::getABIAlignmentForCallingConv(Type *ArgTy,
19995 DataLayout DL) const {
19996 const Align ABITypeAlign = DL.getABITypeAlign(ArgTy);
19997 if (!ArgTy->isVectorTy())
19998 return ABITypeAlign;
19999
20000 // Avoid over-aligning vector parameters. It would require realigning the
20001 // stack and waste space for no real benefit.
20002 return std::min(ABITypeAlign, DL.getStackAlignment());
20003}
20004
20005/// Return true if a type is an AAPCS-VFP homogeneous aggregate or one of
20006/// [N x i32] or [N x i64]. This allows front-ends to skip emitting padding when
20007/// passing according to AAPCS rules.
20008bool ARMTargetLowering::functionArgumentNeedsConsecutiveRegisters(
20009 Type *Ty, CallingConv::ID CallConv, bool isVarArg) const {
20010 if (getEffectiveCallingConv(CallConv, isVarArg) !=
20011 CallingConv::ARM_AAPCS_VFP)
20012 return false;
20013
20014 HABaseType Base = HA_UNKNOWN;
20015 uint64_t Members = 0;
20016 bool IsHA = isHomogeneousAggregate(Ty, Base, Members);
20017 LLVM_DEBUG(dbgs() << "isHA: " << IsHA << " "; Ty->dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("arm-isel")) { dbgs() << "isHA: " << IsHA <<
" "; Ty->dump(); } } while (false)
;
20018
20019 bool IsIntArray = Ty->isArrayTy() && Ty->getArrayElementType()->isIntegerTy();
20020 return IsHA || IsIntArray;
20021}
20022
20023Register ARMTargetLowering::getExceptionPointerRegister(
20024 const Constant *PersonalityFn) const {
20025 // Platforms which do not use SjLj EH may return values in these registers
20026 // via the personality function.
20027 return Subtarget->useSjLjEH() ? Register() : ARM::R0;
20028}
20029
20030Register ARMTargetLowering::getExceptionSelectorRegister(
20031 const Constant *PersonalityFn) const {
20032 // Platforms which do not use SjLj EH may return values in these registers
20033 // via the personality function.
20034 return Subtarget->useSjLjEH() ? Register() : ARM::R1;
20035}
20036
20037void ARMTargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
20038 // Update IsSplitCSR in ARMFunctionInfo.
20039 ARMFunctionInfo *AFI = Entry->getParent()->getInfo<ARMFunctionInfo>();
20040 AFI->setIsSplitCSR(true);
20041}
20042
20043void ARMTargetLowering::insertCopiesSplitCSR(
20044 MachineBasicBlock *Entry,
20045 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
20046 const ARMBaseRegisterInfo *TRI = Subtarget->getRegisterInfo();
20047 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
20048 if (!IStart)
20049 return;
20050
20051 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
20052 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
20053 MachineBasicBlock::iterator MBBI = Entry->begin();
20054 for (const MCPhysReg *I = IStart; *I; ++I) {
20055 const TargetRegisterClass *RC = nullptr;
20056 if (ARM::GPRRegClass.contains(*I))
20057 RC = &ARM::GPRRegClass;
20058 else if (ARM::DPRRegClass.contains(*I))
20059 RC = &ARM::DPRRegClass;
20060 else
20061 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 20061)
;
20062
20063 Register NewVR = MRI->createVirtualRegister(RC);
20064 // Create copy from CSR to a virtual register.
20065 // FIXME: this currently does not emit CFI pseudo-instructions, it works
20066 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
20067 // nounwind. If we want to generalize this later, we may need to emit
20068 // CFI pseudo-instructions.
20069 assert(Entry->getParent()->getFunction().hasFnAttribute((static_cast <bool> (Entry->getParent()->getFunction
().hasFnAttribute( Attribute::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? void (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute( Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 20071, __extension__ __PRETTY_FUNCTION__))
20070 Attribute::NoUnwind) &&(static_cast <bool> (Entry->getParent()->getFunction
().hasFnAttribute( Attribute::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? void (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute( Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 20071, __extension__ __PRETTY_FUNCTION__))
20071 "Function should be nounwind in insertCopiesSplitCSR!")(static_cast <bool> (Entry->getParent()->getFunction
().hasFnAttribute( Attribute::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? void (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute( Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-13~++20210528111147+00ecf18979e3/llvm/lib/Target/ARM/ARMISelLowering.cpp"
, 20071, __extension__ __PRETTY_FUNCTION__))
;
20072 Entry->addLiveIn(*I);
20073 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
20074 .addReg(*I);
20075
20076 // Insert the copy-back instructions right before the terminator.
20077 for (auto *Exit : Exits)
20078 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
20079 TII->get(TargetOpcode::COPY), *I)
20080 .addReg(NewVR);
20081 }
20082}
20083
20084void ARMTargetLowering::finalizeLowering(MachineFunction &MF) const {
20085 MF.getFrameInfo().computeMaxCallFrameSize(MF);
20086 TargetLoweringBase::finalizeLowering(MF);
20087}