Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1153, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64ISelDAGToDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-11/lib/clang/11.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/include -I /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-11/lib/clang/11.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/build-llvm/lib/Target/AArch64 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347=. -ferror-limit 19 -fmessage-length 0 -fvisibility hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-03-09-184146-41876-1 -x c++ /build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64TargetMachine.h"
14#include "MCTargetDesc/AArch64AddressingModes.h"
15#include "llvm/ADT/APSInt.h"
16#include "llvm/CodeGen/SelectionDAGISel.h"
17#include "llvm/IR/Function.h" // To access function attributes.
18#include "llvm/IR/GlobalValue.h"
19#include "llvm/IR/Intrinsics.h"
20#include "llvm/IR/IntrinsicsAArch64.h"
21#include "llvm/Support/Debug.h"
22#include "llvm/Support/ErrorHandling.h"
23#include "llvm/Support/KnownBits.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"
30
31//===--------------------------------------------------------------------===//
32/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
33/// instructions for SelectionDAG operations.
34///
35namespace {
36
37class AArch64DAGToDAGISel : public SelectionDAGISel {
38
39 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
40 /// make the right decision when generating code for different targets.
41 const AArch64Subtarget *Subtarget;
42
43public:
44 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
45 CodeGenOpt::Level OptLevel)
46 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
47
48 StringRef getPassName() const override {
49 return "AArch64 Instruction Selection";
50 }
51
52 bool runOnMachineFunction(MachineFunction &MF) override {
53 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
54 return SelectionDAGISel::runOnMachineFunction(MF);
55 }
56
57 void Select(SDNode *Node) override;
58
59 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
60 /// inline asm expressions.
61 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
62 unsigned ConstraintID,
63 std::vector<SDValue> &OutOps) override;
64
65 template <signed Low, signed High, signed Scale>
66 bool SelectRDVLImm(SDValue N, SDValue &Imm);
67
68 bool tryMLAV64LaneV128(SDNode *N);
69 bool tryMULLV64LaneV128(unsigned IntNo, SDNode *N);
70 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
71 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
72 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
73 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
74 return SelectShiftedRegister(N, false, Reg, Shift);
75 }
76 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
77 return SelectShiftedRegister(N, true, Reg, Shift);
78 }
79 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
80 return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
81 }
82 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
83 return SelectAddrModeIndexed7S(N, 2, Base, OffImm);
84 }
85 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
86 return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
87 }
88 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
89 return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
90 }
91 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
92 return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
93 }
94 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
95 return SelectAddrModeIndexedBitWidth(N, true, 9, 16, Base, OffImm);
96 }
97 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
98 return SelectAddrModeIndexedBitWidth(N, false, 6, 16, Base, OffImm);
99 }
100 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
101 return SelectAddrModeIndexed(N, 1, Base, OffImm);
102 }
103 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
104 return SelectAddrModeIndexed(N, 2, Base, OffImm);
105 }
106 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
107 return SelectAddrModeIndexed(N, 4, Base, OffImm);
108 }
109 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
110 return SelectAddrModeIndexed(N, 8, Base, OffImm);
111 }
112 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
113 return SelectAddrModeIndexed(N, 16, Base, OffImm);
114 }
115 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
116 return SelectAddrModeUnscaled(N, 1, Base, OffImm);
117 }
118 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
119 return SelectAddrModeUnscaled(N, 2, Base, OffImm);
120 }
121 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
122 return SelectAddrModeUnscaled(N, 4, Base, OffImm);
123 }
124 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
125 return SelectAddrModeUnscaled(N, 8, Base, OffImm);
126 }
127 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
128 return SelectAddrModeUnscaled(N, 16, Base, OffImm);
129 }
130
131 template<int Width>
132 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
133 SDValue &SignExtend, SDValue &DoShift) {
134 return SelectAddrModeWRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
135 }
136
137 template<int Width>
138 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
139 SDValue &SignExtend, SDValue &DoShift) {
140 return SelectAddrModeXRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
141 }
142
143 bool SelectDupZeroOrUndef(SDValue N) {
144 switch(N->getOpcode()) {
145 case ISD::UNDEF:
146 return true;
147 case AArch64ISD::DUP:
148 case ISD::SPLAT_VECTOR: {
149 auto Opnd0 = N->getOperand(0);
150 if (auto CN = dyn_cast<ConstantSDNode>(Opnd0))
151 if (CN->isNullValue())
152 return true;
153 if (auto CN = dyn_cast<ConstantFPSDNode>(Opnd0))
154 if (CN->isZero())
155 return true;
156 break;
157 }
158 default:
159 break;
160 }
161
162 return false;
163 }
164
165 bool SelectDupZero(SDValue N) {
166 switch(N->getOpcode()) {
167 case AArch64ISD::DUP:
168 case ISD::SPLAT_VECTOR: {
169 auto Opnd0 = N->getOperand(0);
170 if (auto CN = dyn_cast<ConstantSDNode>(Opnd0))
171 if (CN->isNullValue())
172 return true;
173 if (auto CN = dyn_cast<ConstantFPSDNode>(Opnd0))
174 if (CN->isZero())
175 return true;
176 break;
177 }
178 }
179
180 return false;
181 }
182
183 template<MVT::SimpleValueType VT>
184 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
185 return SelectSVEAddSubImm(N, VT, Imm, Shift);
186 }
187
188 template<MVT::SimpleValueType VT>
189 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
190 return SelectSVELogicalImm(N, VT, Imm);
191 }
192
193 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
194 template<signed Min, signed Max, signed Scale, bool Shift>
195 bool SelectCntImm(SDValue N, SDValue &Imm) {
196 if (!isa<ConstantSDNode>(N))
197 return false;
198
199 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
200 if (Shift)
201 MulImm = 1LL << MulImm;
202
203 if ((MulImm % std::abs(Scale)) != 0)
204 return false;
205
206 MulImm /= Scale;
207 if ((MulImm >= Min) && (MulImm <= Max)) {
208 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32);
209 return true;
210 }
211
212 return false;
213 }
214
215 /// Form sequences of consecutive 64/128-bit registers for use in NEON
216 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
217 /// between 1 and 4 elements. If it contains a single element that is returned
218 /// unchanged; otherwise a REG_SEQUENCE value is returned.
219 SDValue createDTuple(ArrayRef<SDValue> Vecs);
220 SDValue createQTuple(ArrayRef<SDValue> Vecs);
221
222 /// Generic helper for the createDTuple/createQTuple
223 /// functions. Those should almost always be called instead.
224 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
225 const unsigned SubRegs[]);
226
227 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
228
229 bool tryIndexedLoad(SDNode *N);
230
231 bool trySelectStackSlotTagP(SDNode *N);
232 void SelectTagP(SDNode *N);
233
234 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
235 unsigned SubRegIdx);
236 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
237 unsigned SubRegIdx);
238 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
239 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
240
241 bool SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base, SDValue &OffImm);
242 /// SVE Reg+Imm addressing mode.
243 template <int64_t Min, int64_t Max>
244 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
245 SDValue &OffImm);
246 /// SVE Reg+Reg address mode.
247 template <unsigned Scale>
248 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
249 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
250 }
251
252 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
253 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
254 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
255 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
256
257 bool tryBitfieldExtractOp(SDNode *N);
258 bool tryBitfieldExtractOpFromSExt(SDNode *N);
259 bool tryBitfieldInsertOp(SDNode *N);
260 bool tryBitfieldInsertInZeroOp(SDNode *N);
261 bool tryShiftAmountMod(SDNode *N);
262 bool tryHighFPExt(SDNode *N);
263
264 bool tryReadRegister(SDNode *N);
265 bool tryWriteRegister(SDNode *N);
266
267// Include the pieces autogenerated from the target description.
268#include "AArch64GenDAGISel.inc"
269
270private:
271 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
272 SDValue &Shift);
273 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
274 SDValue &OffImm) {
275 return SelectAddrModeIndexedBitWidth(N, true, 7, Size, Base, OffImm);
276 }
277 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
278 unsigned Size, SDValue &Base,
279 SDValue &OffImm);
280 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
281 SDValue &OffImm);
282 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
283 SDValue &OffImm);
284 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
285 SDValue &Offset, SDValue &SignExtend,
286 SDValue &DoShift);
287 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
288 SDValue &Offset, SDValue &SignExtend,
289 SDValue &DoShift);
290 bool isWorthFolding(SDValue V) const;
291 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
292 SDValue &Offset, SDValue &SignExtend);
293
294 template<unsigned RegWidth>
295 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
296 return SelectCVTFixedPosOperand(N, FixedPos, RegWidth);
297 }
298
299 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
300
301 bool SelectCMP_SWAP(SDNode *N);
302
303 bool SelectSVE8BitLslImm(SDValue N, SDValue &Imm, SDValue &Shift);
304
305 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
306
307 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm);
308
309 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
310
311 bool SelectSVEArithImm(SDValue N, SDValue &Imm);
312 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
313 SDValue &Offset);
314};
315} // end anonymous namespace
316
317/// isIntImmediate - This method tests to see if the node is a constant
318/// operand. If so Imm will receive the 32-bit value.
319static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
320 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(N)) {
321 Imm = C->getZExtValue();
322 return true;
323 }
324 return false;
325}
326
327// isIntImmediate - This method tests to see if a constant operand.
328// If so Imm will receive the value.
329static bool isIntImmediate(SDValue N, uint64_t &Imm) {
330 return isIntImmediate(N.getNode(), Imm);
331}
332
333// isOpcWithIntImmediate - This method tests to see if the node is a specific
334// opcode and that it has a immediate integer right operand.
335// If so Imm will receive the 32 bit value.
336static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
337 uint64_t &Imm) {
338 return N->getOpcode() == Opc &&
339 isIntImmediate(N->getOperand(1).getNode(), Imm);
340}
341
342bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
343 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
344 switch(ConstraintID) {
345 default:
346 llvm_unreachable("Unexpected asm memory constraint")::llvm::llvm_unreachable_internal("Unexpected asm memory constraint"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 346)
;
347 case InlineAsm::Constraint_m:
348 case InlineAsm::Constraint_Q:
349 // We need to make sure that this one operand does not end up in XZR, thus
350 // require the address to be in a PointerRegClass register.
351 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
352 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF);
353 SDLoc dl(Op);
354 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64);
355 SDValue NewOp =
356 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
357 dl, Op.getValueType(),
358 Op, RC), 0);
359 OutOps.push_back(NewOp);
360 return false;
361 }
362 return true;
363}
364
365/// SelectArithImmed - Select an immediate value that can be represented as
366/// a 12-bit value shifted left by either 0 or 12. If so, return true with
367/// Val set to the 12-bit value and Shift set to the shifter operand.
368bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
369 SDValue &Shift) {
370 // This function is called from the addsub_shifted_imm ComplexPattern,
371 // which lists [imm] as the list of opcode it's interested in, however
372 // we still need to check whether the operand is actually an immediate
373 // here because the ComplexPattern opcode list is only used in
374 // root-level opcode matching.
375 if (!isa<ConstantSDNode>(N.getNode()))
376 return false;
377
378 uint64_t Immed = cast<ConstantSDNode>(N.getNode())->getZExtValue();
379 unsigned ShiftAmt;
380
381 if (Immed >> 12 == 0) {
382 ShiftAmt = 0;
383 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
384 ShiftAmt = 12;
385 Immed = Immed >> 12;
386 } else
387 return false;
388
389 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
390 SDLoc dl(N);
391 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
392 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
393 return true;
394}
395
396/// SelectNegArithImmed - As above, but negates the value before trying to
397/// select it.
398bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
399 SDValue &Shift) {
400 // This function is called from the addsub_shifted_imm ComplexPattern,
401 // which lists [imm] as the list of opcode it's interested in, however
402 // we still need to check whether the operand is actually an immediate
403 // here because the ComplexPattern opcode list is only used in
404 // root-level opcode matching.
405 if (!isa<ConstantSDNode>(N.getNode()))
406 return false;
407
408 // The immediate operand must be a 24-bit zero-extended immediate.
409 uint64_t Immed = cast<ConstantSDNode>(N.getNode())->getZExtValue();
410
411 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
412 // have the opposite effect on the C flag, so this pattern mustn't match under
413 // those circumstances.
414 if (Immed == 0)
415 return false;
416
417 if (N.getValueType() == MVT::i32)
418 Immed = ~((uint32_t)Immed) + 1;
419 else
420 Immed = ~Immed + 1ULL;
421 if (Immed & 0xFFFFFFFFFF000000ULL)
422 return false;
423
424 Immed &= 0xFFFFFFULL;
425 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val,
426 Shift);
427}
428
429/// getShiftTypeForNode - Translate a shift node to the corresponding
430/// ShiftType value.
431static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
432 switch (N.getOpcode()) {
433 default:
434 return AArch64_AM::InvalidShiftExtend;
435 case ISD::SHL:
436 return AArch64_AM::LSL;
437 case ISD::SRL:
438 return AArch64_AM::LSR;
439 case ISD::SRA:
440 return AArch64_AM::ASR;
441 case ISD::ROTR:
442 return AArch64_AM::ROR;
443 }
444}
445
446/// Determine whether it is worth it to fold SHL into the addressing
447/// mode.
448static bool isWorthFoldingSHL(SDValue V) {
449 assert(V.getOpcode() == ISD::SHL && "invalid opcode")((V.getOpcode() == ISD::SHL && "invalid opcode") ? static_cast
<void> (0) : __assert_fail ("V.getOpcode() == ISD::SHL && \"invalid opcode\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 449, __PRETTY_FUNCTION__))
;
450 // It is worth folding logical shift of up to three places.
451 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
452 if (!CSD)
453 return false;
454 unsigned ShiftVal = CSD->getZExtValue();
455 if (ShiftVal > 3)
456 return false;
457
458 // Check if this particular node is reused in any non-memory related
459 // operation. If yes, do not try to fold this node into the address
460 // computation, since the computation will be kept.
461 const SDNode *Node = V.getNode();
462 for (SDNode *UI : Node->uses())
463 if (!isa<MemSDNode>(*UI))
464 for (SDNode *UII : UI->uses())
465 if (!isa<MemSDNode>(*UII))
466 return false;
467 return true;
468}
469
470/// Determine whether it is worth to fold V into an extended register.
471bool AArch64DAGToDAGISel::isWorthFolding(SDValue V) const {
472 // Trivial if we are optimizing for code size or if there is only
473 // one use of the value.
474 if (CurDAG->shouldOptForSize() || V.hasOneUse())
475 return true;
476 // If a subtarget has a fastpath LSL we can fold a logical shift into
477 // the addressing mode and save a cycle.
478 if (Subtarget->hasLSLFast() && V.getOpcode() == ISD::SHL &&
479 isWorthFoldingSHL(V))
480 return true;
481 if (Subtarget->hasLSLFast() && V.getOpcode() == ISD::ADD) {
482 const SDValue LHS = V.getOperand(0);
483 const SDValue RHS = V.getOperand(1);
484 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(LHS))
485 return true;
486 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(RHS))
487 return true;
488 }
489
490 // It hurts otherwise, since the value will be reused.
491 return false;
492}
493
494/// SelectShiftedRegister - Select a "shifted register" operand. If the value
495/// is not shifted, set the Shift operand to default of "LSL 0". The logical
496/// instructions allow the shifted register to be rotated, but the arithmetic
497/// instructions do not. The AllowROR parameter specifies whether ROR is
498/// supported.
499bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
500 SDValue &Reg, SDValue &Shift) {
501 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
502 if (ShType == AArch64_AM::InvalidShiftExtend)
503 return false;
504 if (!AllowROR && ShType == AArch64_AM::ROR)
505 return false;
506
507 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
508 unsigned BitSize = N.getValueSizeInBits();
509 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
510 unsigned ShVal = AArch64_AM::getShifterImm(ShType, Val);
511
512 Reg = N.getOperand(0);
513 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32);
514 return isWorthFolding(N);
515 }
516
517 return false;
518}
519
520/// getExtendTypeForNode - Translate an extend node to the corresponding
521/// ExtendType value.
522static AArch64_AM::ShiftExtendType
523getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
524 if (N.getOpcode() == ISD::SIGN_EXTEND ||
525 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
526 EVT SrcVT;
527 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
528 SrcVT = cast<VTSDNode>(N.getOperand(1))->getVT();
529 else
530 SrcVT = N.getOperand(0).getValueType();
531
532 if (!IsLoadStore && SrcVT == MVT::i8)
533 return AArch64_AM::SXTB;
534 else if (!IsLoadStore && SrcVT == MVT::i16)
535 return AArch64_AM::SXTH;
536 else if (SrcVT == MVT::i32)
537 return AArch64_AM::SXTW;
538 assert(SrcVT != MVT::i64 && "extend from 64-bits?")((SrcVT != MVT::i64 && "extend from 64-bits?") ? static_cast
<void> (0) : __assert_fail ("SrcVT != MVT::i64 && \"extend from 64-bits?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 538, __PRETTY_FUNCTION__))
;
539
540 return AArch64_AM::InvalidShiftExtend;
541 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
542 N.getOpcode() == ISD::ANY_EXTEND) {
543 EVT SrcVT = N.getOperand(0).getValueType();
544 if (!IsLoadStore && SrcVT == MVT::i8)
545 return AArch64_AM::UXTB;
546 else if (!IsLoadStore && SrcVT == MVT::i16)
547 return AArch64_AM::UXTH;
548 else if (SrcVT == MVT::i32)
549 return AArch64_AM::UXTW;
550 assert(SrcVT != MVT::i64 && "extend from 64-bits?")((SrcVT != MVT::i64 && "extend from 64-bits?") ? static_cast
<void> (0) : __assert_fail ("SrcVT != MVT::i64 && \"extend from 64-bits?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 550, __PRETTY_FUNCTION__))
;
551
552 return AArch64_AM::InvalidShiftExtend;
553 } else if (N.getOpcode() == ISD::AND) {
554 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
555 if (!CSD)
556 return AArch64_AM::InvalidShiftExtend;
557 uint64_t AndMask = CSD->getZExtValue();
558
559 switch (AndMask) {
560 default:
561 return AArch64_AM::InvalidShiftExtend;
562 case 0xFF:
563 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
564 case 0xFFFF:
565 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
566 case 0xFFFFFFFF:
567 return AArch64_AM::UXTW;
568 }
569 }
570
571 return AArch64_AM::InvalidShiftExtend;
572}
573
574// Helper for SelectMLAV64LaneV128 - Recognize high lane extracts.
575static bool checkHighLaneIndex(SDNode *DL, SDValue &LaneOp, int &LaneIdx) {
576 if (DL->getOpcode() != AArch64ISD::DUPLANE16 &&
577 DL->getOpcode() != AArch64ISD::DUPLANE32)
578 return false;
579
580 SDValue SV = DL->getOperand(0);
581 if (SV.getOpcode() != ISD::INSERT_SUBVECTOR)
582 return false;
583
584 SDValue EV = SV.getOperand(1);
585 if (EV.getOpcode() != ISD::EXTRACT_SUBVECTOR)
586 return false;
587
588 ConstantSDNode *DLidx = cast<ConstantSDNode>(DL->getOperand(1).getNode());
589 ConstantSDNode *EVidx = cast<ConstantSDNode>(EV.getOperand(1).getNode());
590 LaneIdx = DLidx->getSExtValue() + EVidx->getSExtValue();
591 LaneOp = EV.getOperand(0);
592
593 return true;
594}
595
596// Helper for SelectOpcV64LaneV128 - Recognize operations where one operand is a
597// high lane extract.
598static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
599 SDValue &LaneOp, int &LaneIdx) {
600
601 if (!checkHighLaneIndex(Op0.getNode(), LaneOp, LaneIdx)) {
602 std::swap(Op0, Op1);
603 if (!checkHighLaneIndex(Op0.getNode(), LaneOp, LaneIdx))
604 return false;
605 }
606 StdOp = Op1;
607 return true;
608}
609
610/// SelectMLAV64LaneV128 - AArch64 supports vector MLAs where one multiplicand
611/// is a lane in the upper half of a 128-bit vector. Recognize and select this
612/// so that we don't emit unnecessary lane extracts.
613bool AArch64DAGToDAGISel::tryMLAV64LaneV128(SDNode *N) {
614 SDLoc dl(N);
615 SDValue Op0 = N->getOperand(0);
616 SDValue Op1 = N->getOperand(1);
617 SDValue MLAOp1; // Will hold ordinary multiplicand for MLA.
618 SDValue MLAOp2; // Will hold lane-accessed multiplicand for MLA.
619 int LaneIdx = -1; // Will hold the lane index.
620
621 if (Op1.getOpcode() != ISD::MUL ||
622 !checkV64LaneV128(Op1.getOperand(0), Op1.getOperand(1), MLAOp1, MLAOp2,
623 LaneIdx)) {
624 std::swap(Op0, Op1);
625 if (Op1.getOpcode() != ISD::MUL ||
626 !checkV64LaneV128(Op1.getOperand(0), Op1.getOperand(1), MLAOp1, MLAOp2,
627 LaneIdx))
628 return false;
629 }
630
631 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
632
633 SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
634
635 unsigned MLAOpc = ~0U;
636
637 switch (N->getSimpleValueType(0).SimpleTy) {
638 default:
639 llvm_unreachable("Unrecognized MLA.")::llvm::llvm_unreachable_internal("Unrecognized MLA.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 639)
;
640 case MVT::v4i16:
641 MLAOpc = AArch64::MLAv4i16_indexed;
642 break;
643 case MVT::v8i16:
644 MLAOpc = AArch64::MLAv8i16_indexed;
645 break;
646 case MVT::v2i32:
647 MLAOpc = AArch64::MLAv2i32_indexed;
648 break;
649 case MVT::v4i32:
650 MLAOpc = AArch64::MLAv4i32_indexed;
651 break;
652 }
653
654 ReplaceNode(N, CurDAG->getMachineNode(MLAOpc, dl, N->getValueType(0), Ops));
655 return true;
656}
657
658bool AArch64DAGToDAGISel::tryMULLV64LaneV128(unsigned IntNo, SDNode *N) {
659 SDLoc dl(N);
660 SDValue SMULLOp0;
661 SDValue SMULLOp1;
662 int LaneIdx;
663
664 if (!checkV64LaneV128(N->getOperand(1), N->getOperand(2), SMULLOp0, SMULLOp1,
665 LaneIdx))
666 return false;
667
668 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
669
670 SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
671
672 unsigned SMULLOpc = ~0U;
673
674 if (IntNo == Intrinsic::aarch64_neon_smull) {
675 switch (N->getSimpleValueType(0).SimpleTy) {
676 default:
677 llvm_unreachable("Unrecognized SMULL.")::llvm::llvm_unreachable_internal("Unrecognized SMULL.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 677)
;
678 case MVT::v4i32:
679 SMULLOpc = AArch64::SMULLv4i16_indexed;
680 break;
681 case MVT::v2i64:
682 SMULLOpc = AArch64::SMULLv2i32_indexed;
683 break;
684 }
685 } else if (IntNo == Intrinsic::aarch64_neon_umull) {
686 switch (N->getSimpleValueType(0).SimpleTy) {
687 default:
688 llvm_unreachable("Unrecognized SMULL.")::llvm::llvm_unreachable_internal("Unrecognized SMULL.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 688)
;
689 case MVT::v4i32:
690 SMULLOpc = AArch64::UMULLv4i16_indexed;
691 break;
692 case MVT::v2i64:
693 SMULLOpc = AArch64::UMULLv2i32_indexed;
694 break;
695 }
696 } else
697 llvm_unreachable("Unrecognized intrinsic.")::llvm::llvm_unreachable_internal("Unrecognized intrinsic.", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 697)
;
698
699 ReplaceNode(N, CurDAG->getMachineNode(SMULLOpc, dl, N->getValueType(0), Ops));
700 return true;
701}
702
703/// Instructions that accept extend modifiers like UXTW expect the register
704/// being extended to be a GPR32, but the incoming DAG might be acting on a
705/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
706/// this is the case.
707static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
708 if (N.getValueType() == MVT::i32)
709 return N;
710
711 SDLoc dl(N);
712 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
713 MachineSDNode *Node = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
714 dl, MVT::i32, N, SubReg);
715 return SDValue(Node, 0);
716}
717
718// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
719template<signed Low, signed High, signed Scale>
720bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
721 if (!isa<ConstantSDNode>(N))
722 return false;
723
724 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
725 if ((MulImm % std::abs(Scale)) == 0) {
726 int64_t RDVLImm = MulImm / Scale;
727 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
728 Imm = CurDAG->getTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
729 return true;
730 }
731 }
732
733 return false;
734}
735
736/// SelectArithExtendedRegister - Select a "extended register" operand. This
737/// operand folds in an extend followed by an optional left shift.
738bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
739 SDValue &Shift) {
740 unsigned ShiftVal = 0;
741 AArch64_AM::ShiftExtendType Ext;
742
743 if (N.getOpcode() == ISD::SHL) {
744 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
745 if (!CSD)
746 return false;
747 ShiftVal = CSD->getZExtValue();
748 if (ShiftVal > 4)
749 return false;
750
751 Ext = getExtendTypeForNode(N.getOperand(0));
752 if (Ext == AArch64_AM::InvalidShiftExtend)
753 return false;
754
755 Reg = N.getOperand(0).getOperand(0);
756 } else {
757 Ext = getExtendTypeForNode(N);
758 if (Ext == AArch64_AM::InvalidShiftExtend)
759 return false;
760
761 Reg = N.getOperand(0);
762
763 // Don't match if free 32-bit -> 64-bit zext can be used instead.
764 if (Ext == AArch64_AM::UXTW &&
765 Reg->getValueType(0).getSizeInBits() == 32 && isDef32(*Reg.getNode()))
766 return false;
767 }
768
769 // AArch64 mandates that the RHS of the operation must use the smallest
770 // register class that could contain the size being extended from. Thus,
771 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
772 // there might not be an actual 32-bit value in the program. We can
773 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
774 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX)((Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX)
? static_cast<void> (0) : __assert_fail ("Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 774, __PRETTY_FUNCTION__))
;
775 Reg = narrowIfNeeded(CurDAG, Reg);
776 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
777 MVT::i32);
778 return isWorthFolding(N);
779}
780
781/// If there's a use of this ADDlow that's not itself a load/store then we'll
782/// need to create a real ADD instruction from it anyway and there's no point in
783/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
784/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
785/// leads to duplicated ADRP instructions.
786static bool isWorthFoldingADDlow(SDValue N) {
787 for (auto Use : N->uses()) {
788 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
789 Use->getOpcode() != ISD::ATOMIC_LOAD &&
790 Use->getOpcode() != ISD::ATOMIC_STORE)
791 return false;
792
793 // ldar and stlr have much more restrictive addressing modes (just a
794 // register).
795 if (isStrongerThanMonotonic(cast<MemSDNode>(Use)->getOrdering()))
796 return false;
797 }
798
799 return true;
800}
801
802/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
803/// immediate" address. The "Size" argument is the size in bytes of the memory
804/// reference, which determines the scale.
805bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
806 unsigned BW, unsigned Size,
807 SDValue &Base,
808 SDValue &OffImm) {
809 SDLoc dl(N);
810 const DataLayout &DL = CurDAG->getDataLayout();
811 const TargetLowering *TLI = getTargetLowering();
812 if (N.getOpcode() == ISD::FrameIndex) {
813 int FI = cast<FrameIndexSDNode>(N)->getIndex();
814 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
815 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
816 return true;
817 }
818
819 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
820 // selected here doesn't support labels/immediates, only base+offset.
821 if (CurDAG->isBaseWithConstantOffset(N)) {
822 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
823 if (IsSignedImm) {
824 int64_t RHSC = RHS->getSExtValue();
825 unsigned Scale = Log2_32(Size);
826 int64_t Range = 0x1LL << (BW - 1);
827
828 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
829 RHSC < (Range << Scale)) {
830 Base = N.getOperand(0);
831 if (Base.getOpcode() == ISD::FrameIndex) {
832 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
833 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
834 }
835 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
836 return true;
837 }
838 } else {
839 // unsigned Immediate
840 uint64_t RHSC = RHS->getZExtValue();
841 unsigned Scale = Log2_32(Size);
842 uint64_t Range = 0x1ULL << BW;
843
844 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
845 Base = N.getOperand(0);
846 if (Base.getOpcode() == ISD::FrameIndex) {
847 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
848 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
849 }
850 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
851 return true;
852 }
853 }
854 }
855 }
856 // Base only. The address will be materialized into a register before
857 // the memory is accessed.
858 // add x0, Xbase, #offset
859 // stp x1, x2, [x0]
860 Base = N;
861 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
862 return true;
863}
864
865/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
866/// immediate" address. The "Size" argument is the size in bytes of the memory
867/// reference, which determines the scale.
868bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
869 SDValue &Base, SDValue &OffImm) {
870 SDLoc dl(N);
871 const DataLayout &DL = CurDAG->getDataLayout();
872 const TargetLowering *TLI = getTargetLowering();
873 if (N.getOpcode() == ISD::FrameIndex) {
874 int FI = cast<FrameIndexSDNode>(N)->getIndex();
875 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
876 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
877 return true;
878 }
879
880 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
881 GlobalAddressSDNode *GAN =
882 dyn_cast<GlobalAddressSDNode>(N.getOperand(1).getNode());
883 Base = N.getOperand(0);
884 OffImm = N.getOperand(1);
885 if (!GAN)
886 return true;
887
888 if (GAN->getOffset() % Size == 0) {
889 const GlobalValue *GV = GAN->getGlobal();
890 unsigned Alignment = GV->getAlignment();
891 Type *Ty = GV->getValueType();
892 if (Alignment == 0 && Ty->isSized())
893 Alignment = DL.getABITypeAlignment(Ty);
894
895 if (Alignment >= Size)
896 return true;
897 }
898 }
899
900 if (CurDAG->isBaseWithConstantOffset(N)) {
901 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
902 int64_t RHSC = (int64_t)RHS->getZExtValue();
903 unsigned Scale = Log2_32(Size);
904 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
905 Base = N.getOperand(0);
906 if (Base.getOpcode() == ISD::FrameIndex) {
907 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
908 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
909 }
910 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
911 return true;
912 }
913 }
914 }
915
916 // Before falling back to our general case, check if the unscaled
917 // instructions can handle this. If so, that's preferable.
918 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
919 return false;
920
921 // Base only. The address will be materialized into a register before
922 // the memory is accessed.
923 // add x0, Xbase, #offset
924 // ldr x0, [x0]
925 Base = N;
926 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
927 return true;
928}
929
930/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
931/// immediate" address. This should only match when there is an offset that
932/// is not valid for a scaled immediate addressing mode. The "Size" argument
933/// is the size in bytes of the memory reference, which is needed here to know
934/// what is valid for a scaled immediate.
935bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
936 SDValue &Base,
937 SDValue &OffImm) {
938 if (!CurDAG->isBaseWithConstantOffset(N))
939 return false;
940 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
941 int64_t RHSC = RHS->getSExtValue();
942 // If the offset is valid as a scaled immediate, don't match here.
943 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 &&
944 RHSC < (0x1000 << Log2_32(Size)))
945 return false;
946 if (RHSC >= -256 && RHSC < 256) {
947 Base = N.getOperand(0);
948 if (Base.getOpcode() == ISD::FrameIndex) {
949 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
950 const TargetLowering *TLI = getTargetLowering();
951 Base = CurDAG->getTargetFrameIndex(
952 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
953 }
954 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
955 return true;
956 }
957 }
958 return false;
959}
960
961static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
962 SDLoc dl(N);
963 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
964 SDValue ImpDef = SDValue(
965 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
966 MachineSDNode *Node = CurDAG->getMachineNode(
967 TargetOpcode::INSERT_SUBREG, dl, MVT::i64, ImpDef, N, SubReg);
968 return SDValue(Node, 0);
969}
970
971/// Check if the given SHL node (\p N), can be used to form an
972/// extended register for an addressing mode.
973bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
974 bool WantExtend, SDValue &Offset,
975 SDValue &SignExtend) {
976 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.")((N.getOpcode() == ISD::SHL && "Invalid opcode.") ? static_cast
<void> (0) : __assert_fail ("N.getOpcode() == ISD::SHL && \"Invalid opcode.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 976, __PRETTY_FUNCTION__))
;
977 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
978 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
979 return false;
980
981 SDLoc dl(N);
982 if (WantExtend) {
983 AArch64_AM::ShiftExtendType Ext =
984 getExtendTypeForNode(N.getOperand(0), true);
985 if (Ext == AArch64_AM::InvalidShiftExtend)
986 return false;
987
988 Offset = narrowIfNeeded(CurDAG, N.getOperand(0).getOperand(0));
989 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
990 MVT::i32);
991 } else {
992 Offset = N.getOperand(0);
993 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
994 }
995
996 unsigned LegalShiftVal = Log2_32(Size);
997 unsigned ShiftVal = CSD->getZExtValue();
998
999 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1000 return false;
1001
1002 return isWorthFolding(N);
1003}
1004
1005bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1006 SDValue &Base, SDValue &Offset,
1007 SDValue &SignExtend,
1008 SDValue &DoShift) {
1009 if (N.getOpcode() != ISD::ADD)
1010 return false;
1011 SDValue LHS = N.getOperand(0);
1012 SDValue RHS = N.getOperand(1);
1013 SDLoc dl(N);
1014
1015 // We don't want to match immediate adds here, because they are better lowered
1016 // to the register-immediate addressing modes.
1017 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
1018 return false;
1019
1020 // Check if this particular node is reused in any non-memory related
1021 // operation. If yes, do not try to fold this node into the address
1022 // computation, since the computation will be kept.
1023 const SDNode *Node = N.getNode();
1024 for (SDNode *UI : Node->uses()) {
1025 if (!isa<MemSDNode>(*UI))
1026 return false;
1027 }
1028
1029 // Remember if it is worth folding N when it produces extended register.
1030 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
1031
1032 // Try to match a shifted extend on the RHS.
1033 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1034 SelectExtendedSHL(RHS, Size, true, Offset, SignExtend)) {
1035 Base = LHS;
1036 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1037 return true;
1038 }
1039
1040 // Try to match a shifted extend on the LHS.
1041 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1042 SelectExtendedSHL(LHS, Size, true, Offset, SignExtend)) {
1043 Base = RHS;
1044 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1045 return true;
1046 }
1047
1048 // There was no shift, whatever else we find.
1049 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32);
1050
1051 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
1052 // Try to match an unshifted extend on the LHS.
1053 if (IsExtendedRegisterWorthFolding &&
1054 (Ext = getExtendTypeForNode(LHS, true)) !=
1055 AArch64_AM::InvalidShiftExtend) {
1056 Base = RHS;
1057 Offset = narrowIfNeeded(CurDAG, LHS.getOperand(0));
1058 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1059 MVT::i32);
1060 if (isWorthFolding(LHS))
1061 return true;
1062 }
1063
1064 // Try to match an unshifted extend on the RHS.
1065 if (IsExtendedRegisterWorthFolding &&
1066 (Ext = getExtendTypeForNode(RHS, true)) !=
1067 AArch64_AM::InvalidShiftExtend) {
1068 Base = LHS;
1069 Offset = narrowIfNeeded(CurDAG, RHS.getOperand(0));
1070 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1071 MVT::i32);
1072 if (isWorthFolding(RHS))
1073 return true;
1074 }
1075
1076 return false;
1077}
1078
1079// Check if the given immediate is preferred by ADD. If an immediate can be
1080// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1081// encoded by one MOVZ, return true.
1082static bool isPreferredADD(int64_t ImmOff) {
1083 // Constant in [0x0, 0xfff] can be encoded in ADD.
1084 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1085 return true;
1086 // Check if it can be encoded in an "ADD LSL #12".
1087 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1088 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1089 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1090 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1091 return false;
1092}
1093
1094bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1095 SDValue &Base, SDValue &Offset,
1096 SDValue &SignExtend,
1097 SDValue &DoShift) {
1098 if (N.getOpcode() != ISD::ADD)
1099 return false;
1100 SDValue LHS = N.getOperand(0);
1101 SDValue RHS = N.getOperand(1);
1102 SDLoc DL(N);
1103
1104 // Check if this particular node is reused in any non-memory related
1105 // operation. If yes, do not try to fold this node into the address
1106 // computation, since the computation will be kept.
1107 const SDNode *Node = N.getNode();
1108 for (SDNode *UI : Node->uses()) {
1109 if (!isa<MemSDNode>(*UI))
1110 return false;
1111 }
1112
1113 // Watch out if RHS is a wide immediate, it can not be selected into
1114 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1115 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1116 // instructions like:
1117 // MOV X0, WideImmediate
1118 // ADD X1, BaseReg, X0
1119 // LDR X2, [X1, 0]
1120 // For such situation, using [BaseReg, XReg] addressing mode can save one
1121 // ADD/SUB:
1122 // MOV X0, WideImmediate
1123 // LDR X2, [BaseReg, X0]
1124 if (isa<ConstantSDNode>(RHS)) {
1125 int64_t ImmOff = (int64_t)cast<ConstantSDNode>(RHS)->getZExtValue();
1126 unsigned Scale = Log2_32(Size);
1127 // Skip the immediate can be selected by load/store addressing mode.
1128 // Also skip the immediate can be encoded by a single ADD (SUB is also
1129 // checked by using -ImmOff).
1130 if ((ImmOff % Size == 0 && ImmOff >= 0 && ImmOff < (0x1000 << Scale)) ||
1131 isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
1132 return false;
1133
1134 SDValue Ops[] = { RHS };
1135 SDNode *MOVI =
1136 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
1137 SDValue MOVIV = SDValue(MOVI, 0);
1138 // This ADD of two X register will be selected into [Reg+Reg] mode.
1139 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV);
1140 }
1141
1142 // Remember if it is worth folding N when it produces extended register.
1143 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
1144
1145 // Try to match a shifted extend on the RHS.
1146 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1147 SelectExtendedSHL(RHS, Size, false, Offset, SignExtend)) {
1148 Base = LHS;
1149 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1150 return true;
1151 }
1152
1153 // Try to match a shifted extend on the LHS.
1154 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1155 SelectExtendedSHL(LHS, Size, false, Offset, SignExtend)) {
1156 Base = RHS;
1157 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1158 return true;
1159 }
1160
1161 // Match any non-shifted, non-extend, non-immediate add expression.
1162 Base = LHS;
1163 Offset = RHS;
1164 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32);
1165 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32);
1166 // Reg1 + Reg2 is free: no check needed.
1167 return true;
1168}
1169
1170SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1171 static const unsigned RegClassIDs[] = {
1172 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1173 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1174 AArch64::dsub2, AArch64::dsub3};
1175
1176 return createTuple(Regs, RegClassIDs, SubRegs);
1177}
1178
1179SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1180 static const unsigned RegClassIDs[] = {
1181 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1182 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1183 AArch64::qsub2, AArch64::qsub3};
1184
1185 return createTuple(Regs, RegClassIDs, SubRegs);
1186}
1187
1188SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1189 const unsigned RegClassIDs[],
1190 const unsigned SubRegs[]) {
1191 // There's no special register-class for a vector-list of 1 element: it's just
1192 // a vector.
1193 if (Regs.size() == 1)
1194 return Regs[0];
1195
1196 assert(Regs.size() >= 2 && Regs.size() <= 4)((Regs.size() >= 2 && Regs.size() <= 4) ? static_cast
<void> (0) : __assert_fail ("Regs.size() >= 2 && Regs.size() <= 4"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1196, __PRETTY_FUNCTION__))
;
1197
1198 SDLoc DL(Regs[0]);
1199
1200 SmallVector<SDValue, 4> Ops;
1201
1202 // First operand of REG_SEQUENCE is the desired RegClass.
1203 Ops.push_back(
1204 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32));
1205
1206 // Then we get pairs of source & subregister-position for the components.
1207 for (unsigned i = 0; i < Regs.size(); ++i) {
1208 Ops.push_back(Regs[i]);
1209 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32));
1210 }
1211
1212 SDNode *N =
1213 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
1214 return SDValue(N, 0);
1215}
1216
1217void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1218 bool isExt) {
1219 SDLoc dl(N);
1220 EVT VT = N->getValueType(0);
1221
1222 unsigned ExtOff = isExt;
1223
1224 // Form a REG_SEQUENCE to force register allocation.
1225 unsigned Vec0Off = ExtOff + 1;
1226 SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Off,
1227 N->op_begin() + Vec0Off + NumVecs);
1228 SDValue RegSeq = createQTuple(Regs);
1229
1230 SmallVector<SDValue, 6> Ops;
1231 if (isExt)
1232 Ops.push_back(N->getOperand(1));
1233 Ops.push_back(RegSeq);
1234 Ops.push_back(N->getOperand(NumVecs + ExtOff + 1));
1235 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1236}
1237
1238bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1239 LoadSDNode *LD = cast<LoadSDNode>(N);
1240 if (LD->isUnindexed())
1241 return false;
1242 EVT VT = LD->getMemoryVT();
1243 EVT DstVT = N->getValueType(0);
1244 ISD::MemIndexedMode AM = LD->getAddressingMode();
1245 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1246
1247 // We're not doing validity checking here. That was done when checking
1248 // if we should mark the load as indexed or not. We're just selecting
1249 // the right instruction.
1250 unsigned Opcode = 0;
1251
1252 ISD::LoadExtType ExtType = LD->getExtensionType();
1253 bool InsertTo64 = false;
1254 if (VT == MVT::i64)
1255 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1256 else if (VT == MVT::i32) {
1257 if (ExtType == ISD::NON_EXTLOAD)
1258 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1259 else if (ExtType == ISD::SEXTLOAD)
1260 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1261 else {
1262 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1263 InsertTo64 = true;
1264 // The result of the load is only i32. It's the subreg_to_reg that makes
1265 // it into an i64.
1266 DstVT = MVT::i32;
1267 }
1268 } else if (VT == MVT::i16) {
1269 if (ExtType == ISD::SEXTLOAD) {
1270 if (DstVT == MVT::i64)
1271 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1272 else
1273 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1274 } else {
1275 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1276 InsertTo64 = DstVT == MVT::i64;
1277 // The result of the load is only i32. It's the subreg_to_reg that makes
1278 // it into an i64.
1279 DstVT = MVT::i32;
1280 }
1281 } else if (VT == MVT::i8) {
1282 if (ExtType == ISD::SEXTLOAD) {
1283 if (DstVT == MVT::i64)
1284 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1285 else
1286 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1287 } else {
1288 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1289 InsertTo64 = DstVT == MVT::i64;
1290 // The result of the load is only i32. It's the subreg_to_reg that makes
1291 // it into an i64.
1292 DstVT = MVT::i32;
1293 }
1294 } else if (VT == MVT::f16) {
1295 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1296 } else if (VT == MVT::f32) {
1297 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1298 } else if (VT == MVT::f64 || VT.is64BitVector()) {
1299 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1300 } else if (VT.is128BitVector()) {
1301 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1302 } else
1303 return false;
1304 SDValue Chain = LD->getChain();
1305 SDValue Base = LD->getBasePtr();
1306 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
1307 int OffsetVal = (int)OffsetOp->getZExtValue();
1308 SDLoc dl(N);
1309 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1310 SDValue Ops[] = { Base, Offset, Chain };
1311 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1312 MVT::Other, Ops);
1313 // Either way, we're replacing the node, so tell the caller that.
1314 SDValue LoadedVal = SDValue(Res, 1);
1315 if (InsertTo64) {
1316 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1317 LoadedVal =
1318 SDValue(CurDAG->getMachineNode(
1319 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1320 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1321 SubReg),
1322 0);
1323 }
1324
1325 ReplaceUses(SDValue(N, 0), LoadedVal);
1326 ReplaceUses(SDValue(N, 1), SDValue(Res, 0));
1327 ReplaceUses(SDValue(N, 2), SDValue(Res, 2));
1328 CurDAG->RemoveDeadNode(N);
1329 return true;
1330}
1331
1332void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
1333 unsigned SubRegIdx) {
1334 SDLoc dl(N);
1335 EVT VT = N->getValueType(0);
1336 SDValue Chain = N->getOperand(0);
1337
1338 SDValue Ops[] = {N->getOperand(2), // Mem operand;
1339 Chain};
1340
1341 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1342
1343 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1344 SDValue SuperReg = SDValue(Ld, 0);
1345 for (unsigned i = 0; i < NumVecs; ++i)
1346 ReplaceUses(SDValue(N, i),
1347 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1348
1349 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
1350
1351 // Transfer memoperands.
1352 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1353 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
1354
1355 CurDAG->RemoveDeadNode(N);
1356}
1357
1358void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
1359 unsigned Opc, unsigned SubRegIdx) {
1360 SDLoc dl(N);
1361 EVT VT = N->getValueType(0);
1362 SDValue Chain = N->getOperand(0);
1363
1364 SDValue Ops[] = {N->getOperand(1), // Mem operand
1365 N->getOperand(2), // Incremental
1366 Chain};
1367
1368 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1369 MVT::Untyped, MVT::Other};
1370
1371 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1372
1373 // Update uses of write back register
1374 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
1375
1376 // Update uses of vector list
1377 SDValue SuperReg = SDValue(Ld, 1);
1378 if (NumVecs == 1)
1379 ReplaceUses(SDValue(N, 0), SuperReg);
1380 else
1381 for (unsigned i = 0; i < NumVecs; ++i)
1382 ReplaceUses(SDValue(N, i),
1383 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1384
1385 // Update the chain
1386 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
1387 CurDAG->RemoveDeadNode(N);
1388}
1389
1390void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
1391 unsigned Opc) {
1392 SDLoc dl(N);
1393 EVT VT = N->getOperand(2)->getValueType(0);
1394
1395 // Form a REG_SEQUENCE to force register allocation.
1396 bool Is128Bit = VT.getSizeInBits() == 128;
1397 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1398 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1399
1400 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)};
1401 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
1402
1403 // Transfer memoperands.
1404 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1405 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1406
1407 ReplaceNode(N, St);
1408}
1409
1410bool AArch64DAGToDAGISel::SelectAddrModeFrameIndexSVE(SDValue N, SDValue &Base,
1411 SDValue &OffImm) {
1412 SDLoc dl(N);
1413 const DataLayout &DL = CurDAG->getDataLayout();
1414 const TargetLowering *TLI = getTargetLowering();
1415
1416 // Try to match it for the frame address
1417 if (auto FINode = dyn_cast<FrameIndexSDNode>(N)) {
1418 int FI = FINode->getIndex();
1419 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1420 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1421 return true;
1422 }
1423
1424 return false;
1425}
1426
1427void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
1428 unsigned Opc) {
1429 SDLoc dl(N);
1430 EVT VT = N->getOperand(2)->getValueType(0);
1431 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1432 MVT::Other}; // Type for the Chain
1433
1434 // Form a REG_SEQUENCE to force register allocation.
1435 bool Is128Bit = VT.getSizeInBits() == 128;
1436 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1437 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1438
1439 SDValue Ops[] = {RegSeq,
1440 N->getOperand(NumVecs + 1), // base register
1441 N->getOperand(NumVecs + 2), // Incremental
1442 N->getOperand(0)}; // Chain
1443 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1444
1445 ReplaceNode(N, St);
1446}
1447
1448namespace {
1449/// WidenVector - Given a value in the V64 register class, produce the
1450/// equivalent value in the V128 register class.
1451class WidenVector {
1452 SelectionDAG &DAG;
1453
1454public:
1455 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
1456
1457 SDValue operator()(SDValue V64Reg) {
1458 EVT VT = V64Reg.getValueType();
1459 unsigned NarrowSize = VT.getVectorNumElements();
1460 MVT EltTy = VT.getVectorElementType().getSimpleVT();
1461 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
1462 SDLoc DL(V64Reg);
1463
1464 SDValue Undef =
1465 SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, WideTy), 0);
1466 return DAG.getTargetInsertSubreg(AArch64::dsub, DL, WideTy, Undef, V64Reg);
1467 }
1468};
1469} // namespace
1470
1471/// NarrowVector - Given a value in the V128 register class, produce the
1472/// equivalent value in the V64 register class.
1473static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
1474 EVT VT = V128Reg.getValueType();
8
Calling 'SDValue::getValueType'
1475 unsigned WideSize = VT.getVectorNumElements();
1476 MVT EltTy = VT.getVectorElementType().getSimpleVT();
1477 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
1478
1479 return DAG.getTargetExtractSubreg(AArch64::dsub, SDLoc(V128Reg), NarrowTy,
1480 V128Reg);
1481}
1482
1483void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
1484 unsigned Opc) {
1485 SDLoc dl(N);
1486 EVT VT = N->getValueType(0);
1487 bool Narrow = VT.getSizeInBits() == 64;
1488
1489 // Form a REG_SEQUENCE to force register allocation.
1490 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1491
1492 if (Narrow)
1493 transform(Regs, Regs.begin(),
1494 WidenVector(*CurDAG));
1495
1496 SDValue RegSeq = createQTuple(Regs);
1497
1498 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1499
1500 unsigned LaneNo =
1501 cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
1502
1503 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1504 N->getOperand(NumVecs + 3), N->getOperand(0)};
1505 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1506 SDValue SuperReg = SDValue(Ld, 0);
1507
1508 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1509 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1510 AArch64::qsub2, AArch64::qsub3 };
1511 for (unsigned i = 0; i < NumVecs; ++i) {
1512 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
1513 if (Narrow)
1514 NV = NarrowVector(NV, *CurDAG);
1515 ReplaceUses(SDValue(N, i), NV);
1516 }
1517
1518 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
1519 CurDAG->RemoveDeadNode(N);
1520}
1521
1522void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
1523 unsigned Opc) {
1524 SDLoc dl(N);
1525 EVT VT = N->getValueType(0);
1526 bool Narrow = VT.getSizeInBits() == 64;
1
Assuming the condition is true
1527
1528 // Form a REG_SEQUENCE to force register allocation.
1529 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1530
1531 if (Narrow
1.1
'Narrow' is true
1.1
'Narrow' is true
)
2
Taking true branch
1532 transform(Regs, Regs.begin(),
1533 WidenVector(*CurDAG));
1534
1535 SDValue RegSeq = createQTuple(Regs);
1536
1537 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1538 RegSeq->getValueType(0), MVT::Other};
1539
1540 unsigned LaneNo =
1541 cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
1542
1543 SDValue Ops[] = {RegSeq,
1544 CurDAG->getTargetConstant(LaneNo, dl,
1545 MVT::i64), // Lane Number
1546 N->getOperand(NumVecs + 2), // Base register
1547 N->getOperand(NumVecs + 3), // Incremental
1548 N->getOperand(0)};
1549 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1550
1551 // Update uses of the write back register
1552 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
1553
1554 // Update uses of the vector list
1555 SDValue SuperReg = SDValue(Ld, 1);
1556 if (NumVecs == 1) {
3
Assuming 'NumVecs' is equal to 1
4
Taking true branch
1557 ReplaceUses(SDValue(N, 0),
1558 Narrow
4.1
'Narrow' is true
4.1
'Narrow' is true
? NarrowVector(SuperReg, *CurDAG) : SuperReg);
5
'?' condition is true
6
Null pointer value stored to 'V128Reg.Node'
7
Calling 'NarrowVector'
1559 } else {
1560 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1561 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1562 AArch64::qsub2, AArch64::qsub3 };
1563 for (unsigned i = 0; i < NumVecs; ++i) {
1564 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
1565 SuperReg);
1566 if (Narrow)
1567 NV = NarrowVector(NV, *CurDAG);
1568 ReplaceUses(SDValue(N, i), NV);
1569 }
1570 }
1571
1572 // Update the Chain
1573 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
1574 CurDAG->RemoveDeadNode(N);
1575}
1576
1577void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
1578 unsigned Opc) {
1579 SDLoc dl(N);
1580 EVT VT = N->getOperand(2)->getValueType(0);
1581 bool Narrow = VT.getSizeInBits() == 64;
1582
1583 // Form a REG_SEQUENCE to force register allocation.
1584 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1585
1586 if (Narrow)
1587 transform(Regs, Regs.begin(),
1588 WidenVector(*CurDAG));
1589
1590 SDValue RegSeq = createQTuple(Regs);
1591
1592 unsigned LaneNo =
1593 cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
1594
1595 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1596 N->getOperand(NumVecs + 3), N->getOperand(0)};
1597 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
1598
1599 // Transfer memoperands.
1600 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1601 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1602
1603 ReplaceNode(N, St);
1604}
1605
1606void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
1607 unsigned Opc) {
1608 SDLoc dl(N);
1609 EVT VT = N->getOperand(2)->getValueType(0);
1610 bool Narrow = VT.getSizeInBits() == 64;
1611
1612 // Form a REG_SEQUENCE to force register allocation.
1613 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1614
1615 if (Narrow)
1616 transform(Regs, Regs.begin(),
1617 WidenVector(*CurDAG));
1618
1619 SDValue RegSeq = createQTuple(Regs);
1620
1621 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1622 MVT::Other};
1623
1624 unsigned LaneNo =
1625 cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
1626
1627 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1628 N->getOperand(NumVecs + 2), // Base Register
1629 N->getOperand(NumVecs + 3), // Incremental
1630 N->getOperand(0)};
1631 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1632
1633 // Transfer memoperands.
1634 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1635 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1636
1637 ReplaceNode(N, St);
1638}
1639
1640static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
1641 unsigned &Opc, SDValue &Opd0,
1642 unsigned &LSB, unsigned &MSB,
1643 unsigned NumberOfIgnoredLowBits,
1644 bool BiggerPattern) {
1645 assert(N->getOpcode() == ISD::AND &&((N->getOpcode() == ISD::AND && "N must be a AND operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::AND && \"N must be a AND operation to call this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1646, __PRETTY_FUNCTION__))
1646 "N must be a AND operation to call this function")((N->getOpcode() == ISD::AND && "N must be a AND operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::AND && \"N must be a AND operation to call this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1646, __PRETTY_FUNCTION__))
;
1647
1648 EVT VT = N->getValueType(0);
1649
1650 // Here we can test the type of VT and return false when the type does not
1651 // match, but since it is done prior to that call in the current context
1652 // we turned that into an assert to avoid redundant code.
1653 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1654, __PRETTY_FUNCTION__))
1654 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1654, __PRETTY_FUNCTION__))
;
1655
1656 // FIXME: simplify-demanded-bits in DAGCombine will probably have
1657 // changed the AND node to a 32-bit mask operation. We'll have to
1658 // undo that as part of the transform here if we want to catch all
1659 // the opportunities.
1660 // Currently the NumberOfIgnoredLowBits argument helps to recover
1661 // form these situations when matching bigger pattern (bitfield insert).
1662
1663 // For unsigned extracts, check for a shift right and mask
1664 uint64_t AndImm = 0;
1665 if (!isOpcWithIntImmediate(N, ISD::AND, AndImm))
1666 return false;
1667
1668 const SDNode *Op0 = N->getOperand(0).getNode();
1669
1670 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
1671 // simplified. Try to undo that
1672 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
1673
1674 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1675 if (AndImm & (AndImm + 1))
1676 return false;
1677
1678 bool ClampMSB = false;
1679 uint64_t SrlImm = 0;
1680 // Handle the SRL + ANY_EXTEND case.
1681 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
1682 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, SrlImm)) {
1683 // Extend the incoming operand of the SRL to 64-bit.
1684 Opd0 = Widen(CurDAG, Op0->getOperand(0).getOperand(0));
1685 // Make sure to clamp the MSB so that we preserve the semantics of the
1686 // original operations.
1687 ClampMSB = true;
1688 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
1689 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL,
1690 SrlImm)) {
1691 // If the shift result was truncated, we can still combine them.
1692 Opd0 = Op0->getOperand(0).getOperand(0);
1693
1694 // Use the type of SRL node.
1695 VT = Opd0->getValueType(0);
1696 } else if (isOpcWithIntImmediate(Op0, ISD::SRL, SrlImm)) {
1697 Opd0 = Op0->getOperand(0);
1698 } else if (BiggerPattern) {
1699 // Let's pretend a 0 shift right has been performed.
1700 // The resulting code will be at least as good as the original one
1701 // plus it may expose more opportunities for bitfield insert pattern.
1702 // FIXME: Currently we limit this to the bigger pattern, because
1703 // some optimizations expect AND and not UBFM.
1704 Opd0 = N->getOperand(0);
1705 } else
1706 return false;
1707
1708 // Bail out on large immediates. This happens when no proper
1709 // combining/constant folding was performed.
1710 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
1711 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1712 (dbgs() << Ndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1713 << ": Found large shift immediate, this should not happen\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
;
1714 return false;
1715 }
1716
1717 LSB = SrlImm;
1718 MSB = SrlImm + (VT == MVT::i32 ? countTrailingOnes<uint32_t>(AndImm)
1719 : countTrailingOnes<uint64_t>(AndImm)) -
1720 1;
1721 if (ClampMSB)
1722 // Since we're moving the extend before the right shift operation, we need
1723 // to clamp the MSB to make sure we don't shift in undefined bits instead of
1724 // the zeros which would get shifted in with the original right shift
1725 // operation.
1726 MSB = MSB > 31 ? 31 : MSB;
1727
1728 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
1729 return true;
1730}
1731
1732static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
1733 SDValue &Opd0, unsigned &Immr,
1734 unsigned &Imms) {
1735 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG)((N->getOpcode() == ISD::SIGN_EXTEND_INREG) ? static_cast<
void> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND_INREG"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1735, __PRETTY_FUNCTION__))
;
1736
1737 EVT VT = N->getValueType(0);
1738 unsigned BitWidth = VT.getSizeInBits();
1739 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1740, __PRETTY_FUNCTION__))
1740 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1740, __PRETTY_FUNCTION__))
;
1741
1742 SDValue Op = N->getOperand(0);
1743 if (Op->getOpcode() == ISD::TRUNCATE) {
1744 Op = Op->getOperand(0);
1745 VT = Op->getValueType(0);
1746 BitWidth = VT.getSizeInBits();
1747 }
1748
1749 uint64_t ShiftImm;
1750 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SRL, ShiftImm) &&
1751 !isOpcWithIntImmediate(Op.getNode(), ISD::SRA, ShiftImm))
1752 return false;
1753
1754 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
1755 if (ShiftImm + Width > BitWidth)
1756 return false;
1757
1758 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
1759 Opd0 = Op.getOperand(0);
1760 Immr = ShiftImm;
1761 Imms = ShiftImm + Width - 1;
1762 return true;
1763}
1764
1765static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
1766 SDValue &Opd0, unsigned &LSB,
1767 unsigned &MSB) {
1768 // We are looking for the following pattern which basically extracts several
1769 // continuous bits from the source value and places it from the LSB of the
1770 // destination value, all other bits of the destination value or set to zero:
1771 //
1772 // Value2 = AND Value, MaskImm
1773 // SRL Value2, ShiftImm
1774 //
1775 // with MaskImm >> ShiftImm to search for the bit width.
1776 //
1777 // This gets selected into a single UBFM:
1778 //
1779 // UBFM Value, ShiftImm, BitWide + SrlImm -1
1780 //
1781
1782 if (N->getOpcode() != ISD::SRL)
1783 return false;
1784
1785 uint64_t AndMask = 0;
1786 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, AndMask))
1787 return false;
1788
1789 Opd0 = N->getOperand(0).getOperand(0);
1790
1791 uint64_t SrlImm = 0;
1792 if (!isIntImmediate(N->getOperand(1), SrlImm))
1793 return false;
1794
1795 // Check whether we really have several bits extract here.
1796 unsigned BitWide = 64 - countLeadingOnes(~(AndMask >> SrlImm));
1797 if (BitWide && isMask_64(AndMask >> SrlImm)) {
1798 if (N->getValueType(0) == MVT::i32)
1799 Opc = AArch64::UBFMWri;
1800 else
1801 Opc = AArch64::UBFMXri;
1802
1803 LSB = SrlImm;
1804 MSB = BitWide + SrlImm - 1;
1805 return true;
1806 }
1807
1808 return false;
1809}
1810
1811static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
1812 unsigned &Immr, unsigned &Imms,
1813 bool BiggerPattern) {
1814 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&(((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::
SRL) && "N must be a SHR/SRA operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"N must be a SHR/SRA operation to call this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1815, __PRETTY_FUNCTION__))
1815 "N must be a SHR/SRA operation to call this function")(((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::
SRL) && "N must be a SHR/SRA operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"N must be a SHR/SRA operation to call this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1815, __PRETTY_FUNCTION__))
;
1816
1817 EVT VT = N->getValueType(0);
1818
1819 // Here we can test the type of VT and return false when the type does not
1820 // match, but since it is done prior to that call in the current context
1821 // we turned that into an assert to avoid redundant code.
1822 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1823, __PRETTY_FUNCTION__))
1823 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1823, __PRETTY_FUNCTION__))
;
1824
1825 // Check for AND + SRL doing several bits extract.
1826 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, Immr, Imms))
1827 return true;
1828
1829 // We're looking for a shift of a shift.
1830 uint64_t ShlImm = 0;
1831 uint64_t TruncBits = 0;
1832 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, ShlImm)) {
1833 Opd0 = N->getOperand(0).getOperand(0);
1834 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
1835 N->getOperand(0).getNode()->getOpcode() == ISD::TRUNCATE) {
1836 // We are looking for a shift of truncate. Truncate from i64 to i32 could
1837 // be considered as setting high 32 bits as zero. Our strategy here is to
1838 // always generate 64bit UBFM. This consistency will help the CSE pass
1839 // later find more redundancy.
1840 Opd0 = N->getOperand(0).getOperand(0);
1841 TruncBits = Opd0->getValueType(0).getSizeInBits() - VT.getSizeInBits();
1842 VT = Opd0.getValueType();
1843 assert(VT == MVT::i64 && "the promoted type should be i64")((VT == MVT::i64 && "the promoted type should be i64"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::i64 && \"the promoted type should be i64\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1843, __PRETTY_FUNCTION__))
;
1844 } else if (BiggerPattern) {
1845 // Let's pretend a 0 shift left has been performed.
1846 // FIXME: Currently we limit this to the bigger pattern case,
1847 // because some optimizations expect AND and not UBFM
1848 Opd0 = N->getOperand(0);
1849 } else
1850 return false;
1851
1852 // Missing combines/constant folding may have left us with strange
1853 // constants.
1854 if (ShlImm >= VT.getSizeInBits()) {
1855 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1856 (dbgs() << Ndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1857 << ": Found large shift immediate, this should not happen\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
;
1858 return false;
1859 }
1860
1861 uint64_t SrlImm = 0;
1862 if (!isIntImmediate(N->getOperand(1), SrlImm))
1863 return false;
1864
1865 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&((SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
"bad amount in shift node!") ? static_cast<void> (0) :
__assert_fail ("SrlImm > 0 && SrlImm < VT.getSizeInBits() && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1866, __PRETTY_FUNCTION__))
1866 "bad amount in shift node!")((SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
"bad amount in shift node!") ? static_cast<void> (0) :
__assert_fail ("SrlImm > 0 && SrlImm < VT.getSizeInBits() && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1866, __PRETTY_FUNCTION__))
;
1867 int immr = SrlImm - ShlImm;
1868 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
1869 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
1870 // SRA requires a signed extraction
1871 if (VT == MVT::i32)
1872 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
1873 else
1874 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
1875 return true;
1876}
1877
1878bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
1879 assert(N->getOpcode() == ISD::SIGN_EXTEND)((N->getOpcode() == ISD::SIGN_EXTEND) ? static_cast<void
> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1879, __PRETTY_FUNCTION__))
;
1880
1881 EVT VT = N->getValueType(0);
1882 EVT NarrowVT = N->getOperand(0)->getValueType(0);
1883 if (VT != MVT::i64 || NarrowVT != MVT::i32)
1884 return false;
1885
1886 uint64_t ShiftImm;
1887 SDValue Op = N->getOperand(0);
1888 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SRA, ShiftImm))
1889 return false;
1890
1891 SDLoc dl(N);
1892 // Extend the incoming operand of the shift to 64-bits.
1893 SDValue Opd0 = Widen(CurDAG, Op.getOperand(0));
1894 unsigned Immr = ShiftImm;
1895 unsigned Imms = NarrowVT.getSizeInBits() - 1;
1896 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1897 CurDAG->getTargetConstant(Imms, dl, VT)};
1898 CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops);
1899 return true;
1900}
1901
1902/// Try to form fcvtl2 instructions from a floating-point extend of a high-half
1903/// extract of a subvector.
1904bool AArch64DAGToDAGISel::tryHighFPExt(SDNode *N) {
1905 assert(N->getOpcode() == ISD::FP_EXTEND)((N->getOpcode() == ISD::FP_EXTEND) ? static_cast<void>
(0) : __assert_fail ("N->getOpcode() == ISD::FP_EXTEND", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1905, __PRETTY_FUNCTION__))
;
1906
1907 // There are 2 forms of fcvtl2 - extend to double or extend to float.
1908 SDValue Extract = N->getOperand(0);
1909 EVT VT = N->getValueType(0);
1910 EVT NarrowVT = Extract.getValueType();
1911 if ((VT != MVT::v2f64 || NarrowVT != MVT::v2f32) &&
1912 (VT != MVT::v4f32 || NarrowVT != MVT::v4f16))
1913 return false;
1914
1915 // Optionally look past a bitcast.
1916 Extract = peekThroughBitcasts(Extract);
1917 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
1918 return false;
1919
1920 // Match extract from start of high half index.
1921 // Example: v8i16 -> v4i16 means the extract must begin at index 4.
1922 unsigned ExtractIndex = Extract.getConstantOperandVal(1);
1923 if (ExtractIndex != Extract.getValueType().getVectorNumElements())
1924 return false;
1925
1926 auto Opcode = VT == MVT::v2f64 ? AArch64::FCVTLv4i32 : AArch64::FCVTLv8i16;
1927 CurDAG->SelectNodeTo(N, Opcode, VT, Extract.getOperand(0));
1928 return true;
1929}
1930
1931static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
1932 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
1933 unsigned NumberOfIgnoredLowBits = 0,
1934 bool BiggerPattern = false) {
1935 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
1936 return false;
1937
1938 switch (N->getOpcode()) {
1939 default:
1940 if (!N->isMachineOpcode())
1941 return false;
1942 break;
1943 case ISD::AND:
1944 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, Immr, Imms,
1945 NumberOfIgnoredLowBits, BiggerPattern);
1946 case ISD::SRL:
1947 case ISD::SRA:
1948 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
1949
1950 case ISD::SIGN_EXTEND_INREG:
1951 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
1952 }
1953
1954 unsigned NOpc = N->getMachineOpcode();
1955 switch (NOpc) {
1956 default:
1957 return false;
1958 case AArch64::SBFMWri:
1959 case AArch64::UBFMWri:
1960 case AArch64::SBFMXri:
1961 case AArch64::UBFMXri:
1962 Opc = NOpc;
1963 Opd0 = N->getOperand(0);
1964 Immr = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
1965 Imms = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
1966 return true;
1967 }
1968 // Unreachable
1969 return false;
1970}
1971
1972bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
1973 unsigned Opc, Immr, Imms;
1974 SDValue Opd0;
1975 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
1976 return false;
1977
1978 EVT VT = N->getValueType(0);
1979 SDLoc dl(N);
1980
1981 // If the bit extract operation is 64bit but the original type is 32bit, we
1982 // need to add one EXTRACT_SUBREG.
1983 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
1984 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
1985 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
1986
1987 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
1988 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1989 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
1990 MVT::i32, SDValue(BFM, 0), SubReg));
1991 return true;
1992 }
1993
1994 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1995 CurDAG->getTargetConstant(Imms, dl, VT)};
1996 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
1997 return true;
1998}
1999
2000/// Does DstMask form a complementary pair with the mask provided by
2001/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
2002/// this asks whether DstMask zeroes precisely those bits that will be set by
2003/// the other half.
2004static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
2005 unsigned NumberOfIgnoredHighBits, EVT VT) {
2006 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "i32 or i64 mask type expected!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"i32 or i64 mask type expected!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2007, __PRETTY_FUNCTION__))
2007 "i32 or i64 mask type expected!")(((VT == MVT::i32 || VT == MVT::i64) && "i32 or i64 mask type expected!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"i32 or i64 mask type expected!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2007, __PRETTY_FUNCTION__))
;
2008 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
2009
2010 APInt SignificantDstMask = APInt(BitWidth, DstMask);
2011 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(BitWidth);
2012
2013 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
2014 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnesValue();
2015}
2016
2017// Look for bits that will be useful for later uses.
2018// A bit is consider useless as soon as it is dropped and never used
2019// before it as been dropped.
2020// E.g., looking for useful bit of x
2021// 1. y = x & 0x7
2022// 2. z = y >> 2
2023// After #1, x useful bits are 0x7, then the useful bits of x, live through
2024// y.
2025// After #2, the useful bits of x are 0x4.
2026// However, if x is used on an unpredicatable instruction, then all its bits
2027// are useful.
2028// E.g.
2029// 1. y = x & 0x7
2030// 2. z = y >> 2
2031// 3. str x, [@x]
2032static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
2033
2034static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
2035 unsigned Depth) {
2036 uint64_t Imm =
2037 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2038 Imm = AArch64_AM::decodeLogicalImmediate(Imm, UsefulBits.getBitWidth());
2039 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
2040 getUsefulBits(Op, UsefulBits, Depth + 1);
2041}
2042
2043static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
2044 uint64_t Imm, uint64_t MSB,
2045 unsigned Depth) {
2046 // inherit the bitwidth value
2047 APInt OpUsefulBits(UsefulBits);
2048 OpUsefulBits = 1;
2049
2050 if (MSB >= Imm) {
2051 OpUsefulBits <<= MSB - Imm + 1;
2052 --OpUsefulBits;
2053 // The interesting part will be in the lower part of the result
2054 getUsefulBits(Op, OpUsefulBits, Depth + 1);
2055 // The interesting part was starting at Imm in the argument
2056 OpUsefulBits <<= Imm;
2057 } else {
2058 OpUsefulBits <<= MSB + 1;
2059 --OpUsefulBits;
2060 // The interesting part will be shifted in the result
2061 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
2062 getUsefulBits(Op, OpUsefulBits, Depth + 1);
2063 // The interesting part was at zero in the argument
2064 OpUsefulBits.lshrInPlace(OpUsefulBits.getBitWidth() - Imm);
2065 }
2066
2067 UsefulBits &= OpUsefulBits;
2068}
2069
2070static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
2071 unsigned Depth) {
2072 uint64_t Imm =
2073 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
2074 uint64_t MSB =
2075 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2076
2077 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
2078}
2079
2080static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
2081 unsigned Depth) {
2082 uint64_t ShiftTypeAndValue =
2083 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2084 APInt Mask(UsefulBits);
2085 Mask.clearAllBits();
2086 Mask.flipAllBits();
2087
2088 if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSL) {
2089 // Shift Left
2090 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
2091 Mask <<= ShiftAmt;
2092 getUsefulBits(Op, Mask, Depth + 1);
2093 Mask.lshrInPlace(ShiftAmt);
2094 } else if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSR) {
2095 // Shift Right
2096 // We do not handle AArch64_AM::ASR, because the sign will change the
2097 // number of useful bits
2098 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
2099 Mask.lshrInPlace(ShiftAmt);
2100 getUsefulBits(Op, Mask, Depth + 1);
2101 Mask <<= ShiftAmt;
2102 } else
2103 return;
2104
2105 UsefulBits &= Mask;
2106}
2107
2108static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
2109 unsigned Depth) {
2110 uint64_t Imm =
2111 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
2112 uint64_t MSB =
2113 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue();
2114
2115 APInt OpUsefulBits(UsefulBits);
2116 OpUsefulBits = 1;
2117
2118 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
2119 ResultUsefulBits.flipAllBits();
2120 APInt Mask(UsefulBits.getBitWidth(), 0);
2121
2122 getUsefulBits(Op, ResultUsefulBits, Depth + 1);
2123
2124 if (MSB >= Imm) {
2125 // The instruction is a BFXIL.
2126 uint64_t Width = MSB - Imm + 1;
2127 uint64_t LSB = Imm;
2128
2129 OpUsefulBits <<= Width;
2130 --OpUsefulBits;
2131
2132 if (Op.getOperand(1) == Orig) {
2133 // Copy the low bits from the result to bits starting from LSB.
2134 Mask = ResultUsefulBits & OpUsefulBits;
2135 Mask <<= LSB;
2136 }
2137
2138 if (Op.getOperand(0) == Orig)
2139 // Bits starting from LSB in the input contribute to the result.
2140 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2141 } else {
2142 // The instruction is a BFI.
2143 uint64_t Width = MSB + 1;
2144 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
2145
2146 OpUsefulBits <<= Width;
2147 --OpUsefulBits;
2148 OpUsefulBits <<= LSB;
2149
2150 if (Op.getOperand(1) == Orig) {
2151 // Copy the bits from the result to the zero bits.
2152 Mask = ResultUsefulBits & OpUsefulBits;
2153 Mask.lshrInPlace(LSB);
2154 }
2155
2156 if (Op.getOperand(0) == Orig)
2157 Mask |= (ResultUsefulBits & ~OpUsefulBits);
2158 }
2159
2160 UsefulBits &= Mask;
2161}
2162
2163static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
2164 SDValue Orig, unsigned Depth) {
2165
2166 // Users of this node should have already been instruction selected
2167 // FIXME: Can we turn that into an assert?
2168 if (!UserNode->isMachineOpcode())
2169 return;
2170
2171 switch (UserNode->getMachineOpcode()) {
2172 default:
2173 return;
2174 case AArch64::ANDSWri:
2175 case AArch64::ANDSXri:
2176 case AArch64::ANDWri:
2177 case AArch64::ANDXri:
2178 // We increment Depth only when we call the getUsefulBits
2179 return getUsefulBitsFromAndWithImmediate(SDValue(UserNode, 0), UsefulBits,
2180 Depth);
2181 case AArch64::UBFMWri:
2182 case AArch64::UBFMXri:
2183 return getUsefulBitsFromUBFM(SDValue(UserNode, 0), UsefulBits, Depth);
2184
2185 case AArch64::ORRWrs:
2186 case AArch64::ORRXrs:
2187 if (UserNode->getOperand(1) != Orig)
2188 return;
2189 return getUsefulBitsFromOrWithShiftedReg(SDValue(UserNode, 0), UsefulBits,
2190 Depth);
2191 case AArch64::BFMWri:
2192 case AArch64::BFMXri:
2193 return getUsefulBitsFromBFM(SDValue(UserNode, 0), Orig, UsefulBits, Depth);
2194
2195 case AArch64::STRBBui:
2196 case AArch64::STURBBi:
2197 if (UserNode->getOperand(0) != Orig)
2198 return;
2199 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
2200 return;
2201
2202 case AArch64::STRHHui:
2203 case AArch64::STURHHi:
2204 if (UserNode->getOperand(0) != Orig)
2205 return;
2206 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
2207 return;
2208 }
2209}
2210
2211static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
2212 if (Depth >= SelectionDAG::MaxRecursionDepth)
2213 return;
2214 // Initialize UsefulBits
2215 if (!Depth) {
2216 unsigned Bitwidth = Op.getScalarValueSizeInBits();
2217 // At the beginning, assume every produced bits is useful
2218 UsefulBits = APInt(Bitwidth, 0);
2219 UsefulBits.flipAllBits();
2220 }
2221 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
2222
2223 for (SDNode *Node : Op.getNode()->uses()) {
2224 // A use cannot produce useful bits
2225 APInt UsefulBitsForUse = APInt(UsefulBits);
2226 getUsefulBitsForUse(Node, UsefulBitsForUse, Op, Depth);
2227 UsersUsefulBits |= UsefulBitsForUse;
2228 }
2229 // UsefulBits contains the produced bits that are meaningful for the
2230 // current definition, thus a user cannot make a bit meaningful at
2231 // this point
2232 UsefulBits &= UsersUsefulBits;
2233}
2234
2235/// Create a machine node performing a notional SHL of Op by ShlAmount. If
2236/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
2237/// 0, return Op unchanged.
2238static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
2239 if (ShlAmount == 0)
2240 return Op;
2241
2242 EVT VT = Op.getValueType();
2243 SDLoc dl(Op);
2244 unsigned BitWidth = VT.getSizeInBits();
2245 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2246
2247 SDNode *ShiftNode;
2248 if (ShlAmount > 0) {
2249 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
2250 ShiftNode = CurDAG->getMachineNode(
2251 UBFMOpc, dl, VT, Op,
2252 CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT),
2253 CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT));
2254 } else {
2255 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
2256 assert(ShlAmount < 0 && "expected right shift")((ShlAmount < 0 && "expected right shift") ? static_cast
<void> (0) : __assert_fail ("ShlAmount < 0 && \"expected right shift\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2256, __PRETTY_FUNCTION__))
;
2257 int ShrAmount = -ShlAmount;
2258 ShiftNode = CurDAG->getMachineNode(
2259 UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT),
2260 CurDAG->getTargetConstant(BitWidth - 1, dl, VT));
2261 }
2262
2263 return SDValue(ShiftNode, 0);
2264}
2265
2266/// Does this tree qualify as an attempt to move a bitfield into position,
2267/// essentially "(and (shl VAL, N), Mask)".
2268static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
2269 bool BiggerPattern,
2270 SDValue &Src, int &ShiftAmount,
2271 int &MaskWidth) {
2272 EVT VT = Op.getValueType();
2273 unsigned BitWidth = VT.getSizeInBits();
2274 (void)BitWidth;
2275 assert(BitWidth == 32 || BitWidth == 64)((BitWidth == 32 || BitWidth == 64) ? static_cast<void>
(0) : __assert_fail ("BitWidth == 32 || BitWidth == 64", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2275, __PRETTY_FUNCTION__))
;
2276
2277 KnownBits Known = CurDAG->computeKnownBits(Op);
2278
2279 // Non-zero in the sense that they're not provably zero, which is the key
2280 // point if we want to use this value
2281 uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2282
2283 // Discard a constant AND mask if present. It's safe because the node will
2284 // already have been factored into the computeKnownBits calculation above.
2285 uint64_t AndImm;
2286 if (isOpcWithIntImmediate(Op.getNode(), ISD::AND, AndImm)) {
2287 assert((~APInt(BitWidth, AndImm) & ~Known.Zero) == 0)(((~APInt(BitWidth, AndImm) & ~Known.Zero) == 0) ? static_cast
<void> (0) : __assert_fail ("(~APInt(BitWidth, AndImm) & ~Known.Zero) == 0"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2287, __PRETTY_FUNCTION__))
;
2288 Op = Op.getOperand(0);
2289 }
2290
2291 // Don't match if the SHL has more than one use, since then we'll end up
2292 // generating SHL+UBFIZ instead of just keeping SHL+AND.
2293 if (!BiggerPattern && !Op.hasOneUse())
2294 return false;
2295
2296 uint64_t ShlImm;
2297 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SHL, ShlImm))
2298 return false;
2299 Op = Op.getOperand(0);
2300
2301 if (!isShiftedMask_64(NonZeroBits))
2302 return false;
2303
2304 ShiftAmount = countTrailingZeros(NonZeroBits);
2305 MaskWidth = countTrailingOnes(NonZeroBits >> ShiftAmount);
2306
2307 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
2308 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
2309 // amount. BiggerPattern is true when this pattern is being matched for BFI,
2310 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
2311 // which case it is not profitable to insert an extra shift.
2312 if (ShlImm - ShiftAmount != 0 && !BiggerPattern)
2313 return false;
2314 Src = getLeftShift(CurDAG, Op, ShlImm - ShiftAmount);
2315
2316 return true;
2317}
2318
2319static bool isShiftedMask(uint64_t Mask, EVT VT) {
2320 assert(VT == MVT::i32 || VT == MVT::i64)((VT == MVT::i32 || VT == MVT::i64) ? static_cast<void>
(0) : __assert_fail ("VT == MVT::i32 || VT == MVT::i64", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2320, __PRETTY_FUNCTION__))
;
2321 if (VT == MVT::i32)
2322 return isShiftedMask_32(Mask);
2323 return isShiftedMask_64(Mask);
2324}
2325
2326// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
2327// inserted only sets known zero bits.
2328static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
2329 assert(N->getOpcode() == ISD::OR && "Expect a OR operation")((N->getOpcode() == ISD::OR && "Expect a OR operation"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Expect a OR operation\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2329, __PRETTY_FUNCTION__))
;
2330
2331 EVT VT = N->getValueType(0);
2332 if (VT != MVT::i32 && VT != MVT::i64)
2333 return false;
2334
2335 unsigned BitWidth = VT.getSizeInBits();
2336
2337 uint64_t OrImm;
2338 if (!isOpcWithIntImmediate(N, ISD::OR, OrImm))
2339 return false;
2340
2341 // Skip this transformation if the ORR immediate can be encoded in the ORR.
2342 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
2343 // performance neutral.
2344 if (AArch64_AM::isLogicalImmediate(OrImm, BitWidth))
2345 return false;
2346
2347 uint64_t MaskImm;
2348 SDValue And = N->getOperand(0);
2349 // Must be a single use AND with an immediate operand.
2350 if (!And.hasOneUse() ||
2351 !isOpcWithIntImmediate(And.getNode(), ISD::AND, MaskImm))
2352 return false;
2353
2354 // Compute the Known Zero for the AND as this allows us to catch more general
2355 // cases than just looking for AND with imm.
2356 KnownBits Known = CurDAG->computeKnownBits(And);
2357
2358 // Non-zero in the sense that they're not provably zero, which is the key
2359 // point if we want to use this value.
2360 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
2361
2362 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
2363 if (!isShiftedMask(Known.Zero.getZExtValue(), VT))
2364 return false;
2365
2366 // The bits being inserted must only set those bits that are known to be zero.
2367 if ((OrImm & NotKnownZero) != 0) {
2368 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
2369 // currently handle this case.
2370 return false;
2371 }
2372
2373 // BFI/BFXIL dst, src, #lsb, #width.
2374 int LSB = countTrailingOnes(NotKnownZero);
2375 int Width = BitWidth - APInt(BitWidth, NotKnownZero).countPopulation();
2376
2377 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
2378 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2379 unsigned ImmS = Width - 1;
2380
2381 // If we're creating a BFI instruction avoid cases where we need more
2382 // instructions to materialize the BFI constant as compared to the original
2383 // ORR. A BFXIL will use the same constant as the original ORR, so the code
2384 // should be no worse in this case.
2385 bool IsBFI = LSB != 0;
2386 uint64_t BFIImm = OrImm >> LSB;
2387 if (IsBFI && !AArch64_AM::isLogicalImmediate(BFIImm, BitWidth)) {
2388 // We have a BFI instruction and we know the constant can't be materialized
2389 // with a ORR-immediate with the zero register.
2390 unsigned OrChunks = 0, BFIChunks = 0;
2391 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
2392 if (((OrImm >> Shift) & 0xFFFF) != 0)
2393 ++OrChunks;
2394 if (((BFIImm >> Shift) & 0xFFFF) != 0)
2395 ++BFIChunks;
2396 }
2397 if (BFIChunks > OrChunks)
2398 return false;
2399 }
2400
2401 // Materialize the constant to be inserted.
2402 SDLoc DL(N);
2403 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
2404 SDNode *MOVI = CurDAG->getMachineNode(
2405 MOVIOpc, DL, VT, CurDAG->getTargetConstant(BFIImm, DL, VT));
2406
2407 // Create the BFI/BFXIL instruction.
2408 SDValue Ops[] = {And.getOperand(0), SDValue(MOVI, 0),
2409 CurDAG->getTargetConstant(ImmR, DL, VT),
2410 CurDAG->getTargetConstant(ImmS, DL, VT)};
2411 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2412 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2413 return true;
2414}
2415
2416static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
2417 SelectionDAG *CurDAG) {
2418 assert(N->getOpcode() == ISD::OR && "Expect a OR operation")((N->getOpcode() == ISD::OR && "Expect a OR operation"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Expect a OR operation\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2418, __PRETTY_FUNCTION__))
;
2419
2420 EVT VT = N->getValueType(0);
2421 if (VT != MVT::i32 && VT != MVT::i64)
2422 return false;
2423
2424 unsigned BitWidth = VT.getSizeInBits();
2425
2426 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
2427 // have the expected shape. Try to undo that.
2428
2429 unsigned NumberOfIgnoredLowBits = UsefulBits.countTrailingZeros();
2430 unsigned NumberOfIgnoredHighBits = UsefulBits.countLeadingZeros();
2431
2432 // Given a OR operation, check if we have the following pattern
2433 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
2434 // isBitfieldExtractOp)
2435 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
2436 // countTrailingZeros(mask2) == imm2 - imm + 1
2437 // f = d | c
2438 // if yes, replace the OR instruction with:
2439 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
2440
2441 // OR is commutative, check all combinations of operand order and values of
2442 // BiggerPattern, i.e.
2443 // Opd0, Opd1, BiggerPattern=false
2444 // Opd1, Opd0, BiggerPattern=false
2445 // Opd0, Opd1, BiggerPattern=true
2446 // Opd1, Opd0, BiggerPattern=true
2447 // Several of these combinations may match, so check with BiggerPattern=false
2448 // first since that will produce better results by matching more instructions
2449 // and/or inserting fewer extra instructions.
2450 for (int I = 0; I < 4; ++I) {
2451
2452 SDValue Dst, Src;
2453 unsigned ImmR, ImmS;
2454 bool BiggerPattern = I / 2;
2455 SDValue OrOpd0Val = N->getOperand(I % 2);
2456 SDNode *OrOpd0 = OrOpd0Val.getNode();
2457 SDValue OrOpd1Val = N->getOperand((I + 1) % 2);
2458 SDNode *OrOpd1 = OrOpd1Val.getNode();
2459
2460 unsigned BFXOpc;
2461 int DstLSB, Width;
2462 if (isBitfieldExtractOp(CurDAG, OrOpd0, BFXOpc, Src, ImmR, ImmS,
2463 NumberOfIgnoredLowBits, BiggerPattern)) {
2464 // Check that the returned opcode is compatible with the pattern,
2465 // i.e., same type and zero extended (U and not S)
2466 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
2467 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
2468 continue;
2469
2470 // Compute the width of the bitfield insertion
2471 DstLSB = 0;
2472 Width = ImmS - ImmR + 1;
2473 // FIXME: This constraint is to catch bitfield insertion we may
2474 // want to widen the pattern if we want to grab general bitfied
2475 // move case
2476 if (Width <= 0)
2477 continue;
2478
2479 // If the mask on the insertee is correct, we have a BFXIL operation. We
2480 // can share the ImmR and ImmS values from the already-computed UBFM.
2481 } else if (isBitfieldPositioningOp(CurDAG, OrOpd0Val,
2482 BiggerPattern,
2483 Src, DstLSB, Width)) {
2484 ImmR = (BitWidth - DstLSB) % BitWidth;
2485 ImmS = Width - 1;
2486 } else
2487 continue;
2488
2489 // Check the second part of the pattern
2490 EVT VT = OrOpd1Val.getValueType();
2491 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand")(((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"unexpected OR operand\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2491, __PRETTY_FUNCTION__))
;
2492
2493 // Compute the Known Zero for the candidate of the first operand.
2494 // This allows to catch more general case than just looking for
2495 // AND with imm. Indeed, simplify-demanded-bits may have removed
2496 // the AND instruction because it proves it was useless.
2497 KnownBits Known = CurDAG->computeKnownBits(OrOpd1Val);
2498
2499 // Check if there is enough room for the second operand to appear
2500 // in the first one
2501 APInt BitsToBeInserted =
2502 APInt::getBitsSet(Known.getBitWidth(), DstLSB, DstLSB + Width);
2503
2504 if ((BitsToBeInserted & ~Known.Zero) != 0)
2505 continue;
2506
2507 // Set the first operand
2508 uint64_t Imm;
2509 if (isOpcWithIntImmediate(OrOpd1, ISD::AND, Imm) &&
2510 isBitfieldDstMask(Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
2511 // In that case, we can eliminate the AND
2512 Dst = OrOpd1->getOperand(0);
2513 else
2514 // Maybe the AND has been removed by simplify-demanded-bits
2515 // or is useful because it discards more bits
2516 Dst = OrOpd1Val;
2517
2518 // both parts match
2519 SDLoc DL(N);
2520 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT),
2521 CurDAG->getTargetConstant(ImmS, DL, VT)};
2522 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2523 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2524 return true;
2525 }
2526
2527 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
2528 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
2529 // mask (e.g., 0x000ffff0).
2530 uint64_t Mask0Imm, Mask1Imm;
2531 SDValue And0 = N->getOperand(0);
2532 SDValue And1 = N->getOperand(1);
2533 if (And0.hasOneUse() && And1.hasOneUse() &&
2534 isOpcWithIntImmediate(And0.getNode(), ISD::AND, Mask0Imm) &&
2535 isOpcWithIntImmediate(And1.getNode(), ISD::AND, Mask1Imm) &&
2536 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
2537 (isShiftedMask(Mask0Imm, VT) || isShiftedMask(Mask1Imm, VT))) {
2538
2539 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
2540 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
2541 // bits to be inserted.
2542 if (isShiftedMask(Mask0Imm, VT)) {
2543 std::swap(And0, And1);
2544 std::swap(Mask0Imm, Mask1Imm);
2545 }
2546
2547 SDValue Src = And1->getOperand(0);
2548 SDValue Dst = And0->getOperand(0);
2549 unsigned LSB = countTrailingZeros(Mask1Imm);
2550 int Width = BitWidth - APInt(BitWidth, Mask0Imm).countPopulation();
2551
2552 // The BFXIL inserts the low-order bits from a source register, so right
2553 // shift the needed bits into place.
2554 SDLoc DL(N);
2555 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2556 SDNode *LSR = CurDAG->getMachineNode(
2557 ShiftOpc, DL, VT, Src, CurDAG->getTargetConstant(LSB, DL, VT),
2558 CurDAG->getTargetConstant(BitWidth - 1, DL, VT));
2559
2560 // BFXIL is an alias of BFM, so translate to BFM operands.
2561 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2562 unsigned ImmS = Width - 1;
2563
2564 // Create the BFXIL instruction.
2565 SDValue Ops[] = {Dst, SDValue(LSR, 0),
2566 CurDAG->getTargetConstant(ImmR, DL, VT),
2567 CurDAG->getTargetConstant(ImmS, DL, VT)};
2568 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2569 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2570 return true;
2571 }
2572
2573 return false;
2574}
2575
2576bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
2577 if (N->getOpcode() != ISD::OR)
2578 return false;
2579
2580 APInt NUsefulBits;
2581 getUsefulBits(SDValue(N, 0), NUsefulBits);
2582
2583 // If all bits are not useful, just return UNDEF.
2584 if (!NUsefulBits) {
2585 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2586 return true;
2587 }
2588
2589 if (tryBitfieldInsertOpFromOr(N, NUsefulBits, CurDAG))
2590 return true;
2591
2592 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
2593}
2594
2595/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
2596/// equivalent of a left shift by a constant amount followed by an and masking
2597/// out a contiguous set of bits.
2598bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
2599 if (N->getOpcode() != ISD::AND)
2600 return false;
2601
2602 EVT VT = N->getValueType(0);
2603 if (VT != MVT::i32 && VT != MVT::i64)
2604 return false;
2605
2606 SDValue Op0;
2607 int DstLSB, Width;
2608 if (!isBitfieldPositioningOp(CurDAG, SDValue(N, 0), /*BiggerPattern=*/false,
2609 Op0, DstLSB, Width))
2610 return false;
2611
2612 // ImmR is the rotate right amount.
2613 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
2614 // ImmS is the most significant bit of the source to be moved.
2615 unsigned ImmS = Width - 1;
2616
2617 SDLoc DL(N);
2618 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
2619 CurDAG->getTargetConstant(ImmS, DL, VT)};
2620 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2621 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2622 return true;
2623}
2624
2625/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
2626/// variable shift/rotate instructions.
2627bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
2628 EVT VT = N->getValueType(0);
2629
2630 unsigned Opc;
2631 switch (N->getOpcode()) {
2632 case ISD::ROTR:
2633 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
2634 break;
2635 case ISD::SHL:
2636 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
2637 break;
2638 case ISD::SRL:
2639 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
2640 break;
2641 case ISD::SRA:
2642 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
2643 break;
2644 default:
2645 return false;
2646 }
2647
2648 uint64_t Size;
2649 uint64_t Bits;
2650 if (VT == MVT::i32) {
2651 Bits = 5;
2652 Size = 32;
2653 } else if (VT == MVT::i64) {
2654 Bits = 6;
2655 Size = 64;
2656 } else
2657 return false;
2658
2659 SDValue ShiftAmt = N->getOperand(1);
2660 SDLoc DL(N);
2661 SDValue NewShiftAmt;
2662
2663 // Skip over an extend of the shift amount.
2664 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
2665 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
2666 ShiftAmt = ShiftAmt->getOperand(0);
2667
2668 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
2669 SDValue Add0 = ShiftAmt->getOperand(0);
2670 SDValue Add1 = ShiftAmt->getOperand(1);
2671 uint64_t Add0Imm;
2672 uint64_t Add1Imm;
2673 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
2674 // to avoid the ADD/SUB.
2675 if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0))
2676 NewShiftAmt = Add0;
2677 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2678 // generate a NEG instead of a SUB of a constant.
2679 else if (ShiftAmt->getOpcode() == ISD::SUB &&
2680 isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 &&
2681 (Add0Imm % Size == 0)) {
2682 unsigned NegOpc;
2683 unsigned ZeroReg;
2684 EVT SubVT = ShiftAmt->getValueType(0);
2685 if (SubVT == MVT::i32) {
2686 NegOpc = AArch64::SUBWrr;
2687 ZeroReg = AArch64::WZR;
2688 } else {
2689 assert(SubVT == MVT::i64)((SubVT == MVT::i64) ? static_cast<void> (0) : __assert_fail
("SubVT == MVT::i64", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2689, __PRETTY_FUNCTION__))
;
2690 NegOpc = AArch64::SUBXrr;
2691 ZeroReg = AArch64::XZR;
2692 }
2693 SDValue Zero =
2694 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
2695 MachineSDNode *Neg =
2696 CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1);
2697 NewShiftAmt = SDValue(Neg, 0);
2698 } else
2699 return false;
2700 } else {
2701 // If the shift amount is masked with an AND, check that the mask covers the
2702 // bits that are implicitly ANDed off by the above opcodes and if so, skip
2703 // the AND.
2704 uint64_t MaskImm;
2705 if (!isOpcWithIntImmediate(ShiftAmt.getNode(), ISD::AND, MaskImm) &&
2706 !isOpcWithIntImmediate(ShiftAmt.getNode(), AArch64ISD::ANDS, MaskImm))
2707 return false;
2708
2709 if (countTrailingOnes(MaskImm) < Bits)
2710 return false;
2711
2712 NewShiftAmt = ShiftAmt->getOperand(0);
2713 }
2714
2715 // Narrow/widen the shift amount to match the size of the shift operation.
2716 if (VT == MVT::i32)
2717 NewShiftAmt = narrowIfNeeded(CurDAG, NewShiftAmt);
2718 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) {
2719 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32);
2720 MachineSDNode *Ext = CurDAG->getMachineNode(
2721 AArch64::SUBREG_TO_REG, DL, VT,
2722 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg);
2723 NewShiftAmt = SDValue(Ext, 0);
2724 }
2725
2726 SDValue Ops[] = {N->getOperand(0), NewShiftAmt};
2727 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2728 return true;
2729}
2730
2731bool
2732AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
2733 unsigned RegWidth) {
2734 APFloat FVal(0.0);
2735 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
2736 FVal = CN->getValueAPF();
2737 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(N)) {
2738 // Some otherwise illegal constants are allowed in this case.
2739 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow ||
2740 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
2741 return false;
2742
2743 ConstantPoolSDNode *CN =
2744 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
2745 FVal = cast<ConstantFP>(CN->getConstVal())->getValueAPF();
2746 } else
2747 return false;
2748
2749 // An FCVT[SU] instruction performs: convertToInt(Val * 2^fbits) where fbits
2750 // is between 1 and 32 for a destination w-register, or 1 and 64 for an
2751 // x-register.
2752 //
2753 // By this stage, we've detected (fp_to_[su]int (fmul Val, THIS_NODE)) so we
2754 // want THIS_NODE to be 2^fbits. This is much easier to deal with using
2755 // integers.
2756 bool IsExact;
2757
2758 // fbits is between 1 and 64 in the worst-case, which means the fmul
2759 // could have 2^64 as an actual operand. Need 65 bits of precision.
2760 APSInt IntVal(65, true);
2761 FVal.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact);
2762
2763 // N.b. isPowerOf2 also checks for > 0.
2764 if (!IsExact || !IntVal.isPowerOf2()) return false;
2765 unsigned FBits = IntVal.logBase2();
2766
2767 // Checks above should have guaranteed that we haven't lost information in
2768 // finding FBits, but it must still be in range.
2769 if (FBits == 0 || FBits > RegWidth) return false;
2770
2771 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
2772 return true;
2773}
2774
2775// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
2776// of the string and obtains the integer values from them and combines these
2777// into a single value to be used in the MRS/MSR instruction.
2778static int getIntOperandFromRegisterString(StringRef RegString) {
2779 SmallVector<StringRef, 5> Fields;
2780 RegString.split(Fields, ':');
2781
2782 if (Fields.size() == 1)
2783 return -1;
2784
2785 assert(Fields.size() == 5((Fields.size() == 5 && "Invalid number of fields in read register string"
) ? static_cast<void> (0) : __assert_fail ("Fields.size() == 5 && \"Invalid number of fields in read register string\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2786, __PRETTY_FUNCTION__))
2786 && "Invalid number of fields in read register string")((Fields.size() == 5 && "Invalid number of fields in read register string"
) ? static_cast<void> (0) : __assert_fail ("Fields.size() == 5 && \"Invalid number of fields in read register string\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2786, __PRETTY_FUNCTION__))
;
2787
2788 SmallVector<int, 5> Ops;
2789 bool AllIntFields = true;
2790
2791 for (StringRef Field : Fields) {
2792 unsigned IntField;
2793 AllIntFields &= !Field.getAsInteger(10, IntField);
2794 Ops.push_back(IntField);
2795 }
2796
2797 assert(AllIntFields &&((AllIntFields && "Unexpected non-integer value in special register string."
) ? static_cast<void> (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2798, __PRETTY_FUNCTION__))
2798 "Unexpected non-integer value in special register string.")((AllIntFields && "Unexpected non-integer value in special register string."
) ? static_cast<void> (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2798, __PRETTY_FUNCTION__))
;
2799
2800 // Need to combine the integer fields of the string into a single value
2801 // based on the bit encoding of MRS/MSR instruction.
2802 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
2803 (Ops[3] << 3) | (Ops[4]);
2804}
2805
2806// Lower the read_register intrinsic to an MRS instruction node if the special
2807// register string argument is either of the form detailed in the ALCE (the
2808// form described in getIntOperandsFromRegsterString) or is a named register
2809// known by the MRS SysReg mapper.
2810bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
2811 const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
2812 const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2813 SDLoc DL(N);
2814
2815 int Reg = getIntOperandFromRegisterString(RegString->getString());
2816 if (Reg != -1) {
2817 ReplaceNode(N, CurDAG->getMachineNode(
2818 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
2819 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2820 N->getOperand(0)));
2821 return true;
2822 }
2823
2824 // Use the sysreg mapper to map the remaining possible strings to the
2825 // value for the register to be used for the instruction operand.
2826 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
2827 if (TheReg && TheReg->Readable &&
2828 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2829 Reg = TheReg->Encoding;
2830 else
2831 Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
2832
2833 if (Reg != -1) {
2834 ReplaceNode(N, CurDAG->getMachineNode(
2835 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
2836 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2837 N->getOperand(0)));
2838 return true;
2839 }
2840
2841 if (RegString->getString() == "pc") {
2842 ReplaceNode(N, CurDAG->getMachineNode(
2843 AArch64::ADR, DL, N->getSimpleValueType(0), MVT::Other,
2844 CurDAG->getTargetConstant(0, DL, MVT::i32),
2845 N->getOperand(0)));
2846 return true;
2847 }
2848
2849 return false;
2850}
2851
2852// Lower the write_register intrinsic to an MSR instruction node if the special
2853// register string argument is either of the form detailed in the ALCE (the
2854// form described in getIntOperandsFromRegsterString) or is a named register
2855// known by the MSR SysReg mapper.
2856bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
2857 const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
2858 const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2859 SDLoc DL(N);
2860
2861 int Reg = getIntOperandFromRegisterString(RegString->getString());
2862 if (Reg != -1) {
2863 ReplaceNode(
2864 N, CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
2865 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2866 N->getOperand(2), N->getOperand(0)));
2867 return true;
2868 }
2869
2870 // Check if the register was one of those allowed as the pstatefield value in
2871 // the MSR (immediate) instruction. To accept the values allowed in the
2872 // pstatefield for the MSR (immediate) instruction, we also require that an
2873 // immediate value has been provided as an argument, we know that this is
2874 // the case as it has been ensured by semantic checking.
2875 auto PMapper = AArch64PState::lookupPStateByName(RegString->getString());
2876 if (PMapper) {
2877 assert (isa<ConstantSDNode>(N->getOperand(2))((isa<ConstantSDNode>(N->getOperand(2)) && "Expected a constant integer expression."
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(2)) && \"Expected a constant integer expression.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2878, __PRETTY_FUNCTION__))
2878 && "Expected a constant integer expression.")((isa<ConstantSDNode>(N->getOperand(2)) && "Expected a constant integer expression."
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(2)) && \"Expected a constant integer expression.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2878, __PRETTY_FUNCTION__))
;
2879 unsigned Reg = PMapper->Encoding;
2880 uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
2881 unsigned State;
2882 if (Reg == AArch64PState::PAN || Reg == AArch64PState::UAO || Reg == AArch64PState::SSBS) {
2883 assert(Immed < 2 && "Bad imm")((Immed < 2 && "Bad imm") ? static_cast<void>
(0) : __assert_fail ("Immed < 2 && \"Bad imm\"", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2883, __PRETTY_FUNCTION__))
;
2884 State = AArch64::MSRpstateImm1;
2885 } else {
2886 assert(Immed < 16 && "Bad imm")((Immed < 16 && "Bad imm") ? static_cast<void>
(0) : __assert_fail ("Immed < 16 && \"Bad imm\"",
"/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2886, __PRETTY_FUNCTION__))
;
2887 State = AArch64::MSRpstateImm4;
2888 }
2889 ReplaceNode(N, CurDAG->getMachineNode(
2890 State, DL, MVT::Other,
2891 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2892 CurDAG->getTargetConstant(Immed, DL, MVT::i16),
2893 N->getOperand(0)));
2894 return true;
2895 }
2896
2897 // Use the sysreg mapper to attempt to map the remaining possible strings
2898 // to the value for the register to be used for the MSR (register)
2899 // instruction operand.
2900 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
2901 if (TheReg && TheReg->Writeable &&
2902 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2903 Reg = TheReg->Encoding;
2904 else
2905 Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
2906 if (Reg != -1) {
2907 ReplaceNode(N, CurDAG->getMachineNode(
2908 AArch64::MSR, DL, MVT::Other,
2909 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2910 N->getOperand(2), N->getOperand(0)));
2911 return true;
2912 }
2913
2914 return false;
2915}
2916
2917/// We've got special pseudo-instructions for these
2918bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2919 unsigned Opcode;
2920 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2921
2922 // Leave IR for LSE if subtarget supports it.
2923 if (Subtarget->hasLSE()) return false;
2924
2925 if (MemTy == MVT::i8)
2926 Opcode = AArch64::CMP_SWAP_8;
2927 else if (MemTy == MVT::i16)
2928 Opcode = AArch64::CMP_SWAP_16;
2929 else if (MemTy == MVT::i32)
2930 Opcode = AArch64::CMP_SWAP_32;
2931 else if (MemTy == MVT::i64)
2932 Opcode = AArch64::CMP_SWAP_64;
2933 else
2934 llvm_unreachable("Unknown AtomicCmpSwap type")::llvm::llvm_unreachable_internal("Unknown AtomicCmpSwap type"
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2934)
;
2935
2936 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
2937 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2938 N->getOperand(0)};
2939 SDNode *CmpSwap = CurDAG->getMachineNode(
2940 Opcode, SDLoc(N),
2941 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
2942
2943 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2944 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2945
2946 ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2947 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2948 CurDAG->RemoveDeadNode(N);
2949
2950 return true;
2951}
2952
2953bool AArch64DAGToDAGISel::SelectSVE8BitLslImm(SDValue N, SDValue &Base,
2954 SDValue &Offset) {
2955 auto C = dyn_cast<ConstantSDNode>(N);
2956 if (!C)
2957 return false;
2958
2959 auto Ty = N->getValueType(0);
2960
2961 int64_t Imm = C->getSExtValue();
2962 SDLoc DL(N);
2963
2964 if ((Imm >= -128) && (Imm <= 127)) {
2965 Base = CurDAG->getTargetConstant(Imm, DL, Ty);
2966 Offset = CurDAG->getTargetConstant(0, DL, Ty);
2967 return true;
2968 }
2969
2970 if (((Imm % 256) == 0) && (Imm >= -32768) && (Imm <= 32512)) {
2971 Base = CurDAG->getTargetConstant(Imm/256, DL, Ty);
2972 Offset = CurDAG->getTargetConstant(8, DL, Ty);
2973 return true;
2974 }
2975
2976 return false;
2977}
2978
2979bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift) {
2980 if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
2981 const int64_t ImmVal = CNode->getZExtValue();
2982 SDLoc DL(N);
2983
2984 switch (VT.SimpleTy) {
2985 case MVT::i8:
2986 if ((ImmVal & 0xFF) == ImmVal) {
2987 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
2988 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
2989 return true;
2990 }
2991 break;
2992 case MVT::i16:
2993 case MVT::i32:
2994 case MVT::i64:
2995 if ((ImmVal & 0xFF) == ImmVal) {
2996 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
2997 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
2998 return true;
2999 } else if ((ImmVal & 0xFF00) == ImmVal) {
3000 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
3001 Imm = CurDAG->getTargetConstant(ImmVal >> 8, DL, MVT::i32);
3002 return true;
3003 }
3004 break;
3005 default:
3006 break;
3007 }
3008 }
3009
3010 return false;
3011}
3012
3013bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
3014 if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
3015 int64_t ImmVal = CNode->getSExtValue();
3016 SDLoc DL(N);
3017 if (ImmVal >= -127 && ImmVal < 127) {
3018 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
3019 return true;
3020 }
3021 }
3022 return false;
3023}
3024
3025bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, SDValue &Imm) {
3026 if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
3027 uint64_t ImmVal = CNode->getSExtValue();
3028 SDLoc DL(N);
3029 ImmVal = ImmVal & 0xFF;
3030 if (ImmVal < 256) {
3031 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
3032 return true;
3033 }
3034 }
3035 return false;
3036}
3037
3038bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm) {
3039 if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
3040 uint64_t ImmVal = CNode->getZExtValue();
3041 SDLoc DL(N);
3042
3043 // Shift mask depending on type size.
3044 switch (VT.SimpleTy) {
3045 case MVT::i8:
3046 ImmVal &= 0xFF;
3047 ImmVal |= ImmVal << 8;
3048 ImmVal |= ImmVal << 16;
3049 ImmVal |= ImmVal << 32;
3050 break;
3051 case MVT::i16:
3052 ImmVal &= 0xFFFF;
3053 ImmVal |= ImmVal << 16;
3054 ImmVal |= ImmVal << 32;
3055 break;
3056 case MVT::i32:
3057 ImmVal &= 0xFFFFFFFF;
3058 ImmVal |= ImmVal << 32;
3059 break;
3060 case MVT::i64:
3061 break;
3062 default:
3063 llvm_unreachable("Unexpected type")::llvm::llvm_unreachable_internal("Unexpected type", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 3063)
;
3064 }
3065
3066 uint64_t encoding;
3067 if (AArch64_AM::processLogicalImmediate(ImmVal, 64, encoding)) {
3068 Imm = CurDAG->getTargetConstant(encoding, DL, MVT::i64);
3069 return true;
3070 }
3071 }
3072 return false;
3073}
3074
3075bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
3076 // tagp(FrameIndex, IRGstack, tag_offset):
3077 // since the offset between FrameIndex and IRGstack is a compile-time
3078 // constant, this can be lowered to a single ADDG instruction.
3079 if (!(isa<FrameIndexSDNode>(N->getOperand(1)))) {
3080 return false;
3081 }
3082
3083 SDValue IRG_SP = N->getOperand(2);
3084 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
3085 cast<ConstantSDNode>(IRG_SP->getOperand(1))->getZExtValue() !=
3086 Intrinsic::aarch64_irg_sp) {
3087 return false;
3088 }
3089
3090 const TargetLowering *TLI = getTargetLowering();
3091 SDLoc DL(N);
3092 int FI = cast<FrameIndexSDNode>(N->getOperand(1))->getIndex();
3093 SDValue FiOp = CurDAG->getTargetFrameIndex(
3094 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3095 int TagOffset = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
3096
3097 SDNode *Out = CurDAG->getMachineNode(
3098 AArch64::TAGPstack, DL, MVT::i64,
3099 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2),
3100 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
3101 ReplaceNode(N, Out);
3102 return true;
3103}
3104
3105void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
3106 assert(isa<ConstantSDNode>(N->getOperand(3)) &&((isa<ConstantSDNode>(N->getOperand(3)) && "llvm.aarch64.tagp third argument must be an immediate"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(3)) && \"llvm.aarch64.tagp third argument must be an immediate\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 3107, __PRETTY_FUNCTION__))
3107 "llvm.aarch64.tagp third argument must be an immediate")((isa<ConstantSDNode>(N->getOperand(3)) && "llvm.aarch64.tagp third argument must be an immediate"
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(3)) && \"llvm.aarch64.tagp third argument must be an immediate\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 3107, __PRETTY_FUNCTION__))
;
3108 if (trySelectStackSlotTagP(N))
3109 return;
3110 // FIXME: above applies in any case when offset between Op1 and Op2 is a
3111 // compile-time constant, not just for stack allocations.
3112
3113 // General case for unrelated pointers in Op1 and Op2.
3114 SDLoc DL(N);
3115 int TagOffset = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
3116 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64,
3117 {N->getOperand(1), N->getOperand(2)});
3118 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64,
3119 {SDValue(N1, 0), N->getOperand(2)});
3120 SDNode *N3 = CurDAG->getMachineNode(
3121 AArch64::ADDG, DL, MVT::i64,
3122 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64),
3123 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
3124 ReplaceNode(N, N3);
3125}
3126
3127void AArch64DAGToDAGISel::Select(SDNode *Node) {
3128 // If we have a custom node, we already have selected!
3129 if (Node->isMachineOpcode()) {
3130 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { errs() << "== "; Node->dump(CurDAG
); errs() << "\n"; } } while (false)
;
3131 Node->setNodeId(-1);
3132 return;
3133 }
3134
3135 // Few custom selection stuff.
3136 EVT VT = Node->getValueType(0);
3137
3138 switch (Node->getOpcode()) {
3139 default:
3140 break;
3141
3142 case ISD::ATOMIC_CMP_SWAP:
3143 if (SelectCMP_SWAP(Node))
3144 return;
3145 break;
3146
3147 case ISD::READ_REGISTER:
3148 if (tryReadRegister(Node))
3149 return;
3150 break;
3151
3152 case ISD::WRITE_REGISTER:
3153 if (tryWriteRegister(Node))
3154 return;
3155 break;
3156
3157 case ISD::ADD:
3158 if (tryMLAV64LaneV128(Node))
3159 return;
3160 break;
3161
3162 case ISD::LOAD: {
3163 // Try to select as an indexed load. Fall through to normal processing
3164 // if we can't.
3165 if (tryIndexedLoad(Node))
3166 return;
3167 break;
3168 }
3169
3170 case ISD::SRL:
3171 case ISD::AND:
3172 case ISD::SRA:
3173 case ISD::SIGN_EXTEND_INREG:
3174 if (tryBitfieldExtractOp(Node))
3175 return;
3176 if (tryBitfieldInsertInZeroOp(Node))
3177 return;
3178 LLVM_FALLTHROUGH[[gnu::fallthrough]];
3179 case ISD::ROTR:
3180 case ISD::SHL:
3181 if (tryShiftAmountMod(Node))
3182 return;
3183 break;
3184
3185 case ISD::SIGN_EXTEND:
3186 if (tryBitfieldExtractOpFromSExt(Node))
3187 return;
3188 break;
3189
3190 case ISD::FP_EXTEND:
3191 if (tryHighFPExt(Node))
3192 return;
3193 break;
3194
3195 case ISD::OR:
3196 if (tryBitfieldInsertOp(Node))
3197 return;
3198 break;
3199
3200 case ISD::Constant: {
3201 // Materialize zero constants as copies from WZR/XZR. This allows
3202 // the coalescer to propagate these into other instructions.
3203 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Node);
3204 if (ConstNode->isNullValue()) {
3205 if (VT == MVT::i32) {
3206 SDValue New = CurDAG->getCopyFromReg(
3207 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32);
3208 ReplaceNode(Node, New.getNode());
3209 return;
3210 } else if (VT == MVT::i64) {
3211 SDValue New = CurDAG->getCopyFromReg(
3212 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64);
3213 ReplaceNode(Node, New.getNode());
3214 return;
3215 }
3216 }
3217 break;
3218 }
3219
3220 case ISD::FrameIndex: {
3221 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
3222 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
3223 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
3224 const TargetLowering *TLI = getTargetLowering();
3225 SDValue TFI = CurDAG->getTargetFrameIndex(
3226 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3227 SDLoc DL(Node);
3228 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
3229 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
3230 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops);
3231 return;
3232 }
3233 case ISD::INTRINSIC_W_CHAIN: {
3234 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
3235 switch (IntNo) {
3236 default:
3237 break;
3238 case Intrinsic::aarch64_ldaxp:
3239 case Intrinsic::aarch64_ldxp: {
3240 unsigned Op =
3241 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
3242 SDValue MemAddr = Node->getOperand(2);
3243 SDLoc DL(Node);
3244 SDValue Chain = Node->getOperand(0);
3245
3246 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64,
3247 MVT::Other, MemAddr, Chain);
3248
3249 // Transfer memoperands.
3250 MachineMemOperand *MemOp =
3251 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
3252 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3253 ReplaceNode(Node, Ld);
3254 return;
3255 }
3256 case Intrinsic::aarch64_stlxp:
3257 case Intrinsic::aarch64_stxp: {
3258 unsigned Op =
3259 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
3260 SDLoc DL(Node);
3261 SDValue Chain = Node->getOperand(0);
3262 SDValue ValLo = Node->getOperand(2);
3263 SDValue ValHi = Node->getOperand(3);
3264 SDValue MemAddr = Node->getOperand(4);
3265
3266 // Place arguments in the right order.
3267 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
3268
3269 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops);
3270 // Transfer memoperands.
3271 MachineMemOperand *MemOp =
3272 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
3273 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3274
3275 ReplaceNode(Node, St);
3276 return;
3277 }
3278 case Intrinsic::aarch64_neon_ld1x2:
3279 if (VT == MVT::v8i8) {
3280 SelectLoad(Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
3281 return;
3282 } else if (VT == MVT::v16i8) {
3283 SelectLoad(Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
3284 return;
3285 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3286 SelectLoad(Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
3287 return;
3288 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3289 SelectLoad(Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
3290 return;
3291 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3292 SelectLoad(Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
3293 return;
3294 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3295 SelectLoad(Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
3296 return;
3297 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3298 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
3299 return;
3300 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3301 SelectLoad(Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
3302 return;
3303 }
3304 break;
3305 case Intrinsic::aarch64_neon_ld1x3:
3306 if (VT == MVT::v8i8) {
3307 SelectLoad(Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
3308 return;
3309 } else if (VT == MVT::v16i8) {
3310 SelectLoad(Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
3311 return;
3312 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3313 SelectLoad(Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
3314 return;
3315 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3316 SelectLoad(Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
3317 return;
3318 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3319 SelectLoad(Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
3320 return;
3321 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3322 SelectLoad(Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
3323 return;
3324 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3325 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
3326 return;
3327 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3328 SelectLoad(Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
3329 return;
3330 }
3331 break;
3332 case Intrinsic::aarch64_neon_ld1x4:
3333 if (VT == MVT::v8i8) {
3334 SelectLoad(Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
3335 return;
3336 } else if (VT == MVT::v16i8) {
3337 SelectLoad(Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
3338 return;
3339 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3340 SelectLoad(Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
3341 return;
3342 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3343 SelectLoad(Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
3344 return;
3345 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3346 SelectLoad(Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
3347 return;
3348 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3349 SelectLoad(Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
3350 return;
3351 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3352 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
3353 return;
3354 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3355 SelectLoad(Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
3356 return;
3357 }
3358 break;
3359 case Intrinsic::aarch64_neon_ld2:
3360 if (VT == MVT::v8i8) {
3361 SelectLoad(Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
3362 return;
3363 } else if (VT == MVT::v16i8) {
3364 SelectLoad(Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
3365 return;
3366 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3367 SelectLoad(Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
3368 return;
3369 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3370 SelectLoad(Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
3371 return;
3372 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3373 SelectLoad(Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
3374 return;
3375 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3376 SelectLoad(Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
3377 return;
3378 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3379 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
3380 return;
3381 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3382 SelectLoad(Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
3383 return;
3384 }
3385 break;
3386 case Intrinsic::aarch64_neon_ld3:
3387 if (VT == MVT::v8i8) {
3388 SelectLoad(Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
3389 return;
3390 } else if (VT == MVT::v16i8) {
3391 SelectLoad(Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
3392 return;
3393 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3394 SelectLoad(Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
3395 return;
3396 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3397 SelectLoad(Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
3398 return;
3399 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3400 SelectLoad(Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
3401 return;
3402 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3403 SelectLoad(Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
3404 return;
3405 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3406 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
3407 return;
3408 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3409 SelectLoad(Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
3410 return;
3411 }
3412 break;
3413 case Intrinsic::aarch64_neon_ld4:
3414 if (VT == MVT::v8i8) {
3415 SelectLoad(Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
3416 return;
3417 } else if (VT == MVT::v16i8) {
3418 SelectLoad(Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
3419 return;
3420 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3421 SelectLoad(Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
3422 return;
3423 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3424 SelectLoad(Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
3425 return;
3426 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3427 SelectLoad(Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
3428 return;
3429 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3430 SelectLoad(Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
3431 return;
3432 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3433 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
3434 return;
3435 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3436 SelectLoad(Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
3437 return;
3438 }
3439 break;
3440 case Intrinsic::aarch64_neon_ld2r:
3441 if (VT == MVT::v8i8) {
3442 SelectLoad(Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
3443 return;
3444 } else if (VT == MVT::v16i8) {
3445 SelectLoad(Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
3446 return;
3447 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3448 SelectLoad(Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
3449 return;
3450 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3451 SelectLoad(Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
3452 return;
3453 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3454 SelectLoad(Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
3455 return;
3456 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3457 SelectLoad(Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
3458 return;
3459 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3460 SelectLoad(Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
3461 return;
3462 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3463 SelectLoad(Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
3464 return;
3465 }
3466 break;
3467 case Intrinsic::aarch64_neon_ld3r:
3468 if (VT == MVT::v8i8) {
3469 SelectLoad(Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
3470 return;
3471 } else if (VT == MVT::v16i8) {
3472 SelectLoad(Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
3473 return;
3474 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3475 SelectLoad(Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
3476 return;
3477 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3478 SelectLoad(Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
3479 return;
3480 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3481 SelectLoad(Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
3482 return;
3483 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3484 SelectLoad(Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
3485 return;
3486 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3487 SelectLoad(Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
3488 return;
3489 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3490 SelectLoad(Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
3491 return;
3492 }
3493 break;
3494 case Intrinsic::aarch64_neon_ld4r:
3495 if (VT == MVT::v8i8) {
3496 SelectLoad(Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
3497 return;
3498 } else if (VT == MVT::v16i8) {
3499 SelectLoad(Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
3500 return;
3501 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3502 SelectLoad(Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
3503 return;
3504 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3505 SelectLoad(Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
3506 return;
3507 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3508 SelectLoad(Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
3509 return;
3510 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3511 SelectLoad(Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
3512 return;
3513 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3514 SelectLoad(Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
3515 return;
3516 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3517 SelectLoad(Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
3518 return;
3519 }
3520 break;
3521 case Intrinsic::aarch64_neon_ld2lane:
3522 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3523 SelectLoadLane(Node, 2, AArch64::LD2i8);
3524 return;
3525 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3526 VT == MVT::v8f16) {
3527 SelectLoadLane(Node, 2, AArch64::LD2i16);
3528 return;
3529 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3530 VT == MVT::v2f32) {
3531 SelectLoadLane(Node, 2, AArch64::LD2i32);
3532 return;
3533 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3534 VT == MVT::v1f64) {
3535 SelectLoadLane(Node, 2, AArch64::LD2i64);
3536 return;
3537 }
3538 break;
3539 case Intrinsic::aarch64_neon_ld3lane:
3540 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3541 SelectLoadLane(Node, 3, AArch64::LD3i8);
3542 return;
3543 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3544 VT == MVT::v8f16) {
3545 SelectLoadLane(Node, 3, AArch64::LD3i16);
3546 return;
3547 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3548 VT == MVT::v2f32) {
3549 SelectLoadLane(Node, 3, AArch64::LD3i32);
3550 return;
3551 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3552 VT == MVT::v1f64) {
3553 SelectLoadLane(Node, 3, AArch64::LD3i64);
3554 return;
3555 }
3556 break;
3557 case Intrinsic::aarch64_neon_ld4lane:
3558 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3559 SelectLoadLane(Node, 4, AArch64::LD4i8);
3560 return;
3561 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3562 VT == MVT::v8f16) {
3563 SelectLoadLane(Node, 4, AArch64::LD4i16);
3564 return;
3565 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3566 VT == MVT::v2f32) {
3567 SelectLoadLane(Node, 4, AArch64::LD4i32);
3568 return;
3569 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3570 VT == MVT::v1f64) {
3571 SelectLoadLane(Node, 4, AArch64::LD4i64);
3572 return;
3573 }
3574 break;
3575 }
3576 } break;
3577 case ISD::INTRINSIC_WO_CHAIN: {
3578 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
3579 switch (IntNo) {
3580 default:
3581 break;
3582 case Intrinsic::aarch64_tagp:
3583 SelectTagP(Node);
3584 return;
3585 case Intrinsic::aarch64_neon_tbl2:
3586 SelectTable(Node, 2,
3587 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
3588 false);
3589 return;
3590 case Intrinsic::aarch64_neon_tbl3:
3591 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
3592 : AArch64::TBLv16i8Three,
3593 false);
3594 return;
3595 case Intrinsic::aarch64_neon_tbl4:
3596 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
3597 : AArch64::TBLv16i8Four,
3598 false);
3599 return;
3600 case Intrinsic::aarch64_neon_tbx2:
3601 SelectTable(Node, 2,
3602 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
3603 true);
3604 return;
3605 case Intrinsic::aarch64_neon_tbx3:
3606 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
3607 : AArch64::TBXv16i8Three,
3608 true);
3609 return;
3610 case Intrinsic::aarch64_neon_tbx4:
3611 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
3612 : AArch64::TBXv16i8Four,
3613 true);
3614 return;
3615 case Intrinsic::aarch64_neon_smull:
3616 case Intrinsic::aarch64_neon_umull:
3617 if (tryMULLV64LaneV128(IntNo, Node))
3618 return;
3619 break;
3620 }
3621 break;
3622 }
3623 case ISD::INTRINSIC_VOID: {
3624 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
3625 if (Node->getNumOperands() >= 3)
3626 VT = Node->getOperand(2)->getValueType(0);
3627 switch (IntNo) {
3628 default:
3629 break;
3630 case Intrinsic::aarch64_neon_st1x2: {
3631 if (VT == MVT::v8i8) {
3632 SelectStore(Node, 2, AArch64::ST1Twov8b);
3633 return;
3634 } else if (VT == MVT::v16i8) {
3635 SelectStore(Node, 2, AArch64::ST1Twov16b);
3636 return;
3637 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3638 SelectStore(Node, 2, AArch64::ST1Twov4h);
3639 return;
3640 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3641 SelectStore(Node, 2, AArch64::ST1Twov8h);
3642 return;
3643 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3644 SelectStore(Node, 2, AArch64::ST1Twov2s);
3645 return;
3646 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3647 SelectStore(Node, 2, AArch64::ST1Twov4s);
3648 return;
3649 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3650 SelectStore(Node, 2, AArch64::ST1Twov2d);
3651 return;
3652 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3653 SelectStore(Node, 2, AArch64::ST1Twov1d);
3654 return;
3655 }
3656 break;
3657 }
3658 case Intrinsic::aarch64_neon_st1x3: {
3659 if (VT == MVT::v8i8) {
3660 SelectStore(Node, 3, AArch64::ST1Threev8b);
3661 return;
3662 } else if (VT == MVT::v16i8) {
3663 SelectStore(Node, 3, AArch64::ST1Threev16b);
3664 return;
3665 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3666 SelectStore(Node, 3, AArch64::ST1Threev4h);
3667 return;
3668 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3669 SelectStore(Node, 3, AArch64::ST1Threev8h);
3670 return;
3671 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3672 SelectStore(Node, 3, AArch64::ST1Threev2s);
3673 return;
3674 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3675 SelectStore(Node, 3, AArch64::ST1Threev4s);
3676 return;
3677 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3678 SelectStore(Node, 3, AArch64::ST1Threev2d);
3679 return;
3680 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3681 SelectStore(Node, 3, AArch64::ST1Threev1d);
3682 return;
3683 }
3684 break;
3685 }
3686 case Intrinsic::aarch64_neon_st1x4: {
3687 if (VT == MVT::v8i8) {
3688 SelectStore(Node, 4, AArch64::ST1Fourv8b);
3689 return;
3690 } else if (VT == MVT::v16i8) {
3691 SelectStore(Node, 4, AArch64::ST1Fourv16b);
3692 return;
3693 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3694 SelectStore(Node, 4, AArch64::ST1Fourv4h);
3695 return;
3696 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3697 SelectStore(Node, 4, AArch64::ST1Fourv8h);
3698 return;
3699 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3700 SelectStore(Node, 4, AArch64::ST1Fourv2s);
3701 return;
3702 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3703 SelectStore(Node, 4, AArch64::ST1Fourv4s);
3704 return;
3705 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3706 SelectStore(Node, 4, AArch64::ST1Fourv2d);
3707 return;
3708 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3709 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3710 return;
3711 }
3712 break;
3713 }
3714 case Intrinsic::aarch64_neon_st2: {
3715 if (VT == MVT::v8i8) {
3716 SelectStore(Node, 2, AArch64::ST2Twov8b);
3717 return;
3718 } else if (VT == MVT::v16i8) {
3719 SelectStore(Node, 2, AArch64::ST2Twov16b);
3720 return;
3721 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3722 SelectStore(Node, 2, AArch64::ST2Twov4h);
3723 return;
3724 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3725 SelectStore(Node, 2, AArch64::ST2Twov8h);
3726 return;
3727 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3728 SelectStore(Node, 2, AArch64::ST2Twov2s);
3729 return;
3730 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3731 SelectStore(Node, 2, AArch64::ST2Twov4s);
3732 return;
3733 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3734 SelectStore(Node, 2, AArch64::ST2Twov2d);
3735 return;
3736 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3737 SelectStore(Node, 2, AArch64::ST1Twov1d);
3738 return;
3739 }
3740 break;
3741 }
3742 case Intrinsic::aarch64_neon_st3: {
3743 if (VT == MVT::v8i8) {
3744 SelectStore(Node, 3, AArch64::ST3Threev8b);
3745 return;
3746 } else if (VT == MVT::v16i8) {
3747 SelectStore(Node, 3, AArch64::ST3Threev16b);
3748 return;
3749 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3750 SelectStore(Node, 3, AArch64::ST3Threev4h);
3751 return;
3752 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3753 SelectStore(Node, 3, AArch64::ST3Threev8h);
3754 return;
3755 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3756 SelectStore(Node, 3, AArch64::ST3Threev2s);
3757 return;
3758 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3759 SelectStore(Node, 3, AArch64::ST3Threev4s);
3760 return;
3761 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3762 SelectStore(Node, 3, AArch64::ST3Threev2d);
3763 return;
3764 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3765 SelectStore(Node, 3, AArch64::ST1Threev1d);
3766 return;
3767 }
3768 break;
3769 }
3770 case Intrinsic::aarch64_neon_st4: {
3771 if (VT == MVT::v8i8) {
3772 SelectStore(Node, 4, AArch64::ST4Fourv8b);
3773 return;
3774 } else if (VT == MVT::v16i8) {
3775 SelectStore(Node, 4, AArch64::ST4Fourv16b);
3776 return;
3777 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3778 SelectStore(Node, 4, AArch64::ST4Fourv4h);
3779 return;
3780 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3781 SelectStore(Node, 4, AArch64::ST4Fourv8h);
3782 return;
3783 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3784 SelectStore(Node, 4, AArch64::ST4Fourv2s);
3785 return;
3786 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3787 SelectStore(Node, 4, AArch64::ST4Fourv4s);
3788 return;
3789 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3790 SelectStore(Node, 4, AArch64::ST4Fourv2d);
3791 return;
3792 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3793 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3794 return;
3795 }
3796 break;
3797 }
3798 case Intrinsic::aarch64_neon_st2lane: {
3799 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3800 SelectStoreLane(Node, 2, AArch64::ST2i8);
3801 return;
3802 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3803 VT == MVT::v8f16) {
3804 SelectStoreLane(Node, 2, AArch64::ST2i16);
3805 return;
3806 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3807 VT == MVT::v2f32) {
3808 SelectStoreLane(Node, 2, AArch64::ST2i32);
3809 return;
3810 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3811 VT == MVT::v1f64) {
3812 SelectStoreLane(Node, 2, AArch64::ST2i64);
3813 return;
3814 }
3815 break;
3816 }
3817 case Intrinsic::aarch64_neon_st3lane: {
3818 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3819 SelectStoreLane(Node, 3, AArch64::ST3i8);
3820 return;
3821 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3822 VT == MVT::v8f16) {
3823 SelectStoreLane(Node, 3, AArch64::ST3i16);
3824 return;
3825 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3826 VT == MVT::v2f32) {
3827 SelectStoreLane(Node, 3, AArch64::ST3i32);
3828 return;
3829 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3830 VT == MVT::v1f64) {
3831 SelectStoreLane(Node, 3, AArch64::ST3i64);
3832 return;
3833 }
3834 break;
3835 }
3836 case Intrinsic::aarch64_neon_st4lane: {
3837 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3838 SelectStoreLane(Node, 4, AArch64::ST4i8);
3839 return;
3840 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3841 VT == MVT::v8f16) {
3842 SelectStoreLane(Node, 4, AArch64::ST4i16);
3843 return;
3844 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3845 VT == MVT::v2f32) {
3846 SelectStoreLane(Node, 4, AArch64::ST4i32);
3847 return;
3848 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3849 VT == MVT::v1f64) {
3850 SelectStoreLane(Node, 4, AArch64::ST4i64);
3851 return;
3852 }
3853 break;
3854 }
3855 }
3856 break;
3857 }
3858 case AArch64ISD::LD2post: {
3859 if (VT == MVT::v8i8) {
3860 SelectPostLoad(Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
3861 return;
3862 } else if (VT == MVT::v16i8) {
3863 SelectPostLoad(Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
3864 return;
3865 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3866 SelectPostLoad(Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
3867 return;
3868 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3869 SelectPostLoad(Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
3870 return;
3871 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3872 SelectPostLoad(Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
3873 return;
3874 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3875 SelectPostLoad(Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
3876 return;
3877 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3878 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3879 return;
3880 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3881 SelectPostLoad(Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
3882 return;
3883 }
3884 break;
3885 }
3886 case AArch64ISD::LD3post: {
3887 if (VT == MVT::v8i8) {
3888 SelectPostLoad(Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
3889 return;
3890 } else if (VT == MVT::v16i8) {
3891 SelectPostLoad(Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
3892 return;
3893 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3894 SelectPostLoad(Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
3895 return;
3896 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3897 SelectPostLoad(Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
3898 return;
3899 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3900 SelectPostLoad(Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
3901 return;
3902 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3903 SelectPostLoad(Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
3904 return;
3905 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3906 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3907 return;
3908 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3909 SelectPostLoad(Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
3910 return;
3911 }
3912 break;
3913 }
3914 case AArch64ISD::LD4post: {
3915 if (VT == MVT::v8i8) {
3916 SelectPostLoad(Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
3917 return;
3918 } else if (VT == MVT::v16i8) {
3919 SelectPostLoad(Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
3920 return;
3921 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3922 SelectPostLoad(Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
3923 return;
3924 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3925 SelectPostLoad(Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
3926 return;
3927 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3928 SelectPostLoad(Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
3929 return;
3930 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3931 SelectPostLoad(Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
3932 return;
3933 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3934 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
3935 return;
3936 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3937 SelectPostLoad(Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
3938 return;
3939 }
3940 break;
3941 }
3942 case AArch64ISD::LD1x2post: {
3943 if (VT == MVT::v8i8) {
3944 SelectPostLoad(Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
3945 return;
3946 } else if (VT == MVT::v16i8) {
3947 SelectPostLoad(Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
3948 return;
3949 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3950 SelectPostLoad(Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
3951 return;
3952 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3953 SelectPostLoad(Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
3954 return;
3955 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3956 SelectPostLoad(Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
3957 return;
3958 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3959 SelectPostLoad(Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
3960 return;
3961 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3962 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3963 return;
3964 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3965 SelectPostLoad(Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
3966 return;
3967 }
3968 break;
3969 }
3970 case AArch64ISD::LD1x3post: {
3971 if (VT == MVT::v8i8) {
3972 SelectPostLoad(Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
3973 return;
3974 } else if (VT == MVT::v16i8) {
3975 SelectPostLoad(Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
3976 return;
3977 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3978 SelectPostLoad(Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
3979 return;
3980 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3981 SelectPostLoad(Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
3982 return;
3983 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3984 SelectPostLoad(Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
3985 return;
3986 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3987 SelectPostLoad(Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
3988 return;
3989 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3990 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3991 return;
3992 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3993 SelectPostLoad(Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
3994 return;
3995 }
3996 break;
3997 }
3998 case AArch64ISD::LD1x4post: {
3999 if (VT == MVT::v8i8) {
4000 SelectPostLoad(Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
4001 return;
4002 } else if (VT == MVT::v16i8) {
4003 SelectPostLoad(Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
4004 return;
4005 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4006 SelectPostLoad(Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
4007 return;
4008 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4009 SelectPostLoad(Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
4010 return;
4011 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4012 SelectPostLoad(Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
4013 return;
4014 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4015 SelectPostLoad(Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
4016 return;
4017 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4018 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
4019 return;
4020 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4021 SelectPostLoad(Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
4022 return;
4023 }
4024 break;
4025 }
4026 case AArch64ISD::LD1DUPpost: {
4027 if (VT == MVT::v8i8) {
4028 SelectPostLoad(Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
4029 return;
4030 } else if (VT == MVT::v16i8) {
4031 SelectPostLoad(Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
4032 return;
4033 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4034 SelectPostLoad(Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
4035 return;
4036 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4037 SelectPostLoad(Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
4038 return;
4039 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4040 SelectPostLoad(Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
4041 return;
4042 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4043 SelectPostLoad(Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
4044 return;
4045 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4046 SelectPostLoad(Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
4047 return;
4048 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4049 SelectPostLoad(Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
4050 return;
4051 }
4052 break;
4053 }
4054 case AArch64ISD::LD2DUPpost: {
4055 if (VT == MVT::v8i8) {
4056 SelectPostLoad(Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
4057 return;
4058 } else if (VT == MVT::v16i8) {
4059 SelectPostLoad(Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
4060 return;
4061 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4062 SelectPostLoad(Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
4063 return;
4064 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4065 SelectPostLoad(Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
4066 return;
4067 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4068 SelectPostLoad(Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
4069 return;
4070 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4071 SelectPostLoad(Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
4072 return;
4073 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4074 SelectPostLoad(Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
4075 return;
4076 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4077 SelectPostLoad(Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
4078 return;
4079 }
4080 break;
4081 }
4082 case AArch64ISD::LD3DUPpost: {
4083 if (VT == MVT::v8i8) {
4084 SelectPostLoad(Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
4085 return;
4086 } else if (VT == MVT::v16i8) {
4087 SelectPostLoad(Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
4088 return;
4089 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4090 SelectPostLoad(Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
4091 return;
4092 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4093 SelectPostLoad(Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
4094 return;
4095 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4096 SelectPostLoad(Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
4097 return;
4098 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4099 SelectPostLoad(Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
4100 return;
4101 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4102 SelectPostLoad(Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
4103 return;
4104 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4105 SelectPostLoad(Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
4106 return;
4107 }
4108 break;
4109 }
4110 case AArch64ISD::LD4DUPpost: {
4111 if (VT == MVT::v8i8) {
4112 SelectPostLoad(Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
4113 return;
4114 } else if (VT == MVT::v16i8) {
4115 SelectPostLoad(Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
4116 return;
4117 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4118 SelectPostLoad(Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
4119 return;
4120 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4121 SelectPostLoad(Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
4122 return;
4123 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4124 SelectPostLoad(Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
4125 return;
4126 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4127 SelectPostLoad(Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
4128 return;
4129 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4130 SelectPostLoad(Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
4131 return;
4132 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4133 SelectPostLoad(Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
4134 return;
4135 }
4136 break;
4137 }
4138 case AArch64ISD::LD1LANEpost: {
4139 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4140 SelectPostLoadLane(Node, 1, AArch64::LD1i8_POST);
4141 return;
4142 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4143 VT == MVT::v8f16) {
4144 SelectPostLoadLane(Node, 1, AArch64::LD1i16_POST);
4145 return;
4146 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4147 VT == MVT::v2f32) {
4148 SelectPostLoadLane(Node, 1, AArch64::LD1i32_POST);
4149 return;
4150 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4151 VT == MVT::v1f64) {
4152 SelectPostLoadLane(Node, 1, AArch64::LD1i64_POST);
4153 return;
4154 }
4155 break;
4156 }
4157 case AArch64ISD::LD2LANEpost: {
4158 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4159 SelectPostLoadLane(Node, 2, AArch64::LD2i8_POST);
4160 return;
4161 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4162 VT == MVT::v8f16) {
4163 SelectPostLoadLane(Node, 2, AArch64::LD2i16_POST);
4164 return;
4165 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4166 VT == MVT::v2f32) {
4167 SelectPostLoadLane(Node, 2, AArch64::LD2i32_POST);
4168 return;
4169 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4170 VT == MVT::v1f64) {
4171 SelectPostLoadLane(Node, 2, AArch64::LD2i64_POST);
4172 return;
4173 }
4174 break;
4175 }
4176 case AArch64ISD::LD3LANEpost: {
4177 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4178 SelectPostLoadLane(Node, 3, AArch64::LD3i8_POST);
4179 return;
4180 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4181 VT == MVT::v8f16) {
4182 SelectPostLoadLane(Node, 3, AArch64::LD3i16_POST);
4183 return;
4184 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4185 VT == MVT::v2f32) {
4186 SelectPostLoadLane(Node, 3, AArch64::LD3i32_POST);
4187 return;
4188 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4189 VT == MVT::v1f64) {
4190 SelectPostLoadLane(Node, 3, AArch64::LD3i64_POST);
4191 return;
4192 }
4193 break;
4194 }
4195 case AArch64ISD::LD4LANEpost: {
4196 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4197 SelectPostLoadLane(Node, 4, AArch64::LD4i8_POST);
4198 return;
4199 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4200 VT == MVT::v8f16) {
4201 SelectPostLoadLane(Node, 4, AArch64::LD4i16_POST);
4202 return;
4203 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4204 VT == MVT::v2f32) {
4205 SelectPostLoadLane(Node, 4, AArch64::LD4i32_POST);
4206 return;
4207 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4208 VT == MVT::v1f64) {
4209 SelectPostLoadLane(Node, 4, AArch64::LD4i64_POST);
4210 return;
4211 }
4212 break;
4213 }
4214 case AArch64ISD::ST2post: {
4215 VT = Node->getOperand(1).getValueType();
4216 if (VT == MVT::v8i8) {
4217 SelectPostStore(Node, 2, AArch64::ST2Twov8b_POST);
4218 return;
4219 } else if (VT == MVT::v16i8) {
4220 SelectPostStore(Node, 2, AArch64::ST2Twov16b_POST);
4221 return;
4222 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4223 SelectPostStore(Node, 2, AArch64::ST2Twov4h_POST);
4224 return;
4225 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4226 SelectPostStore(Node, 2, AArch64::ST2Twov8h_POST);
4227 return;
4228 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4229 SelectPostStore(Node, 2, AArch64::ST2Twov2s_POST);
4230 return;
4231 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4232 SelectPostStore(Node, 2, AArch64::ST2Twov4s_POST);
4233 return;
4234 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4235 SelectPostStore(Node, 2, AArch64::ST2Twov2d_POST);
4236 return;
4237 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4238 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
4239 return;
4240 }
4241 break;
4242 }
4243 case AArch64ISD::ST3post: {
4244 VT = Node->getOperand(1).getValueType();
4245 if (VT == MVT::v8i8) {
4246 SelectPostStore(Node, 3, AArch64::ST3Threev8b_POST);
4247 return;
4248 } else if (VT == MVT::v16i8) {
4249 SelectPostStore(Node, 3, AArch64::ST3Threev16b_POST);
4250 return;
4251 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4252 SelectPostStore(Node, 3, AArch64::ST3Threev4h_POST);
4253 return;
4254 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4255 SelectPostStore(Node, 3, AArch64::ST3Threev8h_POST);
4256 return;
4257 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4258 SelectPostStore(Node, 3, AArch64::ST3Threev2s_POST);
4259 return;
4260 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4261 SelectPostStore(Node, 3, AArch64::ST3Threev4s_POST);
4262 return;
4263 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4264 SelectPostStore(Node, 3, AArch64::ST3Threev2d_POST);
4265 return;
4266 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4267 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
4268 return;
4269 }
4270 break;
4271 }
4272 case AArch64ISD::ST4post: {
4273 VT = Node->getOperand(1).getValueType();
4274 if (VT == MVT::v8i8) {
4275 SelectPostStore(Node, 4, AArch64::ST4Fourv8b_POST);
4276 return;
4277 } else if (VT == MVT::v16i8) {
4278 SelectPostStore(Node, 4, AArch64::ST4Fourv16b_POST);
4279 return;
4280 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4281 SelectPostStore(Node, 4, AArch64::ST4Fourv4h_POST);
4282 return;
4283 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4284 SelectPostStore(Node, 4, AArch64::ST4Fourv8h_POST);
4285 return;
4286 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4287 SelectPostStore(Node, 4, AArch64::ST4Fourv2s_POST);
4288 return;
4289 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4290 SelectPostStore(Node, 4, AArch64::ST4Fourv4s_POST);
4291 return;
4292 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4293 SelectPostStore(Node, 4, AArch64::ST4Fourv2d_POST);
4294 return;
4295 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4296 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
4297 return;
4298 }
4299 break;
4300 }
4301 case AArch64ISD::ST1x2post: {
4302 VT = Node->getOperand(1).getValueType();
4303 if (VT == MVT::v8i8) {
4304 SelectPostStore(Node, 2, AArch64::ST1Twov8b_POST);
4305 return;
4306 } else if (VT == MVT::v16i8) {
4307 SelectPostStore(Node, 2, AArch64::ST1Twov16b_POST);
4308 return;
4309 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4310 SelectPostStore(Node, 2, AArch64::ST1Twov4h_POST);
4311 return;
4312 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4313 SelectPostStore(Node, 2, AArch64::ST1Twov8h_POST);
4314 return;
4315 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4316 SelectPostStore(Node, 2, AArch64::ST1Twov2s_POST);
4317 return;
4318 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4319 SelectPostStore(Node, 2, AArch64::ST1Twov4s_POST);
4320 return;
4321 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4322 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
4323 return;
4324 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4325 SelectPostStore(Node, 2, AArch64::ST1Twov2d_POST);
4326 return;
4327 }
4328 break;
4329 }
4330 case AArch64ISD::ST1x3post: {
4331 VT = Node->getOperand(1).getValueType();
4332 if (VT == MVT::v8i8) {
4333 SelectPostStore(Node, 3, AArch64::ST1Threev8b_POST);
4334 return;
4335 } else if (VT == MVT::v16i8) {
4336 SelectPostStore(Node, 3, AArch64::ST1Threev16b_POST);
4337 return;
4338 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4339 SelectPostStore(Node, 3, AArch64::ST1Threev4h_POST);
4340 return;
4341 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4342 SelectPostStore(Node, 3, AArch64::ST1Threev8h_POST);
4343 return;
4344 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4345 SelectPostStore(Node, 3, AArch64::ST1Threev2s_POST);
4346 return;
4347 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4348 SelectPostStore(Node, 3, AArch64::ST1Threev4s_POST);
4349 return;
4350 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4351 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
4352 return;
4353 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4354 SelectPostStore(Node, 3, AArch64::ST1Threev2d_POST);
4355 return;
4356 }
4357 break;
4358 }
4359 case AArch64ISD::ST1x4post: {
4360 VT = Node->getOperand(1).getValueType();
4361 if (VT == MVT::v8i8) {
4362 SelectPostStore(Node, 4, AArch64::ST1Fourv8b_POST);
4363 return;
4364 } else if (VT == MVT::v16i8) {
4365 SelectPostStore(Node, 4, AArch64::ST1Fourv16b_POST);
4366 return;
4367 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4368 SelectPostStore(Node, 4, AArch64::ST1Fourv4h_POST);
4369 return;
4370 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4371 SelectPostStore(Node, 4, AArch64::ST1Fourv8h_POST);
4372 return;
4373 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4374 SelectPostStore(Node, 4, AArch64::ST1Fourv2s_POST);
4375 return;
4376 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4377 SelectPostStore(Node, 4, AArch64::ST1Fourv4s_POST);
4378 return;
4379 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4380 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
4381 return;
4382 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4383 SelectPostStore(Node, 4, AArch64::ST1Fourv2d_POST);
4384 return;
4385 }
4386 break;
4387 }
4388 case AArch64ISD::ST2LANEpost: {
4389 VT = Node->getOperand(1).getValueType();
4390 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4391 SelectPostStoreLane(Node, 2, AArch64::ST2i8_POST);
4392 return;
4393 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4394 VT == MVT::v8f16) {
4395 SelectPostStoreLane(Node, 2, AArch64::ST2i16_POST);
4396 return;
4397 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4398 VT == MVT::v2f32) {
4399 SelectPostStoreLane(Node, 2, AArch64::ST2i32_POST);
4400 return;
4401 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4402 VT == MVT::v1f64) {
4403 SelectPostStoreLane(Node, 2, AArch64::ST2i64_POST);
4404 return;
4405 }
4406 break;
4407 }
4408 case AArch64ISD::ST3LANEpost: {
4409 VT = Node->getOperand(1).getValueType();
4410 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4411 SelectPostStoreLane(Node, 3, AArch64::ST3i8_POST);
4412 return;
4413 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4414 VT == MVT::v8f16) {
4415 SelectPostStoreLane(Node, 3, AArch64::ST3i16_POST);
4416 return;
4417 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4418 VT == MVT::v2f32) {
4419 SelectPostStoreLane(Node, 3, AArch64::ST3i32_POST);
4420 return;
4421 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4422 VT == MVT::v1f64) {
4423 SelectPostStoreLane(Node, 3, AArch64::ST3i64_POST);
4424 return;
4425 }
4426 break;
4427 }
4428 case AArch64ISD::ST4LANEpost: {
4429 VT = Node->getOperand(1).getValueType();
4430 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4431 SelectPostStoreLane(Node, 4, AArch64::ST4i8_POST);
4432 return;
4433 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4434 VT == MVT::v8f16) {
4435 SelectPostStoreLane(Node, 4, AArch64::ST4i16_POST);
4436 return;
4437 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4438 VT == MVT::v2f32) {
4439 SelectPostStoreLane(Node, 4, AArch64::ST4i32_POST);
4440 return;
4441 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4442 VT == MVT::v1f64) {
4443 SelectPostStoreLane(Node, 4, AArch64::ST4i64_POST);
4444 return;
4445 }
4446 break;
4447 }
4448 }
4449
4450 // Select the default instruction
4451 SelectCode(Node);
4452}
4453
4454/// createAArch64ISelDag - This pass converts a legalized DAG into a
4455/// AArch64-specific DAG, ready for instruction scheduling.
4456FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
4457 CodeGenOpt::Level OptLevel) {
4458 return new AArch64DAGToDAGISel(TM, OptLevel);
4459}
4460
4461/// When \p PredVT is a scalable vector predicate in the form
4462/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
4463/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. If the input
4464/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
4465/// EVT.
4466static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT) {
4467 if (!PredVT.isScalableVector() || PredVT.getVectorElementType() != MVT::i1)
4468 return EVT();
4469
4470 const unsigned NumElts = PredVT.getVectorNumElements();
4471
4472 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
4473 return EVT();
4474
4475 EVT ScalarVT = EVT::getIntegerVT(Ctx, AArch64::SVEBitsPerBlock / NumElts);
4476 EVT MemVT = EVT::getVectorVT(Ctx, ScalarVT, NumElts, /*IsScalable=*/true);
4477 return MemVT;
4478}
4479
4480/// Return the EVT of the data associated to a memory operation in \p
4481/// Root. If such EVT cannot be retrived, it returns an invalid EVT.
4482static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root) {
4483 if (isa<MemSDNode>(Root))
4484 return cast<MemSDNode>(Root)->getMemoryVT();
4485
4486 const unsigned Opcode = Root->getOpcode();
4487 if (Opcode != ISD::INTRINSIC_VOID)
4488 return EVT();
4489
4490 const unsigned IntNo =
4491 cast<ConstantSDNode>(Root->getOperand(1))->getZExtValue();
4492 if (IntNo != Intrinsic::aarch64_sve_prf)
4493 return EVT();
4494
4495 // We are using an SVE prefetch intrinsic. Type must be inferred
4496 // from the width of the predicate.
4497 return getPackedVectorTypeFromPredicateType(
4498 Ctx, Root->getOperand(2)->getValueType(0));
4499}
4500
4501/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
4502/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
4503/// where Root is the memory access using N for its address.
4504template <int64_t Min, int64_t Max>
4505bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
4506 SDValue &Base,
4507 SDValue &OffImm) {
4508 const EVT MemVT = getMemVTFromNode(*(CurDAG->getContext()), Root);
4509
4510 if (MemVT == EVT())
4511 return false;
4512
4513 if (N.getOpcode() != ISD::ADD)
4514 return false;
4515
4516 SDValue VScale = N.getOperand(1);
4517 if (VScale.getOpcode() != ISD::VSCALE)
4518 return false;
4519
4520 TypeSize TS = MemVT.getSizeInBits();
4521 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinSize()) / 8;
4522 int64_t MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
4523
4524 if ((MulImm % MemWidthBytes) != 0)
4525 return false;
4526
4527 int64_t Offset = MulImm / MemWidthBytes;
4528 if (Offset < Min || Offset > Max)
4529 return false;
4530
4531 Base = N.getOperand(0);
4532 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64);
4533 return true;
4534}
4535
4536/// Select register plus register addressing mode for SVE, with scaled
4537/// offset.
4538bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
4539 SDValue &Base,
4540 SDValue &Offset) {
4541 if (N.getOpcode() != ISD::ADD)
4542 return false;
4543
4544 // Process an ADD node.
4545 const SDValue LHS = N.getOperand(0);
4546 const SDValue RHS = N.getOperand(1);
4547
4548 // 8 bit data does not come with the SHL node, so it is treated
4549 // separately.
4550 if (Scale == 0) {
4551 Base = LHS;
4552 Offset = RHS;
4553 return true;
4554 }
4555
4556 // Check if the RHS is a shift node with a constant.
4557 if (RHS.getOpcode() != ISD::SHL)
4558 return false;
4559
4560 const SDValue ShiftRHS = RHS.getOperand(1);
4561 if (auto *C = dyn_cast<ConstantSDNode>(ShiftRHS))
4562 if (C->getZExtValue() == Scale) {
4563 Base = LHS;
4564 Offset = RHS.getOperand(0);
4565 return true;
4566 }
4567
4568 return false;
4569}

/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/Support/AlignOf.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MachineValueType.h"
45#include "llvm/Support/TypeSize.h"
46#include <algorithm>
47#include <cassert>
48#include <climits>
49#include <cstddef>
50#include <cstdint>
51#include <cstring>
52#include <iterator>
53#include <string>
54#include <tuple>
55
56namespace llvm {
57
58class APInt;
59class Constant;
60template <typename T> struct DenseMapInfo;
61class GlobalValue;
62class MachineBasicBlock;
63class MachineConstantPoolValue;
64class MCSymbol;
65class raw_ostream;
66class SDNode;
67class SelectionDAG;
68class Type;
69class Value;
70
71void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
72 bool force = false);
73
74/// This represents a list of ValueType's that has been intern'd by
75/// a SelectionDAG. Instances of this simple value class are returned by
76/// SelectionDAG::getVTList(...).
77///
78struct SDVTList {
79 const EVT *VTs;
80 unsigned int NumVTs;
81};
82
83namespace ISD {
84
85 /// Node predicates
86
87 /// If N is a BUILD_VECTOR node whose elements are all the same constant or
88 /// undefined, return true and return the constant value in \p SplatValue.
89 bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
90
91 /// Return true if the specified node is a BUILD_VECTOR where all of the
92 /// elements are ~0 or undef.
93 bool isBuildVectorAllOnes(const SDNode *N);
94
95 /// Return true if the specified node is a BUILD_VECTOR where all of the
96 /// elements are 0 or undef.
97 bool isBuildVectorAllZeros(const SDNode *N);
98
99 /// Return true if the specified node is a BUILD_VECTOR node of all
100 /// ConstantSDNode or undef.
101 bool isBuildVectorOfConstantSDNodes(const SDNode *N);
102
103 /// Return true if the specified node is a BUILD_VECTOR node of all
104 /// ConstantFPSDNode or undef.
105 bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
106
107 /// Return true if the node has at least one operand and all operands of the
108 /// specified node are ISD::UNDEF.
109 bool allOperandsUndef(const SDNode *N);
110
111} // end namespace ISD
112
113//===----------------------------------------------------------------------===//
114/// Unlike LLVM values, Selection DAG nodes may return multiple
115/// values as the result of a computation. Many nodes return multiple values,
116/// from loads (which define a token and a return value) to ADDC (which returns
117/// a result and a carry value), to calls (which may return an arbitrary number
118/// of values).
119///
120/// As such, each use of a SelectionDAG computation must indicate the node that
121/// computes it as well as which return value to use from that node. This pair
122/// of information is represented with the SDValue value type.
123///
124class SDValue {
125 friend struct DenseMapInfo<SDValue>;
126
127 SDNode *Node = nullptr; // The node defining the value we are using.
128 unsigned ResNo = 0; // Which return value of the node we are using.
129
130public:
131 SDValue() = default;
132 SDValue(SDNode *node, unsigned resno);
133
134 /// get the index which selects a specific result in the SDNode
135 unsigned getResNo() const { return ResNo; }
136
137 /// get the SDNode which holds the desired result
138 SDNode *getNode() const { return Node; }
139
140 /// set the SDNode
141 void setNode(SDNode *N) { Node = N; }
142
143 inline SDNode *operator->() const { return Node; }
144
145 bool operator==(const SDValue &O) const {
146 return Node == O.Node && ResNo == O.ResNo;
147 }
148 bool operator!=(const SDValue &O) const {
149 return !operator==(O);
150 }
151 bool operator<(const SDValue &O) const {
152 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
153 }
154 explicit operator bool() const {
155 return Node != nullptr;
156 }
157
158 SDValue getValue(unsigned R) const {
159 return SDValue(Node, R);
160 }
161
162 /// Return true if this node is an operand of N.
163 bool isOperandOf(const SDNode *N) const;
164
165 /// Return the ValueType of the referenced return value.
166 inline EVT getValueType() const;
167
168 /// Return the simple ValueType of the referenced return value.
169 MVT getSimpleValueType() const {
170 return getValueType().getSimpleVT();
171 }
172
173 /// Returns the size of the value in bits.
174 ///
175 /// If the value type is a scalable vector type, the scalable property will
176 /// be set and the runtime size will be a positive integer multiple of the
177 /// base size.
178 TypeSize getValueSizeInBits() const {
179 return getValueType().getSizeInBits();
180 }
181
182 TypeSize getScalarValueSizeInBits() const {
183 return getValueType().getScalarType().getSizeInBits();
184 }
185
186 // Forwarding methods - These forward to the corresponding methods in SDNode.
187 inline unsigned getOpcode() const;
188 inline unsigned getNumOperands() const;
189 inline const SDValue &getOperand(unsigned i) const;
190 inline uint64_t getConstantOperandVal(unsigned i) const;
191 inline const APInt &getConstantOperandAPInt(unsigned i) const;
192 inline bool isTargetMemoryOpcode() const;
193 inline bool isTargetOpcode() const;
194 inline bool isMachineOpcode() const;
195 inline bool isUndef() const;
196 inline unsigned getMachineOpcode() const;
197 inline const DebugLoc &getDebugLoc() const;
198 inline void dump() const;
199 inline void dump(const SelectionDAG *G) const;
200 inline void dumpr() const;
201 inline void dumpr(const SelectionDAG *G) const;
202
203 /// Return true if this operand (which must be a chain) reaches the
204 /// specified operand without crossing any side-effecting instructions.
205 /// In practice, this looks through token factors and non-volatile loads.
206 /// In order to remain efficient, this only
207 /// looks a couple of nodes in, it does not do an exhaustive search.
208 bool reachesChainWithoutSideEffects(SDValue Dest,
209 unsigned Depth = 2) const;
210
211 /// Return true if there are no nodes using value ResNo of Node.
212 inline bool use_empty() const;
213
214 /// Return true if there is exactly one node using value ResNo of Node.
215 inline bool hasOneUse() const;
216};
217
218template<> struct DenseMapInfo<SDValue> {
219 static inline SDValue getEmptyKey() {
220 SDValue V;
221 V.ResNo = -1U;
222 return V;
223 }
224
225 static inline SDValue getTombstoneKey() {
226 SDValue V;
227 V.ResNo = -2U;
228 return V;
229 }
230
231 static unsigned getHashValue(const SDValue &Val) {
232 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
233 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
234 }
235
236 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
237 return LHS == RHS;
238 }
239};
240
241/// Allow casting operators to work directly on
242/// SDValues as if they were SDNode*'s.
243template<> struct simplify_type<SDValue> {
244 using SimpleType = SDNode *;
245
246 static SimpleType getSimplifiedValue(SDValue &Val) {
247 return Val.getNode();
248 }
249};
250template<> struct simplify_type<const SDValue> {
251 using SimpleType = /*const*/ SDNode *;
252
253 static SimpleType getSimplifiedValue(const SDValue &Val) {
254 return Val.getNode();
255 }
256};
257
258/// Represents a use of a SDNode. This class holds an SDValue,
259/// which records the SDNode being used and the result number, a
260/// pointer to the SDNode using the value, and Next and Prev pointers,
261/// which link together all the uses of an SDNode.
262///
263class SDUse {
264 /// Val - The value being used.
265 SDValue Val;
266 /// User - The user of this value.
267 SDNode *User = nullptr;
268 /// Prev, Next - Pointers to the uses list of the SDNode referred by
269 /// this operand.
270 SDUse **Prev = nullptr;
271 SDUse *Next = nullptr;
272
273public:
274 SDUse() = default;
275 SDUse(const SDUse &U) = delete;
276 SDUse &operator=(const SDUse &) = delete;
277
278 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
279 operator const SDValue&() const { return Val; }
280
281 /// If implicit conversion to SDValue doesn't work, the get() method returns
282 /// the SDValue.
283 const SDValue &get() const { return Val; }
284
285 /// This returns the SDNode that contains this Use.
286 SDNode *getUser() { return User; }
287
288 /// Get the next SDUse in the use list.
289 SDUse *getNext() const { return Next; }
290
291 /// Convenience function for get().getNode().
292 SDNode *getNode() const { return Val.getNode(); }
293 /// Convenience function for get().getResNo().
294 unsigned getResNo() const { return Val.getResNo(); }
295 /// Convenience function for get().getValueType().
296 EVT getValueType() const { return Val.getValueType(); }
297
298 /// Convenience function for get().operator==
299 bool operator==(const SDValue &V) const {
300 return Val == V;
301 }
302
303 /// Convenience function for get().operator!=
304 bool operator!=(const SDValue &V) const {
305 return Val != V;
306 }
307
308 /// Convenience function for get().operator<
309 bool operator<(const SDValue &V) const {
310 return Val < V;
311 }
312
313private:
314 friend class SelectionDAG;
315 friend class SDNode;
316 // TODO: unfriend HandleSDNode once we fix its operand handling.
317 friend class HandleSDNode;
318
319 void setUser(SDNode *p) { User = p; }
320
321 /// Remove this use from its existing use list, assign it the
322 /// given value, and add it to the new value's node's use list.
323 inline void set(const SDValue &V);
324 /// Like set, but only supports initializing a newly-allocated
325 /// SDUse with a non-null value.
326 inline void setInitial(const SDValue &V);
327 /// Like set, but only sets the Node portion of the value,
328 /// leaving the ResNo portion unmodified.
329 inline void setNode(SDNode *N);
330
331 void addToList(SDUse **List) {
332 Next = *List;
333 if (Next) Next->Prev = &Next;
334 Prev = List;
335 *List = this;
336 }
337
338 void removeFromList() {
339 *Prev = Next;
340 if (Next) Next->Prev = Prev;
341 }
342};
343
344/// simplify_type specializations - Allow casting operators to work directly on
345/// SDValues as if they were SDNode*'s.
346template<> struct simplify_type<SDUse> {
347 using SimpleType = SDNode *;
348
349 static SimpleType getSimplifiedValue(SDUse &Val) {
350 return Val.getNode();
351 }
352};
353
354/// These are IR-level optimization flags that may be propagated to SDNodes.
355/// TODO: This data structure should be shared by the IR optimizer and the
356/// the backend.
357struct SDNodeFlags {
358private:
359 // This bit is used to determine if the flags are in a defined state.
360 // Flag bits can only be masked out during intersection if the masking flags
361 // are defined.
362 bool AnyDefined : 1;
363
364 bool NoUnsignedWrap : 1;
365 bool NoSignedWrap : 1;
366 bool Exact : 1;
367 bool NoNaNs : 1;
368 bool NoInfs : 1;
369 bool NoSignedZeros : 1;
370 bool AllowReciprocal : 1;
371 bool VectorReduction : 1;
372 bool AllowContract : 1;
373 bool ApproximateFuncs : 1;
374 bool AllowReassociation : 1;
375
376 // We assume instructions do not raise floating-point exceptions by default,
377 // and only those marked explicitly may do so. We could choose to represent
378 // this via a positive "FPExcept" flags like on the MI level, but having a
379 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
380 // intersection logic more straightforward.
381 bool NoFPExcept : 1;
382
383public:
384 /// Default constructor turns off all optimization flags.
385 SDNodeFlags()
386 : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
387 Exact(false), NoNaNs(false), NoInfs(false),
388 NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
389 AllowContract(false), ApproximateFuncs(false),
390 AllowReassociation(false), NoFPExcept(false) {}
391
392 /// Propagate the fast-math-flags from an IR FPMathOperator.
393 void copyFMF(const FPMathOperator &FPMO) {
394 setNoNaNs(FPMO.hasNoNaNs());
395 setNoInfs(FPMO.hasNoInfs());
396 setNoSignedZeros(FPMO.hasNoSignedZeros());
397 setAllowReciprocal(FPMO.hasAllowReciprocal());
398 setAllowContract(FPMO.hasAllowContract());
399 setApproximateFuncs(FPMO.hasApproxFunc());
400 setAllowReassociation(FPMO.hasAllowReassoc());
401 }
402
403 /// Sets the state of the flags to the defined state.
404 void setDefined() { AnyDefined = true; }
405 /// Returns true if the flags are in a defined state.
406 bool isDefined() const { return AnyDefined; }
407
408 // These are mutators for each flag.
409 void setNoUnsignedWrap(bool b) {
410 setDefined();
411 NoUnsignedWrap = b;
412 }
413 void setNoSignedWrap(bool b) {
414 setDefined();
415 NoSignedWrap = b;
416 }
417 void setExact(bool b) {
418 setDefined();
419 Exact = b;
420 }
421 void setNoNaNs(bool b) {
422 setDefined();
423 NoNaNs = b;
424 }
425 void setNoInfs(bool b) {
426 setDefined();
427 NoInfs = b;
428 }
429 void setNoSignedZeros(bool b) {
430 setDefined();
431 NoSignedZeros = b;
432 }
433 void setAllowReciprocal(bool b) {
434 setDefined();
435 AllowReciprocal = b;
436 }
437 void setVectorReduction(bool b) {
438 setDefined();
439 VectorReduction = b;
440 }
441 void setAllowContract(bool b) {
442 setDefined();
443 AllowContract = b;
444 }
445 void setApproximateFuncs(bool b) {
446 setDefined();
447 ApproximateFuncs = b;
448 }
449 void setAllowReassociation(bool b) {
450 setDefined();
451 AllowReassociation = b;
452 }
453 void setNoFPExcept(bool b) {
454 setDefined();
455 NoFPExcept = b;
456 }
457
458 // These are accessors for each flag.
459 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
460 bool hasNoSignedWrap() const { return NoSignedWrap; }
461 bool hasExact() const { return Exact; }
462 bool hasNoNaNs() const { return NoNaNs; }
463 bool hasNoInfs() const { return NoInfs; }
464 bool hasNoSignedZeros() const { return NoSignedZeros; }
465 bool hasAllowReciprocal() const { return AllowReciprocal; }
466 bool hasVectorReduction() const { return VectorReduction; }
467 bool hasAllowContract() const { return AllowContract; }
468 bool hasApproximateFuncs() const { return ApproximateFuncs; }
469 bool hasAllowReassociation() const { return AllowReassociation; }
470 bool hasNoFPExcept() const { return NoFPExcept; }
471
472 /// Clear any flags in this flag set that aren't also set in Flags.
473 /// If the given Flags are undefined then don't do anything.
474 void intersectWith(const SDNodeFlags Flags) {
475 if (!Flags.isDefined())
476 return;
477 NoUnsignedWrap &= Flags.NoUnsignedWrap;
478 NoSignedWrap &= Flags.NoSignedWrap;
479 Exact &= Flags.Exact;
480 NoNaNs &= Flags.NoNaNs;
481 NoInfs &= Flags.NoInfs;
482 NoSignedZeros &= Flags.NoSignedZeros;
483 AllowReciprocal &= Flags.AllowReciprocal;
484 VectorReduction &= Flags.VectorReduction;
485 AllowContract &= Flags.AllowContract;
486 ApproximateFuncs &= Flags.ApproximateFuncs;
487 AllowReassociation &= Flags.AllowReassociation;
488 NoFPExcept &= Flags.NoFPExcept;
489 }
490};
491
492/// Represents one node in the SelectionDAG.
493///
494class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
495private:
496 /// The operation that this node performs.
497 int16_t NodeType;
498
499protected:
500 // We define a set of mini-helper classes to help us interpret the bits in our
501 // SubclassData. These are designed to fit within a uint16_t so they pack
502 // with NodeType.
503
504#if defined(_AIX) && (!defined(__GNUC__4) || defined(__ibmxl__))
505// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
506// and give the `pack` pragma push semantics.
507#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
508#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
509#else
510#define BEGIN_TWO_BYTE_PACK()
511#define END_TWO_BYTE_PACK()
512#endif
513
514BEGIN_TWO_BYTE_PACK()
515 class SDNodeBitfields {
516 friend class SDNode;
517 friend class MemIntrinsicSDNode;
518 friend class MemSDNode;
519 friend class SelectionDAG;
520
521 uint16_t HasDebugValue : 1;
522 uint16_t IsMemIntrinsic : 1;
523 uint16_t IsDivergent : 1;
524 };
525 enum { NumSDNodeBits = 3 };
526
527 class ConstantSDNodeBitfields {
528 friend class ConstantSDNode;
529
530 uint16_t : NumSDNodeBits;
531
532 uint16_t IsOpaque : 1;
533 };
534
535 class MemSDNodeBitfields {
536 friend class MemSDNode;
537 friend class MemIntrinsicSDNode;
538 friend class AtomicSDNode;
539
540 uint16_t : NumSDNodeBits;
541
542 uint16_t IsVolatile : 1;
543 uint16_t IsNonTemporal : 1;
544 uint16_t IsDereferenceable : 1;
545 uint16_t IsInvariant : 1;
546 };
547 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
548
549 class LSBaseSDNodeBitfields {
550 friend class LSBaseSDNode;
551 friend class MaskedLoadStoreSDNode;
552 friend class MaskedGatherScatterSDNode;
553
554 uint16_t : NumMemSDNodeBits;
555
556 // This storage is shared between disparate class hierarchies to hold an
557 // enumeration specific to the class hierarchy in use.
558 // LSBaseSDNode => enum ISD::MemIndexedMode
559 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
560 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
561 uint16_t AddressingMode : 3;
562 };
563 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
564
565 class LoadSDNodeBitfields {
566 friend class LoadSDNode;
567 friend class MaskedLoadSDNode;
568
569 uint16_t : NumLSBaseSDNodeBits;
570
571 uint16_t ExtTy : 2; // enum ISD::LoadExtType
572 uint16_t IsExpanding : 1;
573 };
574
575 class StoreSDNodeBitfields {
576 friend class StoreSDNode;
577 friend class MaskedStoreSDNode;
578
579 uint16_t : NumLSBaseSDNodeBits;
580
581 uint16_t IsTruncating : 1;
582 uint16_t IsCompressing : 1;
583 };
584
585 union {
586 char RawSDNodeBits[sizeof(uint16_t)];
587 SDNodeBitfields SDNodeBits;
588 ConstantSDNodeBitfields ConstantSDNodeBits;
589 MemSDNodeBitfields MemSDNodeBits;
590 LSBaseSDNodeBitfields LSBaseSDNodeBits;
591 LoadSDNodeBitfields LoadSDNodeBits;
592 StoreSDNodeBitfields StoreSDNodeBits;
593 };
594END_TWO_BYTE_PACK()
595#undef BEGIN_TWO_BYTE_PACK
596#undef END_TWO_BYTE_PACK
597
598 // RawSDNodeBits must cover the entirety of the union. This means that all of
599 // the union's members must have size <= RawSDNodeBits. We write the RHS as
600 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
601 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
602 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
603 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
604 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
605 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
606 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
607
608private:
609 friend class SelectionDAG;
610 // TODO: unfriend HandleSDNode once we fix its operand handling.
611 friend class HandleSDNode;
612
613 /// Unique id per SDNode in the DAG.
614 int NodeId = -1;
615
616 /// The values that are used by this operation.
617 SDUse *OperandList = nullptr;
618
619 /// The types of the values this node defines. SDNode's may
620 /// define multiple values simultaneously.
621 const EVT *ValueList;
622
623 /// List of uses for this SDNode.
624 SDUse *UseList = nullptr;
625
626 /// The number of entries in the Operand/Value list.
627 unsigned short NumOperands = 0;
628 unsigned short NumValues;
629
630 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
631 // original LLVM instructions.
632 // This is used for turning off scheduling, because we'll forgo
633 // the normal scheduling algorithms and output the instructions according to
634 // this ordering.
635 unsigned IROrder;
636
637 /// Source line information.
638 DebugLoc debugLoc;
639
640 /// Return a pointer to the specified value type.
641 static const EVT *getValueTypeList(EVT VT);
642
643 SDNodeFlags Flags;
644
645public:
646 /// Unique and persistent id per SDNode in the DAG.
647 /// Used for debug printing.
648 uint16_t PersistentId;
649
650 //===--------------------------------------------------------------------===//
651 // Accessors
652 //
653
654 /// Return the SelectionDAG opcode value for this node. For
655 /// pre-isel nodes (those for which isMachineOpcode returns false), these
656 /// are the opcode values in the ISD and <target>ISD namespaces. For
657 /// post-isel opcodes, see getMachineOpcode.
658 unsigned getOpcode() const { return (unsigned short)NodeType; }
659
660 /// Test if this node has a target-specific opcode (in the
661 /// \<target\>ISD namespace).
662 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
663
664 /// Test if this node has a target-specific opcode that may raise
665 /// FP exceptions (in the \<target\>ISD namespace and greater than
666 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
667 /// opcode are currently automatically considered to possibly raise
668 /// FP exceptions as well.
669 bool isTargetStrictFPOpcode() const {
670 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
671 }
672
673 /// Test if this node has a target-specific
674 /// memory-referencing opcode (in the \<target\>ISD namespace and
675 /// greater than FIRST_TARGET_MEMORY_OPCODE).
676 bool isTargetMemoryOpcode() const {
677 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
678 }
679
680 /// Return true if the type of the node type undefined.
681 bool isUndef() const { return NodeType == ISD::UNDEF; }
682
683 /// Test if this node is a memory intrinsic (with valid pointer information).
684 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
685 /// non-memory intrinsics (with chains) that are not really instances of
686 /// MemSDNode. For such nodes, we need some extra state to determine the
687 /// proper classof relationship.
688 bool isMemIntrinsic() const {
689 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
690 NodeType == ISD::INTRINSIC_VOID) &&
691 SDNodeBits.IsMemIntrinsic;
692 }
693
694 /// Test if this node is a strict floating point pseudo-op.
695 bool isStrictFPOpcode() {
696 switch (NodeType) {
697 default:
698 return false;
699 case ISD::STRICT_FP16_TO_FP:
700 case ISD::STRICT_FP_TO_FP16:
701#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
702 case ISD::STRICT_##DAGN:
703#include "llvm/IR/ConstrainedOps.def"
704 return true;
705 }
706 }
707
708 /// Test if this node has a post-isel opcode, directly
709 /// corresponding to a MachineInstr opcode.
710 bool isMachineOpcode() const { return NodeType < 0; }
711
712 /// This may only be called if isMachineOpcode returns
713 /// true. It returns the MachineInstr opcode value that the node's opcode
714 /// corresponds to.
715 unsigned getMachineOpcode() const {
716 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ?
static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 716, __PRETTY_FUNCTION__))
;
717 return ~NodeType;
718 }
719
720 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
721 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
722
723 bool isDivergent() const { return SDNodeBits.IsDivergent; }
724
725 /// Return true if there are no uses of this node.
726 bool use_empty() const { return UseList == nullptr; }
727
728 /// Return true if there is exactly one use of this node.
729 bool hasOneUse() const {
730 return !use_empty() && std::next(use_begin()) == use_end();
731 }
732
733 /// Return the number of uses of this node. This method takes
734 /// time proportional to the number of uses.
735 size_t use_size() const { return std::distance(use_begin(), use_end()); }
736
737 /// Return the unique node id.
738 int getNodeId() const { return NodeId; }
739
740 /// Set unique node id.
741 void setNodeId(int Id) { NodeId = Id; }
742
743 /// Return the node ordering.
744 unsigned getIROrder() const { return IROrder; }
745
746 /// Set the node ordering.
747 void setIROrder(unsigned Order) { IROrder = Order; }
748
749 /// Return the source location info.
750 const DebugLoc &getDebugLoc() const { return debugLoc; }
751
752 /// Set source location info. Try to avoid this, putting
753 /// it in the constructor is preferable.
754 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
755
756 /// This class provides iterator support for SDUse
757 /// operands that use a specific SDNode.
758 class use_iterator
759 : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
760 friend class SDNode;
761
762 SDUse *Op = nullptr;
763
764 explicit use_iterator(SDUse *op) : Op(op) {}
765
766 public:
767 using reference = std::iterator<std::forward_iterator_tag,
768 SDUse, ptrdiff_t>::reference;
769 using pointer = std::iterator<std::forward_iterator_tag,
770 SDUse, ptrdiff_t>::pointer;
771
772 use_iterator() = default;
773 use_iterator(const use_iterator &I) : Op(I.Op) {}
774
775 bool operator==(const use_iterator &x) const {
776 return Op == x.Op;
777 }
778 bool operator!=(const use_iterator &x) const {
779 return !operator==(x);
780 }
781
782 /// Return true if this iterator is at the end of uses list.
783 bool atEnd() const { return Op == nullptr; }
784
785 // Iterator traversal: forward iteration only.
786 use_iterator &operator++() { // Preincrement
787 assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 787, __PRETTY_FUNCTION__))
;
788 Op = Op->getNext();
789 return *this;
790 }
791
792 use_iterator operator++(int) { // Postincrement
793 use_iterator tmp = *this; ++*this; return tmp;
794 }
795
796 /// Retrieve a pointer to the current user node.
797 SDNode *operator*() const {
798 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 798, __PRETTY_FUNCTION__))
;
799 return Op->getUser();
800 }
801
802 SDNode *operator->() const { return operator*(); }
803
804 SDUse &getUse() const { return *Op; }
805
806 /// Retrieve the operand # of this use in its user.
807 unsigned getOperandNo() const {
808 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 808, __PRETTY_FUNCTION__))
;
809 return (unsigned)(Op - Op->getUser()->OperandList);
810 }
811 };
812
813 /// Provide iteration support to walk over all uses of an SDNode.
814 use_iterator use_begin() const {
815 return use_iterator(UseList);
816 }
817
818 static use_iterator use_end() { return use_iterator(nullptr); }
819
820 inline iterator_range<use_iterator> uses() {
821 return make_range(use_begin(), use_end());
822 }
823 inline iterator_range<use_iterator> uses() const {
824 return make_range(use_begin(), use_end());
825 }
826
827 /// Return true if there are exactly NUSES uses of the indicated value.
828 /// This method ignores uses of other values defined by this operation.
829 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
830
831 /// Return true if there are any use of the indicated value.
832 /// This method ignores uses of other values defined by this operation.
833 bool hasAnyUseOfValue(unsigned Value) const;
834
835 /// Return true if this node is the only use of N.
836 bool isOnlyUserOf(const SDNode *N) const;
837
838 /// Return true if this node is an operand of N.
839 bool isOperandOf(const SDNode *N) const;
840
841 /// Return true if this node is a predecessor of N.
842 /// NOTE: Implemented on top of hasPredecessor and every bit as
843 /// expensive. Use carefully.
844 bool isPredecessorOf(const SDNode *N) const {
845 return N->hasPredecessor(this);
846 }
847
848 /// Return true if N is a predecessor of this node.
849 /// N is either an operand of this node, or can be reached by recursively
850 /// traversing up the operands.
851 /// NOTE: This is an expensive method. Use it carefully.
852 bool hasPredecessor(const SDNode *N) const;
853
854 /// Returns true if N is a predecessor of any node in Worklist. This
855 /// helper keeps Visited and Worklist sets externally to allow unions
856 /// searches to be performed in parallel, caching of results across
857 /// queries and incremental addition to Worklist. Stops early if N is
858 /// found but will resume. Remember to clear Visited and Worklists
859 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
860 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
861 /// topologically ordered (Operands have strictly smaller node id) and search
862 /// can be pruned leveraging this.
863 static bool hasPredecessorHelper(const SDNode *N,
864 SmallPtrSetImpl<const SDNode *> &Visited,
865 SmallVectorImpl<const SDNode *> &Worklist,
866 unsigned int MaxSteps = 0,
867 bool TopologicalPrune = false) {
868 SmallVector<const SDNode *, 8> DeferredNodes;
869 if (Visited.count(N))
870 return true;
871
872 // Node Id's are assigned in three places: As a topological
873 // ordering (> 0), during legalization (results in values set to
874 // 0), new nodes (set to -1). If N has a topolgical id then we
875 // know that all nodes with ids smaller than it cannot be
876 // successors and we need not check them. Filter out all node
877 // that can't be matches. We add them to the worklist before exit
878 // in case of multiple calls. Note that during selection the topological id
879 // may be violated if a node's predecessor is selected before it. We mark
880 // this at selection negating the id of unselected successors and
881 // restricting topological pruning to positive ids.
882
883 int NId = N->getNodeId();
884 // If we Invalidated the Id, reconstruct original NId.
885 if (NId < -1)
886 NId = -(NId + 1);
887
888 bool Found = false;
889 while (!Worklist.empty()) {
890 const SDNode *M = Worklist.pop_back_val();
891 int MId = M->getNodeId();
892 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
893 (MId > 0) && (MId < NId)) {
894 DeferredNodes.push_back(M);
895 continue;
896 }
897 for (const SDValue &OpV : M->op_values()) {
898 SDNode *Op = OpV.getNode();
899 if (Visited.insert(Op).second)
900 Worklist.push_back(Op);
901 if (Op == N)
902 Found = true;
903 }
904 if (Found)
905 break;
906 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
907 break;
908 }
909 // Push deferred nodes back on worklist.
910 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
911 // If we bailed early, conservatively return found.
912 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
913 return true;
914 return Found;
915 }
916
917 /// Return true if all the users of N are contained in Nodes.
918 /// NOTE: Requires at least one match, but doesn't require them all.
919 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
920
921 /// Return the number of values used by this operation.
922 unsigned getNumOperands() const { return NumOperands; }
923
924 /// Return the maximum number of operands that a SDNode can hold.
925 static constexpr size_t getMaxNumOperands() {
926 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
927 }
928
929 /// Helper method returns the integer value of a ConstantSDNode operand.
930 inline uint64_t getConstantOperandVal(unsigned Num) const;
931
932 /// Helper method returns the APInt of a ConstantSDNode operand.
933 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
934
935 const SDValue &getOperand(unsigned Num) const {
936 assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!"
) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 936, __PRETTY_FUNCTION__))
;
937 return OperandList[Num];
938 }
939
940 using op_iterator = SDUse *;
941
942 op_iterator op_begin() const { return OperandList; }
943 op_iterator op_end() const { return OperandList+NumOperands; }
944 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
945
946 /// Iterator for directly iterating over the operand SDValue's.
947 struct value_op_iterator
948 : iterator_adaptor_base<value_op_iterator, op_iterator,
949 std::random_access_iterator_tag, SDValue,
950 ptrdiff_t, value_op_iterator *,
951 value_op_iterator *> {
952 explicit value_op_iterator(SDUse *U = nullptr)
953 : iterator_adaptor_base(U) {}
954
955 const SDValue &operator*() const { return I->get(); }
956 };
957
958 iterator_range<value_op_iterator> op_values() const {
959 return make_range(value_op_iterator(op_begin()),
960 value_op_iterator(op_end()));
961 }
962
963 SDVTList getVTList() const {
964 SDVTList X = { ValueList, NumValues };
965 return X;
966 }
967
968 /// If this node has a glue operand, return the node
969 /// to which the glue operand points. Otherwise return NULL.
970 SDNode *getGluedNode() const {
971 if (getNumOperands() != 0 &&
972 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
973 return getOperand(getNumOperands()-1).getNode();
974 return nullptr;
975 }
976
977 /// If this node has a glue value with a user, return
978 /// the user (there is at most one). Otherwise return NULL.
979 SDNode *getGluedUser() const {
980 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
981 if (UI.getUse().get().getValueType() == MVT::Glue)
982 return *UI;
983 return nullptr;
984 }
985
986 const SDNodeFlags getFlags() const { return Flags; }
987 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
988
989 /// Clear any flags in this node that aren't also set in Flags.
990 /// If Flags is not in a defined state then this has no effect.
991 void intersectFlagsWith(const SDNodeFlags Flags);
992
993 /// Return the number of values defined/returned by this operator.
994 unsigned getNumValues() const { return NumValues; }
995
996 /// Return the type of a specified result.
997 EVT getValueType(unsigned ResNo) const {
998 assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ?
static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 998, __PRETTY_FUNCTION__))
;
999 return ValueList[ResNo];
1000 }
1001
1002 /// Return the type of a specified result as a simple type.
1003 MVT getSimpleValueType(unsigned ResNo) const {
1004 return getValueType(ResNo).getSimpleVT();
1005 }
1006
1007 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
1008 ///
1009 /// If the value type is a scalable vector type, the scalable property will
1010 /// be set and the runtime size will be a positive integer multiple of the
1011 /// base size.
1012 TypeSize getValueSizeInBits(unsigned ResNo) const {
1013 return getValueType(ResNo).getSizeInBits();
1014 }
1015
1016 using value_iterator = const EVT *;
1017
1018 value_iterator value_begin() const { return ValueList; }
1019 value_iterator value_end() const { return ValueList+NumValues; }
1020 iterator_range<value_iterator> values() const {
1021 return llvm::make_range(value_begin(), value_end());
1022 }
1023
1024 /// Return the opcode of this operation for printing.
1025 std::string getOperationName(const SelectionDAG *G = nullptr) const;
1026 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
1027 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
1028 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
1029 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1030 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1031
1032 /// Print a SelectionDAG node and all children down to
1033 /// the leaves. The given SelectionDAG allows target-specific nodes
1034 /// to be printed in human-readable form. Unlike printr, this will
1035 /// print the whole DAG, including children that appear multiple
1036 /// times.
1037 ///
1038 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1039
1040 /// Print a SelectionDAG node and children up to
1041 /// depth "depth." The given SelectionDAG allows target-specific
1042 /// nodes to be printed in human-readable form. Unlike printr, this
1043 /// will print children that appear multiple times wherever they are
1044 /// used.
1045 ///
1046 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1047 unsigned depth = 100) const;
1048
1049 /// Dump this node, for debugging.
1050 void dump() const;
1051
1052 /// Dump (recursively) this node and its use-def subgraph.
1053 void dumpr() const;
1054
1055 /// Dump this node, for debugging.
1056 /// The given SelectionDAG allows target-specific nodes to be printed
1057 /// in human-readable form.
1058 void dump(const SelectionDAG *G) const;
1059
1060 /// Dump (recursively) this node and its use-def subgraph.
1061 /// The given SelectionDAG allows target-specific nodes to be printed
1062 /// in human-readable form.
1063 void dumpr(const SelectionDAG *G) const;
1064
1065 /// printrFull to dbgs(). The given SelectionDAG allows
1066 /// target-specific nodes to be printed in human-readable form.
1067 /// Unlike dumpr, this will print the whole DAG, including children
1068 /// that appear multiple times.
1069 void dumprFull(const SelectionDAG *G = nullptr) const;
1070
1071 /// printrWithDepth to dbgs(). The given
1072 /// SelectionDAG allows target-specific nodes to be printed in
1073 /// human-readable form. Unlike dumpr, this will print children
1074 /// that appear multiple times wherever they are used.
1075 ///
1076 void dumprWithDepth(const SelectionDAG *G = nullptr,
1077 unsigned depth = 100) const;
1078
1079 /// Gather unique data for the node.
1080 void Profile(FoldingSetNodeID &ID) const;
1081
1082 /// This method should only be used by the SDUse class.
1083 void addUse(SDUse &U) { U.addToList(&UseList); }
1084
1085protected:
1086 static SDVTList getSDVTList(EVT VT) {
1087 SDVTList Ret = { getValueTypeList(VT), 1 };
1088 return Ret;
1089 }
1090
1091 /// Create an SDNode.
1092 ///
1093 /// SDNodes are created without any operands, and never own the operand
1094 /// storage. To add operands, see SelectionDAG::createOperands.
1095 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1096 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1097 IROrder(Order), debugLoc(std::move(dl)) {
1098 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1099 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1099, __PRETTY_FUNCTION__))
;
1100 assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1101, __PRETTY_FUNCTION__))
1101 "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1101, __PRETTY_FUNCTION__))
;
1102 }
1103
1104 /// Release the operands and set this node to have zero operands.
1105 void DropOperands();
1106};
1107
1108/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1109/// into SDNode creation functions.
1110/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1111/// from the original Instruction, and IROrder is the ordinal position of
1112/// the instruction.
1113/// When an SDNode is created after the DAG is being built, both DebugLoc and
1114/// the IROrder are propagated from the original SDNode.
1115/// So SDLoc class provides two constructors besides the default one, one to
1116/// be used by the DAGBuilder, the other to be used by others.
1117class SDLoc {
1118private:
1119 DebugLoc DL;
1120 int IROrder = 0;
1121
1122public:
1123 SDLoc() = default;
1124 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1125 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1126 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1127 assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void
> (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1127, __PRETTY_FUNCTION__))
;
1128 if (I)
1129 DL = I->getDebugLoc();
1130 }
1131
1132 unsigned getIROrder() const { return IROrder; }
1133 const DebugLoc &getDebugLoc() const { return DL; }
1134};
1135
1136// Define inline functions from the SDValue class.
1137
1138inline SDValue::SDValue(SDNode *node, unsigned resno)
1139 : Node(node), ResNo(resno) {
1140 // Explicitly check for !ResNo to avoid use-after-free, because there are
1141 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1142 // combines.
1143 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1144, __PRETTY_FUNCTION__))
1144 "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1144, __PRETTY_FUNCTION__))
;
1145 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1145, __PRETTY_FUNCTION__))
;
1146}
1147
1148inline unsigned SDValue::getOpcode() const {
1149 return Node->getOpcode();
1150}
1151
1152inline EVT SDValue::getValueType() const {
1153 return Node->getValueType(ResNo);
9
Called C++ object pointer is null
1154}
1155
1156inline unsigned SDValue::getNumOperands() const {
1157 return Node->getNumOperands();
1158}
1159
1160inline const SDValue &SDValue::getOperand(unsigned i) const {
1161 return Node->getOperand(i);
1162}
1163
1164inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1165 return Node->getConstantOperandVal(i);
1166}
1167
1168inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1169 return Node->getConstantOperandAPInt(i);
1170}
1171
1172inline bool SDValue::isTargetOpcode() const {
1173 return Node->isTargetOpcode();
1174}
1175
1176inline bool SDValue::isTargetMemoryOpcode() const {
1177 return Node->isTargetMemoryOpcode();
1178}
1179
1180inline bool SDValue::isMachineOpcode() const {
1181 return Node->isMachineOpcode();
1182}
1183
1184inline unsigned SDValue::getMachineOpcode() const {
1185 return Node->getMachineOpcode();
1186}
1187
1188inline bool SDValue::isUndef() const {
1189 return Node->isUndef();
1190}
1191
1192inline bool SDValue::use_empty() const {
1193 return !Node->hasAnyUseOfValue(ResNo);
1194}
1195
1196inline bool SDValue::hasOneUse() const {
1197 return Node->hasNUsesOfValue(1, ResNo);
1198}
1199
1200inline const DebugLoc &SDValue::getDebugLoc() const {
1201 return Node->getDebugLoc();
1202}
1203
1204inline void SDValue::dump() const {
1205 return Node->dump();
1206}
1207
1208inline void SDValue::dump(const SelectionDAG *G) const {
1209 return Node->dump(G);
1210}
1211
1212inline void SDValue::dumpr() const {
1213 return Node->dumpr();
1214}
1215
1216inline void SDValue::dumpr(const SelectionDAG *G) const {
1217 return Node->dumpr(G);
1218}
1219
1220// Define inline functions from the SDUse class.
1221
1222inline void SDUse::set(const SDValue &V) {
1223 if (Val.getNode()) removeFromList();
1224 Val = V;
1225 if (V.getNode()) V.getNode()->addUse(*this);
1226}
1227
1228inline void SDUse::setInitial(const SDValue &V) {
1229 Val = V;
1230 V.getNode()->addUse(*this);
1231}
1232
1233inline void SDUse::setNode(SDNode *N) {
1234 if (Val.getNode()) removeFromList();
1235 Val.setNode(N);
1236 if (N) N->addUse(*this);
1237}
1238
1239/// This class is used to form a handle around another node that
1240/// is persistent and is updated across invocations of replaceAllUsesWith on its
1241/// operand. This node should be directly created by end-users and not added to
1242/// the AllNodes list.
1243class HandleSDNode : public SDNode {
1244 SDUse Op;
1245
1246public:
1247 explicit HandleSDNode(SDValue X)
1248 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1249 // HandleSDNodes are never inserted into the DAG, so they won't be
1250 // auto-numbered. Use ID 65535 as a sentinel.
1251 PersistentId = 0xffff;
1252
1253 // Manually set up the operand list. This node type is special in that it's
1254 // always stack allocated and SelectionDAG does not manage its operands.
1255 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1256 // be so special.
1257 Op.setUser(this);
1258 Op.setInitial(X);
1259 NumOperands = 1;
1260 OperandList = &Op;
1261 }
1262 ~HandleSDNode();
1263
1264 const SDValue &getValue() const { return Op; }
1265};
1266
1267class AddrSpaceCastSDNode : public SDNode {
1268private:
1269 unsigned SrcAddrSpace;
1270 unsigned DestAddrSpace;
1271
1272public:
1273 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1274 unsigned SrcAS, unsigned DestAS);
1275
1276 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1277 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1278
1279 static bool classof(const SDNode *N) {
1280 return N->getOpcode() == ISD::ADDRSPACECAST;
1281 }
1282};
1283
1284/// This is an abstract virtual class for memory operations.
1285class MemSDNode : public SDNode {
1286private:
1287 // VT of in-memory value.
1288 EVT MemoryVT;
1289
1290protected:
1291 /// Memory reference information.
1292 MachineMemOperand *MMO;
1293
1294public:
1295 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1296 EVT memvt, MachineMemOperand *MMO);
1297
1298 bool readMem() const { return MMO->isLoad(); }
1299 bool writeMem() const { return MMO->isStore(); }
1300
1301 /// Returns alignment and volatility of the memory access
1302 unsigned getOriginalAlignment() const {
1303 return MMO->getBaseAlignment();
1304 }
1305 unsigned getAlignment() const {
1306 return MMO->getAlignment();
1307 }
1308
1309 /// Return the SubclassData value, without HasDebugValue. This contains an
1310 /// encoding of the volatile flag, as well as bits used by subclasses. This
1311 /// function should only be used to compute a FoldingSetNodeID value.
1312 /// The HasDebugValue bit is masked out because CSE map needs to match
1313 /// nodes with debug info with nodes without debug info. Same is about
1314 /// isDivergent bit.
1315 unsigned getRawSubclassData() const {
1316 uint16_t Data;
1317 union {
1318 char RawSDNodeBits[sizeof(uint16_t)];
1319 SDNodeBitfields SDNodeBits;
1320 };
1321 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1322 SDNodeBits.HasDebugValue = 0;
1323 SDNodeBits.IsDivergent = false;
1324 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1325 return Data;
1326 }
1327
1328 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1329 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1330 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1331 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1332
1333 // Returns the offset from the location of the access.
1334 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1335
1336 /// Returns the AA info that describes the dereference.
1337 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1338
1339 /// Returns the Ranges that describes the dereference.
1340 const MDNode *getRanges() const { return MMO->getRanges(); }
1341
1342 /// Returns the synchronization scope ID for this memory operation.
1343 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1344
1345 /// Return the atomic ordering requirements for this memory operation. For
1346 /// cmpxchg atomic operations, return the atomic ordering requirements when
1347 /// store occurs.
1348 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
1349
1350 /// Return true if the memory operation ordering is Unordered or higher.
1351 bool isAtomic() const { return MMO->isAtomic(); }
1352
1353 /// Returns true if the memory operation doesn't imply any ordering
1354 /// constraints on surrounding memory operations beyond the normal memory
1355 /// aliasing rules.
1356 bool isUnordered() const { return MMO->isUnordered(); }
1357
1358 /// Returns true if the memory operation is neither atomic or volatile.
1359 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1360
1361 /// Return the type of the in-memory value.
1362 EVT getMemoryVT() const { return MemoryVT; }
1363
1364 /// Return a MachineMemOperand object describing the memory
1365 /// reference performed by operation.
1366 MachineMemOperand *getMemOperand() const { return MMO; }
1367
1368 const MachinePointerInfo &getPointerInfo() const {
1369 return MMO->getPointerInfo();
1370 }
1371
1372 /// Return the address space for the associated pointer
1373 unsigned getAddressSpace() const {
1374 return getPointerInfo().getAddrSpace();
1375 }
1376
1377 /// Update this MemSDNode's MachineMemOperand information
1378 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1379 /// This must only be used when the new alignment applies to all users of
1380 /// this MachineMemOperand.
1381 void refineAlignment(const MachineMemOperand *NewMMO) {
1382 MMO->refineAlignment(NewMMO);
1383 }
1384
1385 const SDValue &getChain() const { return getOperand(0); }
1386 const SDValue &getBasePtr() const {
1387 return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
1388 }
1389
1390 // Methods to support isa and dyn_cast
1391 static bool classof(const SDNode *N) {
1392 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1393 // with either an intrinsic or a target opcode.
1394 return N->getOpcode() == ISD::LOAD ||
1395 N->getOpcode() == ISD::STORE ||
1396 N->getOpcode() == ISD::PREFETCH ||
1397 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1398 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1399 N->getOpcode() == ISD::ATOMIC_SWAP ||
1400 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1401 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1402 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1403 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1404 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1405 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1406 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1407 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1408 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1409 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1410 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1411 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1412 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1413 N->getOpcode() == ISD::ATOMIC_LOAD ||
1414 N->getOpcode() == ISD::ATOMIC_STORE ||
1415 N->getOpcode() == ISD::MLOAD ||
1416 N->getOpcode() == ISD::MSTORE ||
1417 N->getOpcode() == ISD::MGATHER ||
1418 N->getOpcode() == ISD::MSCATTER ||
1419 N->isMemIntrinsic() ||
1420 N->isTargetMemoryOpcode();
1421 }
1422};
1423
1424/// This is an SDNode representing atomic operations.
1425class AtomicSDNode : public MemSDNode {
1426public:
1427 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1428 EVT MemVT, MachineMemOperand *MMO)
1429 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1430 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1431, __PRETTY_FUNCTION__))
1431 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1431, __PRETTY_FUNCTION__))
;
1432 }
1433
1434 const SDValue &getBasePtr() const { return getOperand(1); }
1435 const SDValue &getVal() const { return getOperand(2); }
1436
1437 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1438 /// otherwise.
1439 bool isCompareAndSwap() const {
1440 unsigned Op = getOpcode();
1441 return Op == ISD::ATOMIC_CMP_SWAP ||
1442 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1443 }
1444
1445 /// For cmpxchg atomic operations, return the atomic ordering requirements
1446 /// when store does not occur.
1447 AtomicOrdering getFailureOrdering() const {
1448 assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ?
static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1448, __PRETTY_FUNCTION__))
;
1449 return MMO->getFailureOrdering();
1450 }
1451
1452 // Methods to support isa and dyn_cast
1453 static bool classof(const SDNode *N) {
1454 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1455 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1456 N->getOpcode() ==