Bug Summary

File:include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1110, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name AArch64ISelDAGToDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64 -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Target/AArch64 -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an instruction selector for the AArch64 target.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AArch64TargetMachine.h"
15#include "MCTargetDesc/AArch64AddressingModes.h"
16#include "llvm/ADT/APSInt.h"
17#include "llvm/CodeGen/SelectionDAGISel.h"
18#include "llvm/IR/Function.h" // To access function attributes.
19#include "llvm/IR/GlobalValue.h"
20#include "llvm/IR/Intrinsics.h"
21#include "llvm/Support/Debug.h"
22#include "llvm/Support/ErrorHandling.h"
23#include "llvm/Support/KnownBits.h"
24#include "llvm/Support/MathExtras.h"
25#include "llvm/Support/raw_ostream.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE"aarch64-isel" "aarch64-isel"
30
31//===--------------------------------------------------------------------===//
32/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
33/// instructions for SelectionDAG operations.
34///
35namespace {
36
37class AArch64DAGToDAGISel : public SelectionDAGISel {
38
39 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
40 /// make the right decision when generating code for different targets.
41 const AArch64Subtarget *Subtarget;
42
43 bool ForCodeSize;
44
45public:
46 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
47 CodeGenOpt::Level OptLevel)
48 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr),
49 ForCodeSize(false) {}
50
51 StringRef getPassName() const override {
52 return "AArch64 Instruction Selection";
53 }
54
55 bool runOnMachineFunction(MachineFunction &MF) override {
56 ForCodeSize = MF.getFunction().optForSize();
57 Subtarget = &MF.getSubtarget<AArch64Subtarget>();
58 return SelectionDAGISel::runOnMachineFunction(MF);
59 }
60
61 void Select(SDNode *Node) override;
62
63 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
64 /// inline asm expressions.
65 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
66 unsigned ConstraintID,
67 std::vector<SDValue> &OutOps) override;
68
69 bool tryMLAV64LaneV128(SDNode *N);
70 bool tryMULLV64LaneV128(unsigned IntNo, SDNode *N);
71 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
72 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
73 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
74 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
75 return SelectShiftedRegister(N, false, Reg, Shift);
76 }
77 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
78 return SelectShiftedRegister(N, true, Reg, Shift);
79 }
80 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
81 return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
82 }
83 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
84 return SelectAddrModeIndexed7S(N, 2, Base, OffImm);
85 }
86 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
87 return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
88 }
89 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
90 return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
91 }
92 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
93 return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
94 }
95 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
96 return SelectAddrModeIndexed(N, 1, Base, OffImm);
97 }
98 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
99 return SelectAddrModeIndexed(N, 2, Base, OffImm);
100 }
101 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
102 return SelectAddrModeIndexed(N, 4, Base, OffImm);
103 }
104 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
105 return SelectAddrModeIndexed(N, 8, Base, OffImm);
106 }
107 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
108 return SelectAddrModeIndexed(N, 16, Base, OffImm);
109 }
110 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
111 return SelectAddrModeUnscaled(N, 1, Base, OffImm);
112 }
113 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
114 return SelectAddrModeUnscaled(N, 2, Base, OffImm);
115 }
116 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
117 return SelectAddrModeUnscaled(N, 4, Base, OffImm);
118 }
119 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
120 return SelectAddrModeUnscaled(N, 8, Base, OffImm);
121 }
122 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
123 return SelectAddrModeUnscaled(N, 16, Base, OffImm);
124 }
125
126 template<int Width>
127 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
128 SDValue &SignExtend, SDValue &DoShift) {
129 return SelectAddrModeWRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
130 }
131
132 template<int Width>
133 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
134 SDValue &SignExtend, SDValue &DoShift) {
135 return SelectAddrModeXRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
136 }
137
138
139 /// Form sequences of consecutive 64/128-bit registers for use in NEON
140 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
141 /// between 1 and 4 elements. If it contains a single element that is returned
142 /// unchanged; otherwise a REG_SEQUENCE value is returned.
143 SDValue createDTuple(ArrayRef<SDValue> Vecs);
144 SDValue createQTuple(ArrayRef<SDValue> Vecs);
145
146 /// Generic helper for the createDTuple/createQTuple
147 /// functions. Those should almost always be called instead.
148 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
149 const unsigned SubRegs[]);
150
151 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
152
153 bool tryIndexedLoad(SDNode *N);
154
155 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
156 unsigned SubRegIdx);
157 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
158 unsigned SubRegIdx);
159 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
160 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
161
162 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
163 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
164 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
165 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
166
167 bool tryBitfieldExtractOp(SDNode *N);
168 bool tryBitfieldExtractOpFromSExt(SDNode *N);
169 bool tryBitfieldInsertOp(SDNode *N);
170 bool tryBitfieldInsertInZeroOp(SDNode *N);
171 bool tryShiftAmountMod(SDNode *N);
172
173 bool tryReadRegister(SDNode *N);
174 bool tryWriteRegister(SDNode *N);
175
176// Include the pieces autogenerated from the target description.
177#include "AArch64GenDAGISel.inc"
178
179private:
180 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
181 SDValue &Shift);
182 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
183 SDValue &OffImm);
184 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
185 SDValue &OffImm);
186 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
187 SDValue &OffImm);
188 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
189 SDValue &Offset, SDValue &SignExtend,
190 SDValue &DoShift);
191 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
192 SDValue &Offset, SDValue &SignExtend,
193 SDValue &DoShift);
194 bool isWorthFolding(SDValue V) const;
195 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
196 SDValue &Offset, SDValue &SignExtend);
197
198 template<unsigned RegWidth>
199 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
200 return SelectCVTFixedPosOperand(N, FixedPos, RegWidth);
201 }
202
203 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
204
205 bool SelectCMP_SWAP(SDNode *N);
206
207};
208} // end anonymous namespace
209
210/// isIntImmediate - This method tests to see if the node is a constant
211/// operand. If so Imm will receive the 32-bit value.
212static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
213 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(N)) {
214 Imm = C->getZExtValue();
215 return true;
216 }
217 return false;
218}
219
220// isIntImmediate - This method tests to see if a constant operand.
221// If so Imm will receive the value.
222static bool isIntImmediate(SDValue N, uint64_t &Imm) {
223 return isIntImmediate(N.getNode(), Imm);
224}
225
226// isOpcWithIntImmediate - This method tests to see if the node is a specific
227// opcode and that it has a immediate integer right operand.
228// If so Imm will receive the 32 bit value.
229static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
230 uint64_t &Imm) {
231 return N->getOpcode() == Opc &&
232 isIntImmediate(N->getOperand(1).getNode(), Imm);
233}
234
235bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
236 const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
237 switch(ConstraintID) {
238 default:
239 llvm_unreachable("Unexpected asm memory constraint")::llvm::llvm_unreachable_internal("Unexpected asm memory constraint"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 239)
;
240 case InlineAsm::Constraint_i:
241 case InlineAsm::Constraint_m:
242 case InlineAsm::Constraint_Q:
243 // We need to make sure that this one operand does not end up in XZR, thus
244 // require the address to be in a PointerRegClass register.
245 const TargetRegisterInfo *TRI = Subtarget->getRegisterInfo();
246 const TargetRegisterClass *TRC = TRI->getPointerRegClass(*MF);
247 SDLoc dl(Op);
248 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64);
249 SDValue NewOp =
250 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
251 dl, Op.getValueType(),
252 Op, RC), 0);
253 OutOps.push_back(NewOp);
254 return false;
255 }
256 return true;
257}
258
259/// SelectArithImmed - Select an immediate value that can be represented as
260/// a 12-bit value shifted left by either 0 or 12. If so, return true with
261/// Val set to the 12-bit value and Shift set to the shifter operand.
262bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
263 SDValue &Shift) {
264 // This function is called from the addsub_shifted_imm ComplexPattern,
265 // which lists [imm] as the list of opcode it's interested in, however
266 // we still need to check whether the operand is actually an immediate
267 // here because the ComplexPattern opcode list is only used in
268 // root-level opcode matching.
269 if (!isa<ConstantSDNode>(N.getNode()))
270 return false;
271
272 uint64_t Immed = cast<ConstantSDNode>(N.getNode())->getZExtValue();
273 unsigned ShiftAmt;
274
275 if (Immed >> 12 == 0) {
276 ShiftAmt = 0;
277 } else if ((Immed & 0xfff) == 0 && Immed >> 24 == 0) {
278 ShiftAmt = 12;
279 Immed = Immed >> 12;
280 } else
281 return false;
282
283 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
284 SDLoc dl(N);
285 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
286 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
287 return true;
288}
289
290/// SelectNegArithImmed - As above, but negates the value before trying to
291/// select it.
292bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
293 SDValue &Shift) {
294 // This function is called from the addsub_shifted_imm ComplexPattern,
295 // which lists [imm] as the list of opcode it's interested in, however
296 // we still need to check whether the operand is actually an immediate
297 // here because the ComplexPattern opcode list is only used in
298 // root-level opcode matching.
299 if (!isa<ConstantSDNode>(N.getNode()))
300 return false;
301
302 // The immediate operand must be a 24-bit zero-extended immediate.
303 uint64_t Immed = cast<ConstantSDNode>(N.getNode())->getZExtValue();
304
305 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
306 // have the opposite effect on the C flag, so this pattern mustn't match under
307 // those circumstances.
308 if (Immed == 0)
309 return false;
310
311 if (N.getValueType() == MVT::i32)
312 Immed = ~((uint32_t)Immed) + 1;
313 else
314 Immed = ~Immed + 1ULL;
315 if (Immed & 0xFFFFFFFFFF000000ULL)
316 return false;
317
318 Immed &= 0xFFFFFFULL;
319 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val,
320 Shift);
321}
322
323/// getShiftTypeForNode - Translate a shift node to the corresponding
324/// ShiftType value.
325static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N) {
326 switch (N.getOpcode()) {
327 default:
328 return AArch64_AM::InvalidShiftExtend;
329 case ISD::SHL:
330 return AArch64_AM::LSL;
331 case ISD::SRL:
332 return AArch64_AM::LSR;
333 case ISD::SRA:
334 return AArch64_AM::ASR;
335 case ISD::ROTR:
336 return AArch64_AM::ROR;
337 }
338}
339
340/// Determine whether it is worth it to fold SHL into the addressing
341/// mode.
342static bool isWorthFoldingSHL(SDValue V) {
343 assert(V.getOpcode() == ISD::SHL && "invalid opcode")((V.getOpcode() == ISD::SHL && "invalid opcode") ? static_cast
<void> (0) : __assert_fail ("V.getOpcode() == ISD::SHL && \"invalid opcode\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 343, __PRETTY_FUNCTION__))
;
344 // It is worth folding logical shift of up to three places.
345 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
346 if (!CSD)
347 return false;
348 unsigned ShiftVal = CSD->getZExtValue();
349 if (ShiftVal > 3)
350 return false;
351
352 // Check if this particular node is reused in any non-memory related
353 // operation. If yes, do not try to fold this node into the address
354 // computation, since the computation will be kept.
355 const SDNode *Node = V.getNode();
356 for (SDNode *UI : Node->uses())
357 if (!isa<MemSDNode>(*UI))
358 for (SDNode *UII : UI->uses())
359 if (!isa<MemSDNode>(*UII))
360 return false;
361 return true;
362}
363
364/// Determine whether it is worth to fold V into an extended register.
365bool AArch64DAGToDAGISel::isWorthFolding(SDValue V) const {
366 // Trivial if we are optimizing for code size or if there is only
367 // one use of the value.
368 if (ForCodeSize || V.hasOneUse())
369 return true;
370 // If a subtarget has a fastpath LSL we can fold a logical shift into
371 // the addressing mode and save a cycle.
372 if (Subtarget->hasLSLFast() && V.getOpcode() == ISD::SHL &&
373 isWorthFoldingSHL(V))
374 return true;
375 if (Subtarget->hasLSLFast() && V.getOpcode() == ISD::ADD) {
376 const SDValue LHS = V.getOperand(0);
377 const SDValue RHS = V.getOperand(1);
378 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(LHS))
379 return true;
380 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(RHS))
381 return true;
382 }
383
384 // It hurts otherwise, since the value will be reused.
385 return false;
386}
387
388/// SelectShiftedRegister - Select a "shifted register" operand. If the value
389/// is not shifted, set the Shift operand to default of "LSL 0". The logical
390/// instructions allow the shifted register to be rotated, but the arithmetic
391/// instructions do not. The AllowROR parameter specifies whether ROR is
392/// supported.
393bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
394 SDValue &Reg, SDValue &Shift) {
395 AArch64_AM::ShiftExtendType ShType = getShiftTypeForNode(N);
396 if (ShType == AArch64_AM::InvalidShiftExtend)
397 return false;
398 if (!AllowROR && ShType == AArch64_AM::ROR)
399 return false;
400
401 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
402 unsigned BitSize = N.getValueSizeInBits();
403 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
404 unsigned ShVal = AArch64_AM::getShifterImm(ShType, Val);
405
406 Reg = N.getOperand(0);
407 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32);
408 return isWorthFolding(N);
409 }
410
411 return false;
412}
413
414/// getExtendTypeForNode - Translate an extend node to the corresponding
415/// ExtendType value.
416static AArch64_AM::ShiftExtendType
417getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
418 if (N.getOpcode() == ISD::SIGN_EXTEND ||
419 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
420 EVT SrcVT;
421 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
422 SrcVT = cast<VTSDNode>(N.getOperand(1))->getVT();
423 else
424 SrcVT = N.getOperand(0).getValueType();
425
426 if (!IsLoadStore && SrcVT == MVT::i8)
427 return AArch64_AM::SXTB;
428 else if (!IsLoadStore && SrcVT == MVT::i16)
429 return AArch64_AM::SXTH;
430 else if (SrcVT == MVT::i32)
431 return AArch64_AM::SXTW;
432 assert(SrcVT != MVT::i64 && "extend from 64-bits?")((SrcVT != MVT::i64 && "extend from 64-bits?") ? static_cast
<void> (0) : __assert_fail ("SrcVT != MVT::i64 && \"extend from 64-bits?\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 432, __PRETTY_FUNCTION__))
;
433
434 return AArch64_AM::InvalidShiftExtend;
435 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
436 N.getOpcode() == ISD::ANY_EXTEND) {
437 EVT SrcVT = N.getOperand(0).getValueType();
438 if (!IsLoadStore && SrcVT == MVT::i8)
439 return AArch64_AM::UXTB;
440 else if (!IsLoadStore && SrcVT == MVT::i16)
441 return AArch64_AM::UXTH;
442 else if (SrcVT == MVT::i32)
443 return AArch64_AM::UXTW;
444 assert(SrcVT != MVT::i64 && "extend from 64-bits?")((SrcVT != MVT::i64 && "extend from 64-bits?") ? static_cast
<void> (0) : __assert_fail ("SrcVT != MVT::i64 && \"extend from 64-bits?\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 444, __PRETTY_FUNCTION__))
;
445
446 return AArch64_AM::InvalidShiftExtend;
447 } else if (N.getOpcode() == ISD::AND) {
448 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
449 if (!CSD)
450 return AArch64_AM::InvalidShiftExtend;
451 uint64_t AndMask = CSD->getZExtValue();
452
453 switch (AndMask) {
454 default:
455 return AArch64_AM::InvalidShiftExtend;
456 case 0xFF:
457 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
458 case 0xFFFF:
459 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
460 case 0xFFFFFFFF:
461 return AArch64_AM::UXTW;
462 }
463 }
464
465 return AArch64_AM::InvalidShiftExtend;
466}
467
468// Helper for SelectMLAV64LaneV128 - Recognize high lane extracts.
469static bool checkHighLaneIndex(SDNode *DL, SDValue &LaneOp, int &LaneIdx) {
470 if (DL->getOpcode() != AArch64ISD::DUPLANE16 &&
471 DL->getOpcode() != AArch64ISD::DUPLANE32)
472 return false;
473
474 SDValue SV = DL->getOperand(0);
475 if (SV.getOpcode() != ISD::INSERT_SUBVECTOR)
476 return false;
477
478 SDValue EV = SV.getOperand(1);
479 if (EV.getOpcode() != ISD::EXTRACT_SUBVECTOR)
480 return false;
481
482 ConstantSDNode *DLidx = cast<ConstantSDNode>(DL->getOperand(1).getNode());
483 ConstantSDNode *EVidx = cast<ConstantSDNode>(EV.getOperand(1).getNode());
484 LaneIdx = DLidx->getSExtValue() + EVidx->getSExtValue();
485 LaneOp = EV.getOperand(0);
486
487 return true;
488}
489
490// Helper for SelectOpcV64LaneV128 - Recognize operations where one operand is a
491// high lane extract.
492static bool checkV64LaneV128(SDValue Op0, SDValue Op1, SDValue &StdOp,
493 SDValue &LaneOp, int &LaneIdx) {
494
495 if (!checkHighLaneIndex(Op0.getNode(), LaneOp, LaneIdx)) {
496 std::swap(Op0, Op1);
497 if (!checkHighLaneIndex(Op0.getNode(), LaneOp, LaneIdx))
498 return false;
499 }
500 StdOp = Op1;
501 return true;
502}
503
504/// SelectMLAV64LaneV128 - AArch64 supports vector MLAs where one multiplicand
505/// is a lane in the upper half of a 128-bit vector. Recognize and select this
506/// so that we don't emit unnecessary lane extracts.
507bool AArch64DAGToDAGISel::tryMLAV64LaneV128(SDNode *N) {
508 SDLoc dl(N);
509 SDValue Op0 = N->getOperand(0);
510 SDValue Op1 = N->getOperand(1);
511 SDValue MLAOp1; // Will hold ordinary multiplicand for MLA.
512 SDValue MLAOp2; // Will hold lane-accessed multiplicand for MLA.
513 int LaneIdx = -1; // Will hold the lane index.
514
515 if (Op1.getOpcode() != ISD::MUL ||
516 !checkV64LaneV128(Op1.getOperand(0), Op1.getOperand(1), MLAOp1, MLAOp2,
517 LaneIdx)) {
518 std::swap(Op0, Op1);
519 if (Op1.getOpcode() != ISD::MUL ||
520 !checkV64LaneV128(Op1.getOperand(0), Op1.getOperand(1), MLAOp1, MLAOp2,
521 LaneIdx))
522 return false;
523 }
524
525 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
526
527 SDValue Ops[] = { Op0, MLAOp1, MLAOp2, LaneIdxVal };
528
529 unsigned MLAOpc = ~0U;
530
531 switch (N->getSimpleValueType(0).SimpleTy) {
532 default:
533 llvm_unreachable("Unrecognized MLA.")::llvm::llvm_unreachable_internal("Unrecognized MLA.", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 533)
;
534 case MVT::v4i16:
535 MLAOpc = AArch64::MLAv4i16_indexed;
536 break;
537 case MVT::v8i16:
538 MLAOpc = AArch64::MLAv8i16_indexed;
539 break;
540 case MVT::v2i32:
541 MLAOpc = AArch64::MLAv2i32_indexed;
542 break;
543 case MVT::v4i32:
544 MLAOpc = AArch64::MLAv4i32_indexed;
545 break;
546 }
547
548 ReplaceNode(N, CurDAG->getMachineNode(MLAOpc, dl, N->getValueType(0), Ops));
549 return true;
550}
551
552bool AArch64DAGToDAGISel::tryMULLV64LaneV128(unsigned IntNo, SDNode *N) {
553 SDLoc dl(N);
554 SDValue SMULLOp0;
555 SDValue SMULLOp1;
556 int LaneIdx;
557
558 if (!checkV64LaneV128(N->getOperand(1), N->getOperand(2), SMULLOp0, SMULLOp1,
559 LaneIdx))
560 return false;
561
562 SDValue LaneIdxVal = CurDAG->getTargetConstant(LaneIdx, dl, MVT::i64);
563
564 SDValue Ops[] = { SMULLOp0, SMULLOp1, LaneIdxVal };
565
566 unsigned SMULLOpc = ~0U;
567
568 if (IntNo == Intrinsic::aarch64_neon_smull) {
569 switch (N->getSimpleValueType(0).SimpleTy) {
570 default:
571 llvm_unreachable("Unrecognized SMULL.")::llvm::llvm_unreachable_internal("Unrecognized SMULL.", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 571)
;
572 case MVT::v4i32:
573 SMULLOpc = AArch64::SMULLv4i16_indexed;
574 break;
575 case MVT::v2i64:
576 SMULLOpc = AArch64::SMULLv2i32_indexed;
577 break;
578 }
579 } else if (IntNo == Intrinsic::aarch64_neon_umull) {
580 switch (N->getSimpleValueType(0).SimpleTy) {
581 default:
582 llvm_unreachable("Unrecognized SMULL.")::llvm::llvm_unreachable_internal("Unrecognized SMULL.", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 582)
;
583 case MVT::v4i32:
584 SMULLOpc = AArch64::UMULLv4i16_indexed;
585 break;
586 case MVT::v2i64:
587 SMULLOpc = AArch64::UMULLv2i32_indexed;
588 break;
589 }
590 } else
591 llvm_unreachable("Unrecognized intrinsic.")::llvm::llvm_unreachable_internal("Unrecognized intrinsic.", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 591)
;
592
593 ReplaceNode(N, CurDAG->getMachineNode(SMULLOpc, dl, N->getValueType(0), Ops));
594 return true;
595}
596
597/// Instructions that accept extend modifiers like UXTW expect the register
598/// being extended to be a GPR32, but the incoming DAG might be acting on a
599/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
600/// this is the case.
601static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N) {
602 if (N.getValueType() == MVT::i32)
603 return N;
604
605 SDLoc dl(N);
606 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
607 MachineSDNode *Node = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
608 dl, MVT::i32, N, SubReg);
609 return SDValue(Node, 0);
610}
611
612
613/// SelectArithExtendedRegister - Select a "extended register" operand. This
614/// operand folds in an extend followed by an optional left shift.
615bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
616 SDValue &Shift) {
617 unsigned ShiftVal = 0;
618 AArch64_AM::ShiftExtendType Ext;
619
620 if (N.getOpcode() == ISD::SHL) {
621 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
622 if (!CSD)
623 return false;
624 ShiftVal = CSD->getZExtValue();
625 if (ShiftVal > 4)
626 return false;
627
628 Ext = getExtendTypeForNode(N.getOperand(0));
629 if (Ext == AArch64_AM::InvalidShiftExtend)
630 return false;
631
632 Reg = N.getOperand(0).getOperand(0);
633 } else {
634 Ext = getExtendTypeForNode(N);
635 if (Ext == AArch64_AM::InvalidShiftExtend)
636 return false;
637
638 Reg = N.getOperand(0);
639
640 // Don't match if free 32-bit -> 64-bit zext can be used instead.
641 if (Ext == AArch64_AM::UXTW &&
642 Reg->getValueType(0).getSizeInBits() == 32 && isDef32(*Reg.getNode()))
643 return false;
644 }
645
646 // AArch64 mandates that the RHS of the operation must use the smallest
647 // register class that could contain the size being extended from. Thus,
648 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
649 // there might not be an actual 32-bit value in the program. We can
650 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
651 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX)((Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX)
? static_cast<void> (0) : __assert_fail ("Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 651, __PRETTY_FUNCTION__))
;
652 Reg = narrowIfNeeded(CurDAG, Reg);
653 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
654 MVT::i32);
655 return isWorthFolding(N);
656}
657
658/// If there's a use of this ADDlow that's not itself a load/store then we'll
659/// need to create a real ADD instruction from it anyway and there's no point in
660/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
661/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
662/// leads to duplicated ADRP instructions.
663static bool isWorthFoldingADDlow(SDValue N) {
664 for (auto Use : N->uses()) {
665 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
666 Use->getOpcode() != ISD::ATOMIC_LOAD &&
667 Use->getOpcode() != ISD::ATOMIC_STORE)
668 return false;
669
670 // ldar and stlr have much more restrictive addressing modes (just a
671 // register).
672 if (isStrongerThanMonotonic(cast<MemSDNode>(Use)->getOrdering()))
673 return false;
674 }
675
676 return true;
677}
678
679/// SelectAddrModeIndexed7S - Select a "register plus scaled signed 7-bit
680/// immediate" address. The "Size" argument is the size in bytes of the memory
681/// reference, which determines the scale.
682bool AArch64DAGToDAGISel::SelectAddrModeIndexed7S(SDValue N, unsigned Size,
683 SDValue &Base,
684 SDValue &OffImm) {
685 SDLoc dl(N);
686 const DataLayout &DL = CurDAG->getDataLayout();
687 const TargetLowering *TLI = getTargetLowering();
688 if (N.getOpcode() == ISD::FrameIndex) {
689 int FI = cast<FrameIndexSDNode>(N)->getIndex();
690 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
691 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
692 return true;
693 }
694
695 // As opposed to the (12-bit) Indexed addressing mode below, the 7-bit signed
696 // selected here doesn't support labels/immediates, only base+offset.
697
698 if (CurDAG->isBaseWithConstantOffset(N)) {
699 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
700 int64_t RHSC = RHS->getSExtValue();
701 unsigned Scale = Log2_32(Size);
702 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(0x40 << Scale) &&
703 RHSC < (0x40 << Scale)) {
704 Base = N.getOperand(0);
705 if (Base.getOpcode() == ISD::FrameIndex) {
706 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
707 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
708 }
709 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
710 return true;
711 }
712 }
713 }
714
715 // Base only. The address will be materialized into a register before
716 // the memory is accessed.
717 // add x0, Xbase, #offset
718 // stp x1, x2, [x0]
719 Base = N;
720 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
721 return true;
722}
723
724/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
725/// immediate" address. The "Size" argument is the size in bytes of the memory
726/// reference, which determines the scale.
727bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
728 SDValue &Base, SDValue &OffImm) {
729 SDLoc dl(N);
730 const DataLayout &DL = CurDAG->getDataLayout();
731 const TargetLowering *TLI = getTargetLowering();
732 if (N.getOpcode() == ISD::FrameIndex) {
733 int FI = cast<FrameIndexSDNode>(N)->getIndex();
734 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
735 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
736 return true;
737 }
738
739 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
740 GlobalAddressSDNode *GAN =
741 dyn_cast<GlobalAddressSDNode>(N.getOperand(1).getNode());
742 Base = N.getOperand(0);
743 OffImm = N.getOperand(1);
744 if (!GAN)
745 return true;
746
747 if (GAN->getOffset() % Size == 0) {
748 const GlobalValue *GV = GAN->getGlobal();
749 unsigned Alignment = GV->getAlignment();
750 Type *Ty = GV->getValueType();
751 if (Alignment == 0 && Ty->isSized())
752 Alignment = DL.getABITypeAlignment(Ty);
753
754 if (Alignment >= Size)
755 return true;
756 }
757 }
758
759 if (CurDAG->isBaseWithConstantOffset(N)) {
760 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
761 int64_t RHSC = (int64_t)RHS->getZExtValue();
762 unsigned Scale = Log2_32(Size);
763 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 && RHSC < (0x1000 << Scale)) {
764 Base = N.getOperand(0);
765 if (Base.getOpcode() == ISD::FrameIndex) {
766 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
767 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
768 }
769 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
770 return true;
771 }
772 }
773 }
774
775 // Before falling back to our general case, check if the unscaled
776 // instructions can handle this. If so, that's preferable.
777 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
778 return false;
779
780 // Base only. The address will be materialized into a register before
781 // the memory is accessed.
782 // add x0, Xbase, #offset
783 // ldr x0, [x0]
784 Base = N;
785 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
786 return true;
787}
788
789/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
790/// immediate" address. This should only match when there is an offset that
791/// is not valid for a scaled immediate addressing mode. The "Size" argument
792/// is the size in bytes of the memory reference, which is needed here to know
793/// what is valid for a scaled immediate.
794bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
795 SDValue &Base,
796 SDValue &OffImm) {
797 if (!CurDAG->isBaseWithConstantOffset(N))
798 return false;
799 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
800 int64_t RHSC = RHS->getSExtValue();
801 // If the offset is valid as a scaled immediate, don't match here.
802 if ((RHSC & (Size - 1)) == 0 && RHSC >= 0 &&
803 RHSC < (0x1000 << Log2_32(Size)))
804 return false;
805 if (RHSC >= -256 && RHSC < 256) {
806 Base = N.getOperand(0);
807 if (Base.getOpcode() == ISD::FrameIndex) {
808 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
809 const TargetLowering *TLI = getTargetLowering();
810 Base = CurDAG->getTargetFrameIndex(
811 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
812 }
813 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
814 return true;
815 }
816 }
817 return false;
818}
819
820static SDValue Widen(SelectionDAG *CurDAG, SDValue N) {
821 SDLoc dl(N);
822 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
823 SDValue ImpDef = SDValue(
824 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
825 MachineSDNode *Node = CurDAG->getMachineNode(
826 TargetOpcode::INSERT_SUBREG, dl, MVT::i64, ImpDef, N, SubReg);
827 return SDValue(Node, 0);
828}
829
830/// Check if the given SHL node (\p N), can be used to form an
831/// extended register for an addressing mode.
832bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
833 bool WantExtend, SDValue &Offset,
834 SDValue &SignExtend) {
835 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.")((N.getOpcode() == ISD::SHL && "Invalid opcode.") ? static_cast
<void> (0) : __assert_fail ("N.getOpcode() == ISD::SHL && \"Invalid opcode.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 835, __PRETTY_FUNCTION__))
;
836 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
837 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
838 return false;
839
840 SDLoc dl(N);
841 if (WantExtend) {
842 AArch64_AM::ShiftExtendType Ext =
843 getExtendTypeForNode(N.getOperand(0), true);
844 if (Ext == AArch64_AM::InvalidShiftExtend)
845 return false;
846
847 Offset = narrowIfNeeded(CurDAG, N.getOperand(0).getOperand(0));
848 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
849 MVT::i32);
850 } else {
851 Offset = N.getOperand(0);
852 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
853 }
854
855 unsigned LegalShiftVal = Log2_32(Size);
856 unsigned ShiftVal = CSD->getZExtValue();
857
858 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
859 return false;
860
861 return isWorthFolding(N);
862}
863
864bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
865 SDValue &Base, SDValue &Offset,
866 SDValue &SignExtend,
867 SDValue &DoShift) {
868 if (N.getOpcode() != ISD::ADD)
869 return false;
870 SDValue LHS = N.getOperand(0);
871 SDValue RHS = N.getOperand(1);
872 SDLoc dl(N);
873
874 // We don't want to match immediate adds here, because they are better lowered
875 // to the register-immediate addressing modes.
876 if (isa<ConstantSDNode>(LHS) || isa<ConstantSDNode>(RHS))
877 return false;
878
879 // Check if this particular node is reused in any non-memory related
880 // operation. If yes, do not try to fold this node into the address
881 // computation, since the computation will be kept.
882 const SDNode *Node = N.getNode();
883 for (SDNode *UI : Node->uses()) {
884 if (!isa<MemSDNode>(*UI))
885 return false;
886 }
887
888 // Remember if it is worth folding N when it produces extended register.
889 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
890
891 // Try to match a shifted extend on the RHS.
892 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
893 SelectExtendedSHL(RHS, Size, true, Offset, SignExtend)) {
894 Base = LHS;
895 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
896 return true;
897 }
898
899 // Try to match a shifted extend on the LHS.
900 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
901 SelectExtendedSHL(LHS, Size, true, Offset, SignExtend)) {
902 Base = RHS;
903 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
904 return true;
905 }
906
907 // There was no shift, whatever else we find.
908 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32);
909
910 AArch64_AM::ShiftExtendType Ext = AArch64_AM::InvalidShiftExtend;
911 // Try to match an unshifted extend on the LHS.
912 if (IsExtendedRegisterWorthFolding &&
913 (Ext = getExtendTypeForNode(LHS, true)) !=
914 AArch64_AM::InvalidShiftExtend) {
915 Base = RHS;
916 Offset = narrowIfNeeded(CurDAG, LHS.getOperand(0));
917 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
918 MVT::i32);
919 if (isWorthFolding(LHS))
920 return true;
921 }
922
923 // Try to match an unshifted extend on the RHS.
924 if (IsExtendedRegisterWorthFolding &&
925 (Ext = getExtendTypeForNode(RHS, true)) !=
926 AArch64_AM::InvalidShiftExtend) {
927 Base = LHS;
928 Offset = narrowIfNeeded(CurDAG, RHS.getOperand(0));
929 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
930 MVT::i32);
931 if (isWorthFolding(RHS))
932 return true;
933 }
934
935 return false;
936}
937
938// Check if the given immediate is preferred by ADD. If an immediate can be
939// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
940// encoded by one MOVZ, return true.
941static bool isPreferredADD(int64_t ImmOff) {
942 // Constant in [0x0, 0xfff] can be encoded in ADD.
943 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
944 return true;
945 // Check if it can be encoded in an "ADD LSL #12".
946 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
947 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
948 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
949 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
950 return false;
951}
952
953bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
954 SDValue &Base, SDValue &Offset,
955 SDValue &SignExtend,
956 SDValue &DoShift) {
957 if (N.getOpcode() != ISD::ADD)
958 return false;
959 SDValue LHS = N.getOperand(0);
960 SDValue RHS = N.getOperand(1);
961 SDLoc DL(N);
962
963 // Check if this particular node is reused in any non-memory related
964 // operation. If yes, do not try to fold this node into the address
965 // computation, since the computation will be kept.
966 const SDNode *Node = N.getNode();
967 for (SDNode *UI : Node->uses()) {
968 if (!isa<MemSDNode>(*UI))
969 return false;
970 }
971
972 // Watch out if RHS is a wide immediate, it can not be selected into
973 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
974 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
975 // instructions like:
976 // MOV X0, WideImmediate
977 // ADD X1, BaseReg, X0
978 // LDR X2, [X1, 0]
979 // For such situation, using [BaseReg, XReg] addressing mode can save one
980 // ADD/SUB:
981 // MOV X0, WideImmediate
982 // LDR X2, [BaseReg, X0]
983 if (isa<ConstantSDNode>(RHS)) {
984 int64_t ImmOff = (int64_t)cast<ConstantSDNode>(RHS)->getZExtValue();
985 unsigned Scale = Log2_32(Size);
986 // Skip the immediate can be selected by load/store addressing mode.
987 // Also skip the immediate can be encoded by a single ADD (SUB is also
988 // checked by using -ImmOff).
989 if ((ImmOff % Size == 0 && ImmOff >= 0 && ImmOff < (0x1000 << Scale)) ||
990 isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
991 return false;
992
993 SDValue Ops[] = { RHS };
994 SDNode *MOVI =
995 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
996 SDValue MOVIV = SDValue(MOVI, 0);
997 // This ADD of two X register will be selected into [Reg+Reg] mode.
998 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV);
999 }
1000
1001 // Remember if it is worth folding N when it produces extended register.
1002 bool IsExtendedRegisterWorthFolding = isWorthFolding(N);
1003
1004 // Try to match a shifted extend on the RHS.
1005 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1006 SelectExtendedSHL(RHS, Size, false, Offset, SignExtend)) {
1007 Base = LHS;
1008 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1009 return true;
1010 }
1011
1012 // Try to match a shifted extend on the LHS.
1013 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1014 SelectExtendedSHL(LHS, Size, false, Offset, SignExtend)) {
1015 Base = RHS;
1016 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1017 return true;
1018 }
1019
1020 // Match any non-shifted, non-extend, non-immediate add expression.
1021 Base = LHS;
1022 Offset = RHS;
1023 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32);
1024 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32);
1025 // Reg1 + Reg2 is free: no check needed.
1026 return true;
1027}
1028
1029SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1030 static const unsigned RegClassIDs[] = {
1031 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1032 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1033 AArch64::dsub2, AArch64::dsub3};
1034
1035 return createTuple(Regs, RegClassIDs, SubRegs);
1036}
1037
1038SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1039 static const unsigned RegClassIDs[] = {
1040 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1041 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1042 AArch64::qsub2, AArch64::qsub3};
1043
1044 return createTuple(Regs, RegClassIDs, SubRegs);
1045}
1046
1047SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1048 const unsigned RegClassIDs[],
1049 const unsigned SubRegs[]) {
1050 // There's no special register-class for a vector-list of 1 element: it's just
1051 // a vector.
1052 if (Regs.size() == 1)
1053 return Regs[0];
1054
1055 assert(Regs.size() >= 2 && Regs.size() <= 4)((Regs.size() >= 2 && Regs.size() <= 4) ? static_cast
<void> (0) : __assert_fail ("Regs.size() >= 2 && Regs.size() <= 4"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1055, __PRETTY_FUNCTION__))
;
1056
1057 SDLoc DL(Regs[0]);
1058
1059 SmallVector<SDValue, 4> Ops;
1060
1061 // First operand of REG_SEQUENCE is the desired RegClass.
1062 Ops.push_back(
1063 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32));
1064
1065 // Then we get pairs of source & subregister-position for the components.
1066 for (unsigned i = 0; i < Regs.size(); ++i) {
1067 Ops.push_back(Regs[i]);
1068 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32));
1069 }
1070
1071 SDNode *N =
1072 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
1073 return SDValue(N, 0);
1074}
1075
1076void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1077 bool isExt) {
1078 SDLoc dl(N);
1079 EVT VT = N->getValueType(0);
1080
1081 unsigned ExtOff = isExt;
1082
1083 // Form a REG_SEQUENCE to force register allocation.
1084 unsigned Vec0Off = ExtOff + 1;
1085 SmallVector<SDValue, 4> Regs(N->op_begin() + Vec0Off,
1086 N->op_begin() + Vec0Off + NumVecs);
1087 SDValue RegSeq = createQTuple(Regs);
1088
1089 SmallVector<SDValue, 6> Ops;
1090 if (isExt)
1091 Ops.push_back(N->getOperand(1));
1092 Ops.push_back(RegSeq);
1093 Ops.push_back(N->getOperand(NumVecs + ExtOff + 1));
1094 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1095}
1096
1097bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
1098 LoadSDNode *LD = cast<LoadSDNode>(N);
1099 if (LD->isUnindexed())
1100 return false;
1101 EVT VT = LD->getMemoryVT();
1102 EVT DstVT = N->getValueType(0);
1103 ISD::MemIndexedMode AM = LD->getAddressingMode();
1104 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
1105
1106 // We're not doing validity checking here. That was done when checking
1107 // if we should mark the load as indexed or not. We're just selecting
1108 // the right instruction.
1109 unsigned Opcode = 0;
1110
1111 ISD::LoadExtType ExtType = LD->getExtensionType();
1112 bool InsertTo64 = false;
1113 if (VT == MVT::i64)
1114 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
1115 else if (VT == MVT::i32) {
1116 if (ExtType == ISD::NON_EXTLOAD)
1117 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1118 else if (ExtType == ISD::SEXTLOAD)
1119 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
1120 else {
1121 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
1122 InsertTo64 = true;
1123 // The result of the load is only i32. It's the subreg_to_reg that makes
1124 // it into an i64.
1125 DstVT = MVT::i32;
1126 }
1127 } else if (VT == MVT::i16) {
1128 if (ExtType == ISD::SEXTLOAD) {
1129 if (DstVT == MVT::i64)
1130 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
1131 else
1132 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
1133 } else {
1134 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
1135 InsertTo64 = DstVT == MVT::i64;
1136 // The result of the load is only i32. It's the subreg_to_reg that makes
1137 // it into an i64.
1138 DstVT = MVT::i32;
1139 }
1140 } else if (VT == MVT::i8) {
1141 if (ExtType == ISD::SEXTLOAD) {
1142 if (DstVT == MVT::i64)
1143 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
1144 else
1145 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
1146 } else {
1147 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
1148 InsertTo64 = DstVT == MVT::i64;
1149 // The result of the load is only i32. It's the subreg_to_reg that makes
1150 // it into an i64.
1151 DstVT = MVT::i32;
1152 }
1153 } else if (VT == MVT::f16) {
1154 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
1155 } else if (VT == MVT::f32) {
1156 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
1157 } else if (VT == MVT::f64 || VT.is64BitVector()) {
1158 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
1159 } else if (VT.is128BitVector()) {
1160 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
1161 } else
1162 return false;
1163 SDValue Chain = LD->getChain();
1164 SDValue Base = LD->getBasePtr();
1165 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
1166 int OffsetVal = (int)OffsetOp->getZExtValue();
1167 SDLoc dl(N);
1168 SDValue Offset = CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
1169 SDValue Ops[] = { Base, Offset, Chain };
1170 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
1171 MVT::Other, Ops);
1172 // Either way, we're replacing the node, so tell the caller that.
1173 SDValue LoadedVal = SDValue(Res, 1);
1174 if (InsertTo64) {
1175 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1176 LoadedVal =
1177 SDValue(CurDAG->getMachineNode(
1178 AArch64::SUBREG_TO_REG, dl, MVT::i64,
1179 CurDAG->getTargetConstant(0, dl, MVT::i64), LoadedVal,
1180 SubReg),
1181 0);
1182 }
1183
1184 ReplaceUses(SDValue(N, 0), LoadedVal);
1185 ReplaceUses(SDValue(N, 1), SDValue(Res, 0));
1186 ReplaceUses(SDValue(N, 2), SDValue(Res, 2));
1187 CurDAG->RemoveDeadNode(N);
1188 return true;
1189}
1190
1191void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
1192 unsigned SubRegIdx) {
1193 SDLoc dl(N);
1194 EVT VT = N->getValueType(0);
1195 SDValue Chain = N->getOperand(0);
1196
1197 SDValue Ops[] = {N->getOperand(2), // Mem operand;
1198 Chain};
1199
1200 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1201
1202 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1203 SDValue SuperReg = SDValue(Ld, 0);
1204 for (unsigned i = 0; i < NumVecs; ++i)
1205 ReplaceUses(SDValue(N, i),
1206 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1207
1208 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
1209
1210 // Transfer memoperands.
1211 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1212 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
1213
1214 CurDAG->RemoveDeadNode(N);
1215}
1216
1217void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
1218 unsigned Opc, unsigned SubRegIdx) {
1219 SDLoc dl(N);
1220 EVT VT = N->getValueType(0);
1221 SDValue Chain = N->getOperand(0);
1222
1223 SDValue Ops[] = {N->getOperand(1), // Mem operand
1224 N->getOperand(2), // Incremental
1225 Chain};
1226
1227 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1228 MVT::Untyped, MVT::Other};
1229
1230 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1231
1232 // Update uses of write back register
1233 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
1234
1235 // Update uses of vector list
1236 SDValue SuperReg = SDValue(Ld, 1);
1237 if (NumVecs == 1)
1238 ReplaceUses(SDValue(N, 0), SuperReg);
1239 else
1240 for (unsigned i = 0; i < NumVecs; ++i)
1241 ReplaceUses(SDValue(N, i),
1242 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
1243
1244 // Update the chain
1245 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
1246 CurDAG->RemoveDeadNode(N);
1247}
1248
1249void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
1250 unsigned Opc) {
1251 SDLoc dl(N);
1252 EVT VT = N->getOperand(2)->getValueType(0);
1253
1254 // Form a REG_SEQUENCE to force register allocation.
1255 bool Is128Bit = VT.getSizeInBits() == 128;
1256 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1257 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1258
1259 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)};
1260 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
1261
1262 // Transfer memoperands.
1263 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1264 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1265
1266 ReplaceNode(N, St);
1267}
1268
1269void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
1270 unsigned Opc) {
1271 SDLoc dl(N);
1272 EVT VT = N->getOperand(2)->getValueType(0);
1273 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1274 MVT::Other}; // Type for the Chain
1275
1276 // Form a REG_SEQUENCE to force register allocation.
1277 bool Is128Bit = VT.getSizeInBits() == 128;
1278 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1279 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
1280
1281 SDValue Ops[] = {RegSeq,
1282 N->getOperand(NumVecs + 1), // base register
1283 N->getOperand(NumVecs + 2), // Incremental
1284 N->getOperand(0)}; // Chain
1285 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1286
1287 ReplaceNode(N, St);
1288}
1289
1290namespace {
1291/// WidenVector - Given a value in the V64 register class, produce the
1292/// equivalent value in the V128 register class.
1293class WidenVector {
1294 SelectionDAG &DAG;
1295
1296public:
1297 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
1298
1299 SDValue operator()(SDValue V64Reg) {
1300 EVT VT = V64Reg.getValueType();
1301 unsigned NarrowSize = VT.getVectorNumElements();
1302 MVT EltTy = VT.getVectorElementType().getSimpleVT();
1303 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
1304 SDLoc DL(V64Reg);
1305
1306 SDValue Undef =
1307 SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, WideTy), 0);
1308 return DAG.getTargetInsertSubreg(AArch64::dsub, DL, WideTy, Undef, V64Reg);
1309 }
1310};
1311} // namespace
1312
1313/// NarrowVector - Given a value in the V128 register class, produce the
1314/// equivalent value in the V64 register class.
1315static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG) {
1316 EVT VT = V128Reg.getValueType();
8
Calling 'SDValue::getValueType'
1317 unsigned WideSize = VT.getVectorNumElements();
1318 MVT EltTy = VT.getVectorElementType().getSimpleVT();
1319 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
1320
1321 return DAG.getTargetExtractSubreg(AArch64::dsub, SDLoc(V128Reg), NarrowTy,
1322 V128Reg);
1323}
1324
1325void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
1326 unsigned Opc) {
1327 SDLoc dl(N);
1328 EVT VT = N->getValueType(0);
1329 bool Narrow = VT.getSizeInBits() == 64;
1330
1331 // Form a REG_SEQUENCE to force register allocation.
1332 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1333
1334 if (Narrow)
1335 transform(Regs, Regs.begin(),
1336 WidenVector(*CurDAG));
1337
1338 SDValue RegSeq = createQTuple(Regs);
1339
1340 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
1341
1342 unsigned LaneNo =
1343 cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
1344
1345 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1346 N->getOperand(NumVecs + 3), N->getOperand(0)};
1347 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1348 SDValue SuperReg = SDValue(Ld, 0);
1349
1350 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1351 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1352 AArch64::qsub2, AArch64::qsub3 };
1353 for (unsigned i = 0; i < NumVecs; ++i) {
1354 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
1355 if (Narrow)
1356 NV = NarrowVector(NV, *CurDAG);
1357 ReplaceUses(SDValue(N, i), NV);
1358 }
1359
1360 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
1361 CurDAG->RemoveDeadNode(N);
1362}
1363
1364void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
1365 unsigned Opc) {
1366 SDLoc dl(N);
1367 EVT VT = N->getValueType(0);
1368 bool Narrow = VT.getSizeInBits() == 64;
1
Assuming the condition is true
1369
1370 // Form a REG_SEQUENCE to force register allocation.
1371 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1372
1373 if (Narrow)
2
Taking true branch
1374 transform(Regs, Regs.begin(),
1375 WidenVector(*CurDAG));
1376
1377 SDValue RegSeq = createQTuple(Regs);
1378
1379 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1380 RegSeq->getValueType(0), MVT::Other};
1381
1382 unsigned LaneNo =
1383 cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
1384
1385 SDValue Ops[] = {RegSeq,
1386 CurDAG->getTargetConstant(LaneNo, dl,
1387 MVT::i64), // Lane Number
1388 N->getOperand(NumVecs + 2), // Base register
1389 N->getOperand(NumVecs + 3), // Incremental
1390 N->getOperand(0)};
1391 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1392
1393 // Update uses of the write back register
1394 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
1395
1396 // Update uses of the vector list
1397 SDValue SuperReg = SDValue(Ld, 1);
1398 if (NumVecs == 1) {
3
Assuming 'NumVecs' is equal to 1
4
Taking true branch
1399 ReplaceUses(SDValue(N, 0),
1400 Narrow ? NarrowVector(SuperReg, *CurDAG) : SuperReg);
5
'?' condition is true
6
Null pointer value stored to 'V128Reg.Node'
7
Calling 'NarrowVector'
1401 } else {
1402 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
1403 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
1404 AArch64::qsub2, AArch64::qsub3 };
1405 for (unsigned i = 0; i < NumVecs; ++i) {
1406 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
1407 SuperReg);
1408 if (Narrow)
1409 NV = NarrowVector(NV, *CurDAG);
1410 ReplaceUses(SDValue(N, i), NV);
1411 }
1412 }
1413
1414 // Update the Chain
1415 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
1416 CurDAG->RemoveDeadNode(N);
1417}
1418
1419void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
1420 unsigned Opc) {
1421 SDLoc dl(N);
1422 EVT VT = N->getOperand(2)->getValueType(0);
1423 bool Narrow = VT.getSizeInBits() == 64;
1424
1425 // Form a REG_SEQUENCE to force register allocation.
1426 SmallVector<SDValue, 4> Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs);
1427
1428 if (Narrow)
1429 transform(Regs, Regs.begin(),
1430 WidenVector(*CurDAG));
1431
1432 SDValue RegSeq = createQTuple(Regs);
1433
1434 unsigned LaneNo =
1435 cast<ConstantSDNode>(N->getOperand(NumVecs + 2))->getZExtValue();
1436
1437 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1438 N->getOperand(NumVecs + 3), N->getOperand(0)};
1439 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
1440
1441 // Transfer memoperands.
1442 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1443 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1444
1445 ReplaceNode(N, St);
1446}
1447
1448void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
1449 unsigned Opc) {
1450 SDLoc dl(N);
1451 EVT VT = N->getOperand(2)->getValueType(0);
1452 bool Narrow = VT.getSizeInBits() == 64;
1453
1454 // Form a REG_SEQUENCE to force register allocation.
1455 SmallVector<SDValue, 4> Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs);
1456
1457 if (Narrow)
1458 transform(Regs, Regs.begin(),
1459 WidenVector(*CurDAG));
1460
1461 SDValue RegSeq = createQTuple(Regs);
1462
1463 const EVT ResTys[] = {MVT::i64, // Type of the write back register
1464 MVT::Other};
1465
1466 unsigned LaneNo =
1467 cast<ConstantSDNode>(N->getOperand(NumVecs + 1))->getZExtValue();
1468
1469 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
1470 N->getOperand(NumVecs + 2), // Base Register
1471 N->getOperand(NumVecs + 3), // Incremental
1472 N->getOperand(0)};
1473 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1474
1475 // Transfer memoperands.
1476 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1477 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
1478
1479 ReplaceNode(N, St);
1480}
1481
1482static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N,
1483 unsigned &Opc, SDValue &Opd0,
1484 unsigned &LSB, unsigned &MSB,
1485 unsigned NumberOfIgnoredLowBits,
1486 bool BiggerPattern) {
1487 assert(N->getOpcode() == ISD::AND &&((N->getOpcode() == ISD::AND && "N must be a AND operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::AND && \"N must be a AND operation to call this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1488, __PRETTY_FUNCTION__))
1488 "N must be a AND operation to call this function")((N->getOpcode() == ISD::AND && "N must be a AND operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::AND && \"N must be a AND operation to call this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1488, __PRETTY_FUNCTION__))
;
1489
1490 EVT VT = N->getValueType(0);
1491
1492 // Here we can test the type of VT and return false when the type does not
1493 // match, but since it is done prior to that call in the current context
1494 // we turned that into an assert to avoid redundant code.
1495 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1496, __PRETTY_FUNCTION__))
1496 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1496, __PRETTY_FUNCTION__))
;
1497
1498 // FIXME: simplify-demanded-bits in DAGCombine will probably have
1499 // changed the AND node to a 32-bit mask operation. We'll have to
1500 // undo that as part of the transform here if we want to catch all
1501 // the opportunities.
1502 // Currently the NumberOfIgnoredLowBits argument helps to recover
1503 // form these situations when matching bigger pattern (bitfield insert).
1504
1505 // For unsigned extracts, check for a shift right and mask
1506 uint64_t AndImm = 0;
1507 if (!isOpcWithIntImmediate(N, ISD::AND, AndImm))
1508 return false;
1509
1510 const SDNode *Op0 = N->getOperand(0).getNode();
1511
1512 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
1513 // simplified. Try to undo that
1514 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
1515
1516 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
1517 if (AndImm & (AndImm + 1))
1518 return false;
1519
1520 bool ClampMSB = false;
1521 uint64_t SrlImm = 0;
1522 // Handle the SRL + ANY_EXTEND case.
1523 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
1524 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, SrlImm)) {
1525 // Extend the incoming operand of the SRL to 64-bit.
1526 Opd0 = Widen(CurDAG, Op0->getOperand(0).getOperand(0));
1527 // Make sure to clamp the MSB so that we preserve the semantics of the
1528 // original operations.
1529 ClampMSB = true;
1530 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
1531 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL,
1532 SrlImm)) {
1533 // If the shift result was truncated, we can still combine them.
1534 Opd0 = Op0->getOperand(0).getOperand(0);
1535
1536 // Use the type of SRL node.
1537 VT = Opd0->getValueType(0);
1538 } else if (isOpcWithIntImmediate(Op0, ISD::SRL, SrlImm)) {
1539 Opd0 = Op0->getOperand(0);
1540 } else if (BiggerPattern) {
1541 // Let's pretend a 0 shift right has been performed.
1542 // The resulting code will be at least as good as the original one
1543 // plus it may expose more opportunities for bitfield insert pattern.
1544 // FIXME: Currently we limit this to the bigger pattern, because
1545 // some optimizations expect AND and not UBFM.
1546 Opd0 = N->getOperand(0);
1547 } else
1548 return false;
1549
1550 // Bail out on large immediates. This happens when no proper
1551 // combining/constant folding was performed.
1552 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
1553 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1554 (dbgs() << Ndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1555 << ": Found large shift immediate, this should not happen\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
;
1556 return false;
1557 }
1558
1559 LSB = SrlImm;
1560 MSB = SrlImm + (VT == MVT::i32 ? countTrailingOnes<uint32_t>(AndImm)
1561 : countTrailingOnes<uint64_t>(AndImm)) -
1562 1;
1563 if (ClampMSB)
1564 // Since we're moving the extend before the right shift operation, we need
1565 // to clamp the MSB to make sure we don't shift in undefined bits instead of
1566 // the zeros which would get shifted in with the original right shift
1567 // operation.
1568 MSB = MSB > 31 ? 31 : MSB;
1569
1570 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
1571 return true;
1572}
1573
1574static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc,
1575 SDValue &Opd0, unsigned &Immr,
1576 unsigned &Imms) {
1577 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG)((N->getOpcode() == ISD::SIGN_EXTEND_INREG) ? static_cast<
void> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND_INREG"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1577, __PRETTY_FUNCTION__))
;
1578
1579 EVT VT = N->getValueType(0);
1580 unsigned BitWidth = VT.getSizeInBits();
1581 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1582, __PRETTY_FUNCTION__))
1582 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1582, __PRETTY_FUNCTION__))
;
1583
1584 SDValue Op = N->getOperand(0);
1585 if (Op->getOpcode() == ISD::TRUNCATE) {
1586 Op = Op->getOperand(0);
1587 VT = Op->getValueType(0);
1588 BitWidth = VT.getSizeInBits();
1589 }
1590
1591 uint64_t ShiftImm;
1592 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SRL, ShiftImm) &&
1593 !isOpcWithIntImmediate(Op.getNode(), ISD::SRA, ShiftImm))
1594 return false;
1595
1596 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
1597 if (ShiftImm + Width > BitWidth)
1598 return false;
1599
1600 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
1601 Opd0 = Op.getOperand(0);
1602 Immr = ShiftImm;
1603 Imms = ShiftImm + Width - 1;
1604 return true;
1605}
1606
1607static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc,
1608 SDValue &Opd0, unsigned &LSB,
1609 unsigned &MSB) {
1610 // We are looking for the following pattern which basically extracts several
1611 // continuous bits from the source value and places it from the LSB of the
1612 // destination value, all other bits of the destination value or set to zero:
1613 //
1614 // Value2 = AND Value, MaskImm
1615 // SRL Value2, ShiftImm
1616 //
1617 // with MaskImm >> ShiftImm to search for the bit width.
1618 //
1619 // This gets selected into a single UBFM:
1620 //
1621 // UBFM Value, ShiftImm, BitWide + SrlImm -1
1622 //
1623
1624 if (N->getOpcode() != ISD::SRL)
1625 return false;
1626
1627 uint64_t AndMask = 0;
1628 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, AndMask))
1629 return false;
1630
1631 Opd0 = N->getOperand(0).getOperand(0);
1632
1633 uint64_t SrlImm = 0;
1634 if (!isIntImmediate(N->getOperand(1), SrlImm))
1635 return false;
1636
1637 // Check whether we really have several bits extract here.
1638 unsigned BitWide = 64 - countLeadingOnes(~(AndMask >> SrlImm));
1639 if (BitWide && isMask_64(AndMask >> SrlImm)) {
1640 if (N->getValueType(0) == MVT::i32)
1641 Opc = AArch64::UBFMWri;
1642 else
1643 Opc = AArch64::UBFMXri;
1644
1645 LSB = SrlImm;
1646 MSB = BitWide + SrlImm - 1;
1647 return true;
1648 }
1649
1650 return false;
1651}
1652
1653static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
1654 unsigned &Immr, unsigned &Imms,
1655 bool BiggerPattern) {
1656 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&(((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::
SRL) && "N must be a SHR/SRA operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"N must be a SHR/SRA operation to call this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1657, __PRETTY_FUNCTION__))
1657 "N must be a SHR/SRA operation to call this function")(((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::
SRL) && "N must be a SHR/SRA operation to call this function"
) ? static_cast<void> (0) : __assert_fail ("(N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) && \"N must be a SHR/SRA operation to call this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1657, __PRETTY_FUNCTION__))
;
1658
1659 EVT VT = N->getValueType(0);
1660
1661 // Here we can test the type of VT and return false when the type does not
1662 // match, but since it is done prior to that call in the current context
1663 // we turned that into an assert to avoid redundant code.
1664 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1665, __PRETTY_FUNCTION__))
1665 "Type checking must have been done before calling this function")(((VT == MVT::i32 || VT == MVT::i64) && "Type checking must have been done before calling this function"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"Type checking must have been done before calling this function\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1665, __PRETTY_FUNCTION__))
;
1666
1667 // Check for AND + SRL doing several bits extract.
1668 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, Immr, Imms))
1669 return true;
1670
1671 // We're looking for a shift of a shift.
1672 uint64_t ShlImm = 0;
1673 uint64_t TruncBits = 0;
1674 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, ShlImm)) {
1675 Opd0 = N->getOperand(0).getOperand(0);
1676 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
1677 N->getOperand(0).getNode()->getOpcode() == ISD::TRUNCATE) {
1678 // We are looking for a shift of truncate. Truncate from i64 to i32 could
1679 // be considered as setting high 32 bits as zero. Our strategy here is to
1680 // always generate 64bit UBFM. This consistency will help the CSE pass
1681 // later find more redundancy.
1682 Opd0 = N->getOperand(0).getOperand(0);
1683 TruncBits = Opd0->getValueType(0).getSizeInBits() - VT.getSizeInBits();
1684 VT = Opd0.getValueType();
1685 assert(VT == MVT::i64 && "the promoted type should be i64")((VT == MVT::i64 && "the promoted type should be i64"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::i64 && \"the promoted type should be i64\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1685, __PRETTY_FUNCTION__))
;
1686 } else if (BiggerPattern) {
1687 // Let's pretend a 0 shift left has been performed.
1688 // FIXME: Currently we limit this to the bigger pattern case,
1689 // because some optimizations expect AND and not UBFM
1690 Opd0 = N->getOperand(0);
1691 } else
1692 return false;
1693
1694 // Missing combines/constant folding may have left us with strange
1695 // constants.
1696 if (ShlImm >= VT.getSizeInBits()) {
1697 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1698 (dbgs() << Ndo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
1699 << ": Found large shift immediate, this should not happen\n"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { (dbgs() << N << ": Found large shift immediate, this should not happen\n"
); } } while (false)
;
1700 return false;
1701 }
1702
1703 uint64_t SrlImm = 0;
1704 if (!isIntImmediate(N->getOperand(1), SrlImm))
1705 return false;
1706
1707 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&((SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
"bad amount in shift node!") ? static_cast<void> (0) :
__assert_fail ("SrlImm > 0 && SrlImm < VT.getSizeInBits() && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1708, __PRETTY_FUNCTION__))
1708 "bad amount in shift node!")((SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
"bad amount in shift node!") ? static_cast<void> (0) :
__assert_fail ("SrlImm > 0 && SrlImm < VT.getSizeInBits() && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1708, __PRETTY_FUNCTION__))
;
1709 int immr = SrlImm - ShlImm;
1710 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
1711 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
1712 // SRA requires a signed extraction
1713 if (VT == MVT::i32)
1714 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
1715 else
1716 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
1717 return true;
1718}
1719
1720bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
1721 assert(N->getOpcode() == ISD::SIGN_EXTEND)((N->getOpcode() == ISD::SIGN_EXTEND) ? static_cast<void
> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1721, __PRETTY_FUNCTION__))
;
1722
1723 EVT VT = N->getValueType(0);
1724 EVT NarrowVT = N->getOperand(0)->getValueType(0);
1725 if (VT != MVT::i64 || NarrowVT != MVT::i32)
1726 return false;
1727
1728 uint64_t ShiftImm;
1729 SDValue Op = N->getOperand(0);
1730 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SRA, ShiftImm))
1731 return false;
1732
1733 SDLoc dl(N);
1734 // Extend the incoming operand of the shift to 64-bits.
1735 SDValue Opd0 = Widen(CurDAG, Op.getOperand(0));
1736 unsigned Immr = ShiftImm;
1737 unsigned Imms = NarrowVT.getSizeInBits() - 1;
1738 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1739 CurDAG->getTargetConstant(Imms, dl, VT)};
1740 CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops);
1741 return true;
1742}
1743
1744static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
1745 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
1746 unsigned NumberOfIgnoredLowBits = 0,
1747 bool BiggerPattern = false) {
1748 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
1749 return false;
1750
1751 switch (N->getOpcode()) {
1752 default:
1753 if (!N->isMachineOpcode())
1754 return false;
1755 break;
1756 case ISD::AND:
1757 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, Immr, Imms,
1758 NumberOfIgnoredLowBits, BiggerPattern);
1759 case ISD::SRL:
1760 case ISD::SRA:
1761 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
1762
1763 case ISD::SIGN_EXTEND_INREG:
1764 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
1765 }
1766
1767 unsigned NOpc = N->getMachineOpcode();
1768 switch (NOpc) {
1769 default:
1770 return false;
1771 case AArch64::SBFMWri:
1772 case AArch64::UBFMWri:
1773 case AArch64::SBFMXri:
1774 case AArch64::UBFMXri:
1775 Opc = NOpc;
1776 Opd0 = N->getOperand(0);
1777 Immr = cast<ConstantSDNode>(N->getOperand(1).getNode())->getZExtValue();
1778 Imms = cast<ConstantSDNode>(N->getOperand(2).getNode())->getZExtValue();
1779 return true;
1780 }
1781 // Unreachable
1782 return false;
1783}
1784
1785bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
1786 unsigned Opc, Immr, Imms;
1787 SDValue Opd0;
1788 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
1789 return false;
1790
1791 EVT VT = N->getValueType(0);
1792 SDLoc dl(N);
1793
1794 // If the bit extract operation is 64bit but the original type is 32bit, we
1795 // need to add one EXTRACT_SUBREG.
1796 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
1797 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
1798 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
1799
1800 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
1801 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
1802 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG, dl,
1803 MVT::i32, SDValue(BFM, 0), SubReg));
1804 return true;
1805 }
1806
1807 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
1808 CurDAG->getTargetConstant(Imms, dl, VT)};
1809 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
1810 return true;
1811}
1812
1813/// Does DstMask form a complementary pair with the mask provided by
1814/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
1815/// this asks whether DstMask zeroes precisely those bits that will be set by
1816/// the other half.
1817static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
1818 unsigned NumberOfIgnoredHighBits, EVT VT) {
1819 assert((VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i32 || VT == MVT::i64) && "i32 or i64 mask type expected!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"i32 or i64 mask type expected!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1820, __PRETTY_FUNCTION__))
1820 "i32 or i64 mask type expected!")(((VT == MVT::i32 || VT == MVT::i64) && "i32 or i64 mask type expected!"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"i32 or i64 mask type expected!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 1820, __PRETTY_FUNCTION__))
;
1821 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
1822
1823 APInt SignificantDstMask = APInt(BitWidth, DstMask);
1824 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(BitWidth);
1825
1826 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
1827 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnesValue();
1828}
1829
1830// Look for bits that will be useful for later uses.
1831// A bit is consider useless as soon as it is dropped and never used
1832// before it as been dropped.
1833// E.g., looking for useful bit of x
1834// 1. y = x & 0x7
1835// 2. z = y >> 2
1836// After #1, x useful bits are 0x7, then the useful bits of x, live through
1837// y.
1838// After #2, the useful bits of x are 0x4.
1839// However, if x is used on an unpredicatable instruction, then all its bits
1840// are useful.
1841// E.g.
1842// 1. y = x & 0x7
1843// 2. z = y >> 2
1844// 3. str x, [@x]
1845static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
1846
1847static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits,
1848 unsigned Depth) {
1849 uint64_t Imm =
1850 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
1851 Imm = AArch64_AM::decodeLogicalImmediate(Imm, UsefulBits.getBitWidth());
1852 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
1853 getUsefulBits(Op, UsefulBits, Depth + 1);
1854}
1855
1856static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits,
1857 uint64_t Imm, uint64_t MSB,
1858 unsigned Depth) {
1859 // inherit the bitwidth value
1860 APInt OpUsefulBits(UsefulBits);
1861 OpUsefulBits = 1;
1862
1863 if (MSB >= Imm) {
1864 OpUsefulBits <<= MSB - Imm + 1;
1865 --OpUsefulBits;
1866 // The interesting part will be in the lower part of the result
1867 getUsefulBits(Op, OpUsefulBits, Depth + 1);
1868 // The interesting part was starting at Imm in the argument
1869 OpUsefulBits <<= Imm;
1870 } else {
1871 OpUsefulBits <<= MSB + 1;
1872 --OpUsefulBits;
1873 // The interesting part will be shifted in the result
1874 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
1875 getUsefulBits(Op, OpUsefulBits, Depth + 1);
1876 // The interesting part was at zero in the argument
1877 OpUsefulBits.lshrInPlace(OpUsefulBits.getBitWidth() - Imm);
1878 }
1879
1880 UsefulBits &= OpUsefulBits;
1881}
1882
1883static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
1884 unsigned Depth) {
1885 uint64_t Imm =
1886 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
1887 uint64_t MSB =
1888 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
1889
1890 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
1891}
1892
1893static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits,
1894 unsigned Depth) {
1895 uint64_t ShiftTypeAndValue =
1896 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
1897 APInt Mask(UsefulBits);
1898 Mask.clearAllBits();
1899 Mask.flipAllBits();
1900
1901 if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSL) {
1902 // Shift Left
1903 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
1904 Mask <<= ShiftAmt;
1905 getUsefulBits(Op, Mask, Depth + 1);
1906 Mask.lshrInPlace(ShiftAmt);
1907 } else if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSR) {
1908 // Shift Right
1909 // We do not handle AArch64_AM::ASR, because the sign will change the
1910 // number of useful bits
1911 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
1912 Mask.lshrInPlace(ShiftAmt);
1913 getUsefulBits(Op, Mask, Depth + 1);
1914 Mask <<= ShiftAmt;
1915 } else
1916 return;
1917
1918 UsefulBits &= Mask;
1919}
1920
1921static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
1922 unsigned Depth) {
1923 uint64_t Imm =
1924 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
1925 uint64_t MSB =
1926 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue();
1927
1928 APInt OpUsefulBits(UsefulBits);
1929 OpUsefulBits = 1;
1930
1931 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
1932 ResultUsefulBits.flipAllBits();
1933 APInt Mask(UsefulBits.getBitWidth(), 0);
1934
1935 getUsefulBits(Op, ResultUsefulBits, Depth + 1);
1936
1937 if (MSB >= Imm) {
1938 // The instruction is a BFXIL.
1939 uint64_t Width = MSB - Imm + 1;
1940 uint64_t LSB = Imm;
1941
1942 OpUsefulBits <<= Width;
1943 --OpUsefulBits;
1944
1945 if (Op.getOperand(1) == Orig) {
1946 // Copy the low bits from the result to bits starting from LSB.
1947 Mask = ResultUsefulBits & OpUsefulBits;
1948 Mask <<= LSB;
1949 }
1950
1951 if (Op.getOperand(0) == Orig)
1952 // Bits starting from LSB in the input contribute to the result.
1953 Mask |= (ResultUsefulBits & ~OpUsefulBits);
1954 } else {
1955 // The instruction is a BFI.
1956 uint64_t Width = MSB + 1;
1957 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
1958
1959 OpUsefulBits <<= Width;
1960 --OpUsefulBits;
1961 OpUsefulBits <<= LSB;
1962
1963 if (Op.getOperand(1) == Orig) {
1964 // Copy the bits from the result to the zero bits.
1965 Mask = ResultUsefulBits & OpUsefulBits;
1966 Mask.lshrInPlace(LSB);
1967 }
1968
1969 if (Op.getOperand(0) == Orig)
1970 Mask |= (ResultUsefulBits & ~OpUsefulBits);
1971 }
1972
1973 UsefulBits &= Mask;
1974}
1975
1976static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
1977 SDValue Orig, unsigned Depth) {
1978
1979 // Users of this node should have already been instruction selected
1980 // FIXME: Can we turn that into an assert?
1981 if (!UserNode->isMachineOpcode())
1982 return;
1983
1984 switch (UserNode->getMachineOpcode()) {
1985 default:
1986 return;
1987 case AArch64::ANDSWri:
1988 case AArch64::ANDSXri:
1989 case AArch64::ANDWri:
1990 case AArch64::ANDXri:
1991 // We increment Depth only when we call the getUsefulBits
1992 return getUsefulBitsFromAndWithImmediate(SDValue(UserNode, 0), UsefulBits,
1993 Depth);
1994 case AArch64::UBFMWri:
1995 case AArch64::UBFMXri:
1996 return getUsefulBitsFromUBFM(SDValue(UserNode, 0), UsefulBits, Depth);
1997
1998 case AArch64::ORRWrs:
1999 case AArch64::ORRXrs:
2000 if (UserNode->getOperand(1) != Orig)
2001 return;
2002 return getUsefulBitsFromOrWithShiftedReg(SDValue(UserNode, 0), UsefulBits,
2003 Depth);
2004 case AArch64::BFMWri:
2005 case AArch64::BFMXri:
2006 return getUsefulBitsFromBFM(SDValue(UserNode, 0), Orig, UsefulBits, Depth);
2007
2008 case AArch64::STRBBui:
2009 case AArch64::STURBBi:
2010 if (UserNode->getOperand(0) != Orig)
2011 return;
2012 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
2013 return;
2014
2015 case AArch64::STRHHui:
2016 case AArch64::STURHHi:
2017 if (UserNode->getOperand(0) != Orig)
2018 return;
2019 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
2020 return;
2021 }
2022}
2023
2024static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
2025 if (Depth >= 6)
2026 return;
2027 // Initialize UsefulBits
2028 if (!Depth) {
2029 unsigned Bitwidth = Op.getScalarValueSizeInBits();
2030 // At the beginning, assume every produced bits is useful
2031 UsefulBits = APInt(Bitwidth, 0);
2032 UsefulBits.flipAllBits();
2033 }
2034 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
2035
2036 for (SDNode *Node : Op.getNode()->uses()) {
2037 // A use cannot produce useful bits
2038 APInt UsefulBitsForUse = APInt(UsefulBits);
2039 getUsefulBitsForUse(Node, UsefulBitsForUse, Op, Depth);
2040 UsersUsefulBits |= UsefulBitsForUse;
2041 }
2042 // UsefulBits contains the produced bits that are meaningful for the
2043 // current definition, thus a user cannot make a bit meaningful at
2044 // this point
2045 UsefulBits &= UsersUsefulBits;
2046}
2047
2048/// Create a machine node performing a notional SHL of Op by ShlAmount. If
2049/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
2050/// 0, return Op unchanged.
2051static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
2052 if (ShlAmount == 0)
2053 return Op;
2054
2055 EVT VT = Op.getValueType();
2056 SDLoc dl(Op);
2057 unsigned BitWidth = VT.getSizeInBits();
2058 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
2059
2060 SDNode *ShiftNode;
2061 if (ShlAmount > 0) {
2062 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
2063 ShiftNode = CurDAG->getMachineNode(
2064 UBFMOpc, dl, VT, Op,
2065 CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT),
2066 CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT));
2067 } else {
2068 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
2069 assert(ShlAmount < 0 && "expected right shift")((ShlAmount < 0 && "expected right shift") ? static_cast
<void> (0) : __assert_fail ("ShlAmount < 0 && \"expected right shift\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2069, __PRETTY_FUNCTION__))
;
2070 int ShrAmount = -ShlAmount;
2071 ShiftNode = CurDAG->getMachineNode(
2072 UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT),
2073 CurDAG->getTargetConstant(BitWidth - 1, dl, VT));
2074 }
2075
2076 return SDValue(ShiftNode, 0);
2077}
2078
2079/// Does this tree qualify as an attempt to move a bitfield into position,
2080/// essentially "(and (shl VAL, N), Mask)".
2081static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op,
2082 bool BiggerPattern,
2083 SDValue &Src, int &ShiftAmount,
2084 int &MaskWidth) {
2085 EVT VT = Op.getValueType();
2086 unsigned BitWidth = VT.getSizeInBits();
2087 (void)BitWidth;
2088 assert(BitWidth == 32 || BitWidth == 64)((BitWidth == 32 || BitWidth == 64) ? static_cast<void>
(0) : __assert_fail ("BitWidth == 32 || BitWidth == 64", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2088, __PRETTY_FUNCTION__))
;
2089
2090 KnownBits Known;
2091 CurDAG->computeKnownBits(Op, Known);
2092
2093 // Non-zero in the sense that they're not provably zero, which is the key
2094 // point if we want to use this value
2095 uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
2096
2097 // Discard a constant AND mask if present. It's safe because the node will
2098 // already have been factored into the computeKnownBits calculation above.
2099 uint64_t AndImm;
2100 if (isOpcWithIntImmediate(Op.getNode(), ISD::AND, AndImm)) {
2101 assert((~APInt(BitWidth, AndImm) & ~Known.Zero) == 0)(((~APInt(BitWidth, AndImm) & ~Known.Zero) == 0) ? static_cast
<void> (0) : __assert_fail ("(~APInt(BitWidth, AndImm) & ~Known.Zero) == 0"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2101, __PRETTY_FUNCTION__))
;
2102 Op = Op.getOperand(0);
2103 }
2104
2105 // Don't match if the SHL has more than one use, since then we'll end up
2106 // generating SHL+UBFIZ instead of just keeping SHL+AND.
2107 if (!BiggerPattern && !Op.hasOneUse())
2108 return false;
2109
2110 uint64_t ShlImm;
2111 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SHL, ShlImm))
2112 return false;
2113 Op = Op.getOperand(0);
2114
2115 if (!isShiftedMask_64(NonZeroBits))
2116 return false;
2117
2118 ShiftAmount = countTrailingZeros(NonZeroBits);
2119 MaskWidth = countTrailingOnes(NonZeroBits >> ShiftAmount);
2120
2121 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
2122 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
2123 // amount. BiggerPattern is true when this pattern is being matched for BFI,
2124 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
2125 // which case it is not profitable to insert an extra shift.
2126 if (ShlImm - ShiftAmount != 0 && !BiggerPattern)
2127 return false;
2128 Src = getLeftShift(CurDAG, Op, ShlImm - ShiftAmount);
2129
2130 return true;
2131}
2132
2133static bool isShiftedMask(uint64_t Mask, EVT VT) {
2134 assert(VT == MVT::i32 || VT == MVT::i64)((VT == MVT::i32 || VT == MVT::i64) ? static_cast<void>
(0) : __assert_fail ("VT == MVT::i32 || VT == MVT::i64", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2134, __PRETTY_FUNCTION__))
;
2135 if (VT == MVT::i32)
2136 return isShiftedMask_32(Mask);
2137 return isShiftedMask_64(Mask);
2138}
2139
2140// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
2141// inserted only sets known zero bits.
2142static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG) {
2143 assert(N->getOpcode() == ISD::OR && "Expect a OR operation")((N->getOpcode() == ISD::OR && "Expect a OR operation"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Expect a OR operation\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2143, __PRETTY_FUNCTION__))
;
2144
2145 EVT VT = N->getValueType(0);
2146 if (VT != MVT::i32 && VT != MVT::i64)
2147 return false;
2148
2149 unsigned BitWidth = VT.getSizeInBits();
2150
2151 uint64_t OrImm;
2152 if (!isOpcWithIntImmediate(N, ISD::OR, OrImm))
2153 return false;
2154
2155 // Skip this transformation if the ORR immediate can be encoded in the ORR.
2156 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
2157 // performance neutral.
2158 if (AArch64_AM::isLogicalImmediate(OrImm, BitWidth))
2159 return false;
2160
2161 uint64_t MaskImm;
2162 SDValue And = N->getOperand(0);
2163 // Must be a single use AND with an immediate operand.
2164 if (!And.hasOneUse() ||
2165 !isOpcWithIntImmediate(And.getNode(), ISD::AND, MaskImm))
2166 return false;
2167
2168 // Compute the Known Zero for the AND as this allows us to catch more general
2169 // cases than just looking for AND with imm.
2170 KnownBits Known;
2171 CurDAG->computeKnownBits(And, Known);
2172
2173 // Non-zero in the sense that they're not provably zero, which is the key
2174 // point if we want to use this value.
2175 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
2176
2177 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
2178 if (!isShiftedMask(Known.Zero.getZExtValue(), VT))
2179 return false;
2180
2181 // The bits being inserted must only set those bits that are known to be zero.
2182 if ((OrImm & NotKnownZero) != 0) {
2183 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
2184 // currently handle this case.
2185 return false;
2186 }
2187
2188 // BFI/BFXIL dst, src, #lsb, #width.
2189 int LSB = countTrailingOnes(NotKnownZero);
2190 int Width = BitWidth - APInt(BitWidth, NotKnownZero).countPopulation();
2191
2192 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
2193 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2194 unsigned ImmS = Width - 1;
2195
2196 // If we're creating a BFI instruction avoid cases where we need more
2197 // instructions to materialize the BFI constant as compared to the original
2198 // ORR. A BFXIL will use the same constant as the original ORR, so the code
2199 // should be no worse in this case.
2200 bool IsBFI = LSB != 0;
2201 uint64_t BFIImm = OrImm >> LSB;
2202 if (IsBFI && !AArch64_AM::isLogicalImmediate(BFIImm, BitWidth)) {
2203 // We have a BFI instruction and we know the constant can't be materialized
2204 // with a ORR-immediate with the zero register.
2205 unsigned OrChunks = 0, BFIChunks = 0;
2206 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
2207 if (((OrImm >> Shift) & 0xFFFF) != 0)
2208 ++OrChunks;
2209 if (((BFIImm >> Shift) & 0xFFFF) != 0)
2210 ++BFIChunks;
2211 }
2212 if (BFIChunks > OrChunks)
2213 return false;
2214 }
2215
2216 // Materialize the constant to be inserted.
2217 SDLoc DL(N);
2218 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
2219 SDNode *MOVI = CurDAG->getMachineNode(
2220 MOVIOpc, DL, VT, CurDAG->getTargetConstant(BFIImm, DL, VT));
2221
2222 // Create the BFI/BFXIL instruction.
2223 SDValue Ops[] = {And.getOperand(0), SDValue(MOVI, 0),
2224 CurDAG->getTargetConstant(ImmR, DL, VT),
2225 CurDAG->getTargetConstant(ImmS, DL, VT)};
2226 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2227 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2228 return true;
2229}
2230
2231static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
2232 SelectionDAG *CurDAG) {
2233 assert(N->getOpcode() == ISD::OR && "Expect a OR operation")((N->getOpcode() == ISD::OR && "Expect a OR operation"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Expect a OR operation\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2233, __PRETTY_FUNCTION__))
;
2234
2235 EVT VT = N->getValueType(0);
2236 if (VT != MVT::i32 && VT != MVT::i64)
2237 return false;
2238
2239 unsigned BitWidth = VT.getSizeInBits();
2240
2241 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
2242 // have the expected shape. Try to undo that.
2243
2244 unsigned NumberOfIgnoredLowBits = UsefulBits.countTrailingZeros();
2245 unsigned NumberOfIgnoredHighBits = UsefulBits.countLeadingZeros();
2246
2247 // Given a OR operation, check if we have the following pattern
2248 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
2249 // isBitfieldExtractOp)
2250 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
2251 // countTrailingZeros(mask2) == imm2 - imm + 1
2252 // f = d | c
2253 // if yes, replace the OR instruction with:
2254 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
2255
2256 // OR is commutative, check all combinations of operand order and values of
2257 // BiggerPattern, i.e.
2258 // Opd0, Opd1, BiggerPattern=false
2259 // Opd1, Opd0, BiggerPattern=false
2260 // Opd0, Opd1, BiggerPattern=true
2261 // Opd1, Opd0, BiggerPattern=true
2262 // Several of these combinations may match, so check with BiggerPattern=false
2263 // first since that will produce better results by matching more instructions
2264 // and/or inserting fewer extra instructions.
2265 for (int I = 0; I < 4; ++I) {
2266
2267 SDValue Dst, Src;
2268 unsigned ImmR, ImmS;
2269 bool BiggerPattern = I / 2;
2270 SDValue OrOpd0Val = N->getOperand(I % 2);
2271 SDNode *OrOpd0 = OrOpd0Val.getNode();
2272 SDValue OrOpd1Val = N->getOperand((I + 1) % 2);
2273 SDNode *OrOpd1 = OrOpd1Val.getNode();
2274
2275 unsigned BFXOpc;
2276 int DstLSB, Width;
2277 if (isBitfieldExtractOp(CurDAG, OrOpd0, BFXOpc, Src, ImmR, ImmS,
2278 NumberOfIgnoredLowBits, BiggerPattern)) {
2279 // Check that the returned opcode is compatible with the pattern,
2280 // i.e., same type and zero extended (U and not S)
2281 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
2282 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
2283 continue;
2284
2285 // Compute the width of the bitfield insertion
2286 DstLSB = 0;
2287 Width = ImmS - ImmR + 1;
2288 // FIXME: This constraint is to catch bitfield insertion we may
2289 // want to widen the pattern if we want to grab general bitfied
2290 // move case
2291 if (Width <= 0)
2292 continue;
2293
2294 // If the mask on the insertee is correct, we have a BFXIL operation. We
2295 // can share the ImmR and ImmS values from the already-computed UBFM.
2296 } else if (isBitfieldPositioningOp(CurDAG, OrOpd0Val,
2297 BiggerPattern,
2298 Src, DstLSB, Width)) {
2299 ImmR = (BitWidth - DstLSB) % BitWidth;
2300 ImmS = Width - 1;
2301 } else
2302 continue;
2303
2304 // Check the second part of the pattern
2305 EVT VT = OrOpd1Val.getValueType();
2306 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand")(((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::i32 || VT == MVT::i64) && \"unexpected OR operand\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2306, __PRETTY_FUNCTION__))
;
2307
2308 // Compute the Known Zero for the candidate of the first operand.
2309 // This allows to catch more general case than just looking for
2310 // AND with imm. Indeed, simplify-demanded-bits may have removed
2311 // the AND instruction because it proves it was useless.
2312 KnownBits Known;
2313 CurDAG->computeKnownBits(OrOpd1Val, Known);
2314
2315 // Check if there is enough room for the second operand to appear
2316 // in the first one
2317 APInt BitsToBeInserted =
2318 APInt::getBitsSet(Known.getBitWidth(), DstLSB, DstLSB + Width);
2319
2320 if ((BitsToBeInserted & ~Known.Zero) != 0)
2321 continue;
2322
2323 // Set the first operand
2324 uint64_t Imm;
2325 if (isOpcWithIntImmediate(OrOpd1, ISD::AND, Imm) &&
2326 isBitfieldDstMask(Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
2327 // In that case, we can eliminate the AND
2328 Dst = OrOpd1->getOperand(0);
2329 else
2330 // Maybe the AND has been removed by simplify-demanded-bits
2331 // or is useful because it discards more bits
2332 Dst = OrOpd1Val;
2333
2334 // both parts match
2335 SDLoc DL(N);
2336 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT),
2337 CurDAG->getTargetConstant(ImmS, DL, VT)};
2338 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2339 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2340 return true;
2341 }
2342
2343 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
2344 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
2345 // mask (e.g., 0x000ffff0).
2346 uint64_t Mask0Imm, Mask1Imm;
2347 SDValue And0 = N->getOperand(0);
2348 SDValue And1 = N->getOperand(1);
2349 if (And0.hasOneUse() && And1.hasOneUse() &&
2350 isOpcWithIntImmediate(And0.getNode(), ISD::AND, Mask0Imm) &&
2351 isOpcWithIntImmediate(And1.getNode(), ISD::AND, Mask1Imm) &&
2352 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
2353 (isShiftedMask(Mask0Imm, VT) || isShiftedMask(Mask1Imm, VT))) {
2354
2355 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
2356 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
2357 // bits to be inserted.
2358 if (isShiftedMask(Mask0Imm, VT)) {
2359 std::swap(And0, And1);
2360 std::swap(Mask0Imm, Mask1Imm);
2361 }
2362
2363 SDValue Src = And1->getOperand(0);
2364 SDValue Dst = And0->getOperand(0);
2365 unsigned LSB = countTrailingZeros(Mask1Imm);
2366 int Width = BitWidth - APInt(BitWidth, Mask0Imm).countPopulation();
2367
2368 // The BFXIL inserts the low-order bits from a source register, so right
2369 // shift the needed bits into place.
2370 SDLoc DL(N);
2371 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2372 SDNode *LSR = CurDAG->getMachineNode(
2373 ShiftOpc, DL, VT, Src, CurDAG->getTargetConstant(LSB, DL, VT),
2374 CurDAG->getTargetConstant(BitWidth - 1, DL, VT));
2375
2376 // BFXIL is an alias of BFM, so translate to BFM operands.
2377 unsigned ImmR = (BitWidth - LSB) % BitWidth;
2378 unsigned ImmS = Width - 1;
2379
2380 // Create the BFXIL instruction.
2381 SDValue Ops[] = {Dst, SDValue(LSR, 0),
2382 CurDAG->getTargetConstant(ImmR, DL, VT),
2383 CurDAG->getTargetConstant(ImmS, DL, VT)};
2384 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
2385 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2386 return true;
2387 }
2388
2389 return false;
2390}
2391
2392bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
2393 if (N->getOpcode() != ISD::OR)
2394 return false;
2395
2396 APInt NUsefulBits;
2397 getUsefulBits(SDValue(N, 0), NUsefulBits);
2398
2399 // If all bits are not useful, just return UNDEF.
2400 if (!NUsefulBits) {
2401 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
2402 return true;
2403 }
2404
2405 if (tryBitfieldInsertOpFromOr(N, NUsefulBits, CurDAG))
2406 return true;
2407
2408 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
2409}
2410
2411/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
2412/// equivalent of a left shift by a constant amount followed by an and masking
2413/// out a contiguous set of bits.
2414bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
2415 if (N->getOpcode() != ISD::AND)
2416 return false;
2417
2418 EVT VT = N->getValueType(0);
2419 if (VT != MVT::i32 && VT != MVT::i64)
2420 return false;
2421
2422 SDValue Op0;
2423 int DstLSB, Width;
2424 if (!isBitfieldPositioningOp(CurDAG, SDValue(N, 0), /*BiggerPattern=*/false,
2425 Op0, DstLSB, Width))
2426 return false;
2427
2428 // ImmR is the rotate right amount.
2429 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
2430 // ImmS is the most significant bit of the source to be moved.
2431 unsigned ImmS = Width - 1;
2432
2433 SDLoc DL(N);
2434 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
2435 CurDAG->getTargetConstant(ImmS, DL, VT)};
2436 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
2437 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2438 return true;
2439}
2440
2441/// tryShiftAmountMod - Take advantage of built-in mod of shift amount in
2442/// variable shift/rotate instructions.
2443bool AArch64DAGToDAGISel::tryShiftAmountMod(SDNode *N) {
2444 EVT VT = N->getValueType(0);
2445
2446 unsigned Opc;
2447 switch (N->getOpcode()) {
2448 case ISD::ROTR:
2449 Opc = (VT == MVT::i32) ? AArch64::RORVWr : AArch64::RORVXr;
2450 break;
2451 case ISD::SHL:
2452 Opc = (VT == MVT::i32) ? AArch64::LSLVWr : AArch64::LSLVXr;
2453 break;
2454 case ISD::SRL:
2455 Opc = (VT == MVT::i32) ? AArch64::LSRVWr : AArch64::LSRVXr;
2456 break;
2457 case ISD::SRA:
2458 Opc = (VT == MVT::i32) ? AArch64::ASRVWr : AArch64::ASRVXr;
2459 break;
2460 default:
2461 return false;
2462 }
2463
2464 uint64_t Size;
2465 uint64_t Bits;
2466 if (VT == MVT::i32) {
2467 Bits = 5;
2468 Size = 32;
2469 } else if (VT == MVT::i64) {
2470 Bits = 6;
2471 Size = 64;
2472 } else
2473 return false;
2474
2475 SDValue ShiftAmt = N->getOperand(1);
2476 SDLoc DL(N);
2477 SDValue NewShiftAmt;
2478
2479 // Skip over an extend of the shift amount.
2480 if (ShiftAmt->getOpcode() == ISD::ZERO_EXTEND ||
2481 ShiftAmt->getOpcode() == ISD::ANY_EXTEND)
2482 ShiftAmt = ShiftAmt->getOperand(0);
2483
2484 if (ShiftAmt->getOpcode() == ISD::ADD || ShiftAmt->getOpcode() == ISD::SUB) {
2485 SDValue Add0 = ShiftAmt->getOperand(0);
2486 SDValue Add1 = ShiftAmt->getOperand(1);
2487 uint64_t Add0Imm;
2488 uint64_t Add1Imm;
2489 // If we are shifting by X+/-N where N == 0 mod Size, then just shift by X
2490 // to avoid the ADD/SUB.
2491 if (isIntImmediate(Add1, Add1Imm) && (Add1Imm % Size == 0))
2492 NewShiftAmt = Add0;
2493 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2494 // generate a NEG instead of a SUB of a constant.
2495 else if (ShiftAmt->getOpcode() == ISD::SUB &&
2496 isIntImmediate(Add0, Add0Imm) && Add0Imm != 0 &&
2497 (Add0Imm % Size == 0)) {
2498 unsigned NegOpc;
2499 unsigned ZeroReg;
2500 EVT SubVT = ShiftAmt->getValueType(0);
2501 if (SubVT == MVT::i32) {
2502 NegOpc = AArch64::SUBWrr;
2503 ZeroReg = AArch64::WZR;
2504 } else {
2505 assert(SubVT == MVT::i64)((SubVT == MVT::i64) ? static_cast<void> (0) : __assert_fail
("SubVT == MVT::i64", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2505, __PRETTY_FUNCTION__))
;
2506 NegOpc = AArch64::SUBXrr;
2507 ZeroReg = AArch64::XZR;
2508 }
2509 SDValue Zero =
2510 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, SubVT);
2511 MachineSDNode *Neg =
2512 CurDAG->getMachineNode(NegOpc, DL, SubVT, Zero, Add1);
2513 NewShiftAmt = SDValue(Neg, 0);
2514 } else
2515 return false;
2516 } else {
2517 // If the shift amount is masked with an AND, check that the mask covers the
2518 // bits that are implicitly ANDed off by the above opcodes and if so, skip
2519 // the AND.
2520 uint64_t MaskImm;
2521 if (!isOpcWithIntImmediate(ShiftAmt.getNode(), ISD::AND, MaskImm))
2522 return false;
2523
2524 if (countTrailingOnes(MaskImm) < Bits)
2525 return false;
2526
2527 NewShiftAmt = ShiftAmt->getOperand(0);
2528 }
2529
2530 // Narrow/widen the shift amount to match the size of the shift operation.
2531 if (VT == MVT::i32)
2532 NewShiftAmt = narrowIfNeeded(CurDAG, NewShiftAmt);
2533 else if (VT == MVT::i64 && NewShiftAmt->getValueType(0) == MVT::i32) {
2534 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, DL, MVT::i32);
2535 MachineSDNode *Ext = CurDAG->getMachineNode(
2536 AArch64::SUBREG_TO_REG, DL, VT,
2537 CurDAG->getTargetConstant(0, DL, MVT::i64), NewShiftAmt, SubReg);
2538 NewShiftAmt = SDValue(Ext, 0);
2539 }
2540
2541 SDValue Ops[] = {N->getOperand(0), NewShiftAmt};
2542 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
2543 return true;
2544}
2545
2546bool
2547AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
2548 unsigned RegWidth) {
2549 APFloat FVal(0.0);
2550 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
2551 FVal = CN->getValueAPF();
2552 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(N)) {
2553 // Some otherwise illegal constants are allowed in this case.
2554 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow ||
2555 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
2556 return false;
2557
2558 ConstantPoolSDNode *CN =
2559 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
2560 FVal = cast<ConstantFP>(CN->getConstVal())->getValueAPF();
2561 } else
2562 return false;
2563
2564 // An FCVT[SU] instruction performs: convertToInt(Val * 2^fbits) where fbits
2565 // is between 1 and 32 for a destination w-register, or 1 and 64 for an
2566 // x-register.
2567 //
2568 // By this stage, we've detected (fp_to_[su]int (fmul Val, THIS_NODE)) so we
2569 // want THIS_NODE to be 2^fbits. This is much easier to deal with using
2570 // integers.
2571 bool IsExact;
2572
2573 // fbits is between 1 and 64 in the worst-case, which means the fmul
2574 // could have 2^64 as an actual operand. Need 65 bits of precision.
2575 APSInt IntVal(65, true);
2576 FVal.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact);
2577
2578 // N.b. isPowerOf2 also checks for > 0.
2579 if (!IsExact || !IntVal.isPowerOf2()) return false;
2580 unsigned FBits = IntVal.logBase2();
2581
2582 // Checks above should have guaranteed that we haven't lost information in
2583 // finding FBits, but it must still be in range.
2584 if (FBits == 0 || FBits > RegWidth) return false;
2585
2586 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
2587 return true;
2588}
2589
2590// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
2591// of the string and obtains the integer values from them and combines these
2592// into a single value to be used in the MRS/MSR instruction.
2593static int getIntOperandFromRegisterString(StringRef RegString) {
2594 SmallVector<StringRef, 5> Fields;
2595 RegString.split(Fields, ':');
2596
2597 if (Fields.size() == 1)
2598 return -1;
2599
2600 assert(Fields.size() == 5((Fields.size() == 5 && "Invalid number of fields in read register string"
) ? static_cast<void> (0) : __assert_fail ("Fields.size() == 5 && \"Invalid number of fields in read register string\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2601, __PRETTY_FUNCTION__))
2601 && "Invalid number of fields in read register string")((Fields.size() == 5 && "Invalid number of fields in read register string"
) ? static_cast<void> (0) : __assert_fail ("Fields.size() == 5 && \"Invalid number of fields in read register string\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2601, __PRETTY_FUNCTION__))
;
2602
2603 SmallVector<int, 5> Ops;
2604 bool AllIntFields = true;
2605
2606 for (StringRef Field : Fields) {
2607 unsigned IntField;
2608 AllIntFields &= !Field.getAsInteger(10, IntField);
2609 Ops.push_back(IntField);
2610 }
2611
2612 assert(AllIntFields &&((AllIntFields && "Unexpected non-integer value in special register string."
) ? static_cast<void> (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2613, __PRETTY_FUNCTION__))
2613 "Unexpected non-integer value in special register string.")((AllIntFields && "Unexpected non-integer value in special register string."
) ? static_cast<void> (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2613, __PRETTY_FUNCTION__))
;
2614
2615 // Need to combine the integer fields of the string into a single value
2616 // based on the bit encoding of MRS/MSR instruction.
2617 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) |
2618 (Ops[3] << 3) | (Ops[4]);
2619}
2620
2621// Lower the read_register intrinsic to an MRS instruction node if the special
2622// register string argument is either of the form detailed in the ALCE (the
2623// form described in getIntOperandsFromRegsterString) or is a named register
2624// known by the MRS SysReg mapper.
2625bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
2626 const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
2627 const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2628 SDLoc DL(N);
2629
2630 int Reg = getIntOperandFromRegisterString(RegString->getString());
2631 if (Reg != -1) {
2632 ReplaceNode(N, CurDAG->getMachineNode(
2633 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
2634 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2635 N->getOperand(0)));
2636 return true;
2637 }
2638
2639 // Use the sysreg mapper to map the remaining possible strings to the
2640 // value for the register to be used for the instruction operand.
2641 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
2642 if (TheReg && TheReg->Readable &&
2643 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2644 Reg = TheReg->Encoding;
2645 else
2646 Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
2647
2648 if (Reg != -1) {
2649 ReplaceNode(N, CurDAG->getMachineNode(
2650 AArch64::MRS, DL, N->getSimpleValueType(0), MVT::Other,
2651 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2652 N->getOperand(0)));
2653 return true;
2654 }
2655
2656 return false;
2657}
2658
2659// Lower the write_register intrinsic to an MSR instruction node if the special
2660// register string argument is either of the form detailed in the ALCE (the
2661// form described in getIntOperandsFromRegsterString) or is a named register
2662// known by the MSR SysReg mapper.
2663bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
2664 const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
2665 const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
2666 SDLoc DL(N);
2667
2668 int Reg = getIntOperandFromRegisterString(RegString->getString());
2669 if (Reg != -1) {
2670 ReplaceNode(
2671 N, CurDAG->getMachineNode(AArch64::MSR, DL, MVT::Other,
2672 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2673 N->getOperand(2), N->getOperand(0)));
2674 return true;
2675 }
2676
2677 // Check if the register was one of those allowed as the pstatefield value in
2678 // the MSR (immediate) instruction. To accept the values allowed in the
2679 // pstatefield for the MSR (immediate) instruction, we also require that an
2680 // immediate value has been provided as an argument, we know that this is
2681 // the case as it has been ensured by semantic checking.
2682 auto PMapper = AArch64PState::lookupPStateByName(RegString->getString());
2683 if (PMapper) {
2684 assert (isa<ConstantSDNode>(N->getOperand(2))((isa<ConstantSDNode>(N->getOperand(2)) && "Expected a constant integer expression."
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(2)) && \"Expected a constant integer expression.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2685, __PRETTY_FUNCTION__))
2685 && "Expected a constant integer expression.")((isa<ConstantSDNode>(N->getOperand(2)) && "Expected a constant integer expression."
) ? static_cast<void> (0) : __assert_fail ("isa<ConstantSDNode>(N->getOperand(2)) && \"Expected a constant integer expression.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2685, __PRETTY_FUNCTION__))
;
2686 unsigned Reg = PMapper->Encoding;
2687 uint64_t Immed = cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
2688 unsigned State;
2689 if (Reg == AArch64PState::PAN || Reg == AArch64PState::UAO || Reg == AArch64PState::SSBS) {
2690 assert(Immed < 2 && "Bad imm")((Immed < 2 && "Bad imm") ? static_cast<void>
(0) : __assert_fail ("Immed < 2 && \"Bad imm\"", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2690, __PRETTY_FUNCTION__))
;
2691 State = AArch64::MSRpstateImm1;
2692 } else {
2693 assert(Immed < 16 && "Bad imm")((Immed < 16 && "Bad imm") ? static_cast<void>
(0) : __assert_fail ("Immed < 16 && \"Bad imm\"",
"/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2693, __PRETTY_FUNCTION__))
;
2694 State = AArch64::MSRpstateImm4;
2695 }
2696 ReplaceNode(N, CurDAG->getMachineNode(
2697 State, DL, MVT::Other,
2698 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2699 CurDAG->getTargetConstant(Immed, DL, MVT::i16),
2700 N->getOperand(0)));
2701 return true;
2702 }
2703
2704 // Use the sysreg mapper to attempt to map the remaining possible strings
2705 // to the value for the register to be used for the MSR (register)
2706 // instruction operand.
2707 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
2708 if (TheReg && TheReg->Writeable &&
2709 TheReg->haveFeatures(Subtarget->getFeatureBits()))
2710 Reg = TheReg->Encoding;
2711 else
2712 Reg = AArch64SysReg::parseGenericRegister(RegString->getString());
2713 if (Reg != -1) {
2714 ReplaceNode(N, CurDAG->getMachineNode(
2715 AArch64::MSR, DL, MVT::Other,
2716 CurDAG->getTargetConstant(Reg, DL, MVT::i32),
2717 N->getOperand(2), N->getOperand(0)));
2718 return true;
2719 }
2720
2721 return false;
2722}
2723
2724/// We've got special pseudo-instructions for these
2725bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2726 unsigned Opcode;
2727 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2728
2729 // Leave IR for LSE if subtarget supports it.
2730 if (Subtarget->hasLSE()) return false;
2731
2732 if (MemTy == MVT::i8)
2733 Opcode = AArch64::CMP_SWAP_8;
2734 else if (MemTy == MVT::i16)
2735 Opcode = AArch64::CMP_SWAP_16;
2736 else if (MemTy == MVT::i32)
2737 Opcode = AArch64::CMP_SWAP_32;
2738 else if (MemTy == MVT::i64)
2739 Opcode = AArch64::CMP_SWAP_64;
2740 else
2741 llvm_unreachable("Unknown AtomicCmpSwap type")::llvm::llvm_unreachable_internal("Unknown AtomicCmpSwap type"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2741)
;
2742
2743 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
2744 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2745 N->getOperand(0)};
2746 SDNode *CmpSwap = CurDAG->getMachineNode(
2747 Opcode, SDLoc(N),
2748 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
2749
2750 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2751 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2752
2753 ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2754 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2755 CurDAG->RemoveDeadNode(N);
2756
2757 return true;
2758}
2759
2760void AArch64DAGToDAGISel::Select(SDNode *Node) {
2761 // If we have a custom node, we already have selected!
2762 if (Node->isMachineOpcode()) {
2763 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { errs() << "== "; Node->dump(CurDAG
); errs() << "\n"; } } while (false)
;
2764 Node->setNodeId(-1);
2765 return;
2766 }
2767
2768 // Few custom selection stuff.
2769 EVT VT = Node->getValueType(0);
2770
2771 switch (Node->getOpcode()) {
2772 default:
2773 break;
2774
2775 case ISD::ATOMIC_CMP_SWAP:
2776 if (SelectCMP_SWAP(Node))
2777 return;
2778 break;
2779
2780 case ISD::READ_REGISTER:
2781 if (tryReadRegister(Node))
2782 return;
2783 break;
2784
2785 case ISD::WRITE_REGISTER:
2786 if (tryWriteRegister(Node))
2787 return;
2788 break;
2789
2790 case ISD::ADD:
2791 if (tryMLAV64LaneV128(Node))
2792 return;
2793 break;
2794
2795 case ISD::LOAD: {
2796 // Try to select as an indexed load. Fall through to normal processing
2797 // if we can't.
2798 if (tryIndexedLoad(Node))
2799 return;
2800 break;
2801 }
2802
2803 case ISD::SRL:
2804 case ISD::AND:
2805 case ISD::SRA:
2806 case ISD::SIGN_EXTEND_INREG:
2807 if (tryBitfieldExtractOp(Node))
2808 return;
2809 if (tryBitfieldInsertInZeroOp(Node))
2810 return;
2811 LLVM_FALLTHROUGH[[clang::fallthrough]];
2812 case ISD::ROTR:
2813 case ISD::SHL:
2814 if (tryShiftAmountMod(Node))
2815 return;
2816 break;
2817
2818 case ISD::SIGN_EXTEND:
2819 if (tryBitfieldExtractOpFromSExt(Node))
2820 return;
2821 break;
2822
2823 case ISD::OR:
2824 if (tryBitfieldInsertOp(Node))
2825 return;
2826 break;
2827
2828 case ISD::EXTRACT_VECTOR_ELT: {
2829 // Extracting lane zero is a special case where we can just use a plain
2830 // EXTRACT_SUBREG instruction, which will become FMOV. This is easier for
2831 // the rest of the compiler, especially the register allocator and copyi
2832 // propagation, to reason about, so is preferred when it's possible to
2833 // use it.
2834 ConstantSDNode *LaneNode = cast<ConstantSDNode>(Node->getOperand(1));
2835 // Bail and use the default Select() for non-zero lanes.
2836 if (LaneNode->getZExtValue() != 0)
2837 break;
2838 // If the element type is not the same as the result type, likewise
2839 // bail and use the default Select(), as there's more to do than just
2840 // a cross-class COPY. This catches extracts of i8 and i16 elements
2841 // since they will need an explicit zext.
2842 if (VT != Node->getOperand(0).getValueType().getVectorElementType())
2843 break;
2844 unsigned SubReg;
2845 switch (Node->getOperand(0)
2846 .getValueType()
2847 .getVectorElementType()
2848 .getSizeInBits()) {
2849 default:
2850 llvm_unreachable("Unexpected vector element type!")::llvm::llvm_unreachable_internal("Unexpected vector element type!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2850)
;
2851 case 64:
2852 SubReg = AArch64::dsub;
2853 break;
2854 case 32:
2855 SubReg = AArch64::ssub;
2856 break;
2857 case 16:
2858 SubReg = AArch64::hsub;
2859 break;
2860 case 8:
2861 llvm_unreachable("unexpected zext-requiring extract element!")::llvm::llvm_unreachable_internal("unexpected zext-requiring extract element!"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp"
, 2861)
;
2862 }
2863 SDValue Extract = CurDAG->getTargetExtractSubreg(SubReg, SDLoc(Node), VT,
2864 Node->getOperand(0));
2865 LLVM_DEBUG(dbgs() << "ISEL: Custom selection!\n=> ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "ISEL: Custom selection!\n=> "
; } } while (false)
;
2866 LLVM_DEBUG(Extract->dumpr(CurDAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { Extract->dumpr(CurDAG); } } while (false
)
;
2867 LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("aarch64-isel")) { dbgs() << "\n"; } } while (false)
;
2868 ReplaceNode(Node, Extract.getNode());
2869 return;
2870 }
2871 case ISD::Constant: {
2872 // Materialize zero constants as copies from WZR/XZR. This allows
2873 // the coalescer to propagate these into other instructions.
2874 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Node);
2875 if (ConstNode->isNullValue()) {
2876 if (VT == MVT::i32) {
2877 SDValue New = CurDAG->getCopyFromReg(
2878 CurDAG->getEntryNode(), SDLoc(Node), AArch64::WZR, MVT::i32);
2879 ReplaceNode(Node, New.getNode());
2880 return;
2881 } else if (VT == MVT::i64) {
2882 SDValue New = CurDAG->getCopyFromReg(
2883 CurDAG->getEntryNode(), SDLoc(Node), AArch64::XZR, MVT::i64);
2884 ReplaceNode(Node, New.getNode());
2885 return;
2886 }
2887 }
2888 break;
2889 }
2890
2891 case ISD::FrameIndex: {
2892 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
2893 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
2894 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
2895 const TargetLowering *TLI = getTargetLowering();
2896 SDValue TFI = CurDAG->getTargetFrameIndex(
2897 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2898 SDLoc DL(Node);
2899 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
2900 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
2901 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops);
2902 return;
2903 }
2904 case ISD::INTRINSIC_W_CHAIN: {
2905 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
2906 switch (IntNo) {
2907 default:
2908 break;
2909 case Intrinsic::aarch64_ldaxp:
2910 case Intrinsic::aarch64_ldxp: {
2911 unsigned Op =
2912 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
2913 SDValue MemAddr = Node->getOperand(2);
2914 SDLoc DL(Node);
2915 SDValue Chain = Node->getOperand(0);
2916
2917 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64,
2918 MVT::Other, MemAddr, Chain);
2919
2920 // Transfer memoperands.
2921 MachineMemOperand *MemOp =
2922 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
2923 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
2924 ReplaceNode(Node, Ld);
2925 return;
2926 }
2927 case Intrinsic::aarch64_stlxp:
2928 case Intrinsic::aarch64_stxp: {
2929 unsigned Op =
2930 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
2931 SDLoc DL(Node);
2932 SDValue Chain = Node->getOperand(0);
2933 SDValue ValLo = Node->getOperand(2);
2934 SDValue ValHi = Node->getOperand(3);
2935 SDValue MemAddr = Node->getOperand(4);
2936
2937 // Place arguments in the right order.
2938 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
2939
2940 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops);
2941 // Transfer memoperands.
2942 MachineMemOperand *MemOp =
2943 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
2944 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2945
2946 ReplaceNode(Node, St);
2947 return;
2948 }
2949 case Intrinsic::aarch64_neon_ld1x2:
2950 if (VT == MVT::v8i8) {
2951 SelectLoad(Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
2952 return;
2953 } else if (VT == MVT::v16i8) {
2954 SelectLoad(Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
2955 return;
2956 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
2957 SelectLoad(Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
2958 return;
2959 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
2960 SelectLoad(Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
2961 return;
2962 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
2963 SelectLoad(Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
2964 return;
2965 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
2966 SelectLoad(Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
2967 return;
2968 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
2969 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
2970 return;
2971 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
2972 SelectLoad(Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
2973 return;
2974 }
2975 break;
2976 case Intrinsic::aarch64_neon_ld1x3:
2977 if (VT == MVT::v8i8) {
2978 SelectLoad(Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
2979 return;
2980 } else if (VT == MVT::v16i8) {
2981 SelectLoad(Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
2982 return;
2983 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
2984 SelectLoad(Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
2985 return;
2986 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
2987 SelectLoad(Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
2988 return;
2989 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
2990 SelectLoad(Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
2991 return;
2992 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
2993 SelectLoad(Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
2994 return;
2995 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
2996 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
2997 return;
2998 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
2999 SelectLoad(Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
3000 return;
3001 }
3002 break;
3003 case Intrinsic::aarch64_neon_ld1x4:
3004 if (VT == MVT::v8i8) {
3005 SelectLoad(Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
3006 return;
3007 } else if (VT == MVT::v16i8) {
3008 SelectLoad(Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
3009 return;
3010 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3011 SelectLoad(Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
3012 return;
3013 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3014 SelectLoad(Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
3015 return;
3016 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3017 SelectLoad(Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
3018 return;
3019 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3020 SelectLoad(Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
3021 return;
3022 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3023 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
3024 return;
3025 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3026 SelectLoad(Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
3027 return;
3028 }
3029 break;
3030 case Intrinsic::aarch64_neon_ld2:
3031 if (VT == MVT::v8i8) {
3032 SelectLoad(Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
3033 return;
3034 } else if (VT == MVT::v16i8) {
3035 SelectLoad(Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
3036 return;
3037 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3038 SelectLoad(Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
3039 return;
3040 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3041 SelectLoad(Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
3042 return;
3043 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3044 SelectLoad(Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
3045 return;
3046 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3047 SelectLoad(Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
3048 return;
3049 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3050 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
3051 return;
3052 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3053 SelectLoad(Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
3054 return;
3055 }
3056 break;
3057 case Intrinsic::aarch64_neon_ld3:
3058 if (VT == MVT::v8i8) {
3059 SelectLoad(Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
3060 return;
3061 } else if (VT == MVT::v16i8) {
3062 SelectLoad(Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
3063 return;
3064 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3065 SelectLoad(Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
3066 return;
3067 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3068 SelectLoad(Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
3069 return;
3070 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3071 SelectLoad(Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
3072 return;
3073 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3074 SelectLoad(Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
3075 return;
3076 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3077 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
3078 return;
3079 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3080 SelectLoad(Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
3081 return;
3082 }
3083 break;
3084 case Intrinsic::aarch64_neon_ld4:
3085 if (VT == MVT::v8i8) {
3086 SelectLoad(Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
3087 return;
3088 } else if (VT == MVT::v16i8) {
3089 SelectLoad(Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
3090 return;
3091 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3092 SelectLoad(Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
3093 return;
3094 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3095 SelectLoad(Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
3096 return;
3097 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3098 SelectLoad(Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
3099 return;
3100 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3101 SelectLoad(Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
3102 return;
3103 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3104 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
3105 return;
3106 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3107 SelectLoad(Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
3108 return;
3109 }
3110 break;
3111 case Intrinsic::aarch64_neon_ld2r:
3112 if (VT == MVT::v8i8) {
3113 SelectLoad(Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
3114 return;
3115 } else if (VT == MVT::v16i8) {
3116 SelectLoad(Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
3117 return;
3118 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3119 SelectLoad(Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
3120 return;
3121 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3122 SelectLoad(Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
3123 return;
3124 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3125 SelectLoad(Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
3126 return;
3127 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3128 SelectLoad(Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
3129 return;
3130 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3131 SelectLoad(Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
3132 return;
3133 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3134 SelectLoad(Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
3135 return;
3136 }
3137 break;
3138 case Intrinsic::aarch64_neon_ld3r:
3139 if (VT == MVT::v8i8) {
3140 SelectLoad(Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
3141 return;
3142 } else if (VT == MVT::v16i8) {
3143 SelectLoad(Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
3144 return;
3145 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3146 SelectLoad(Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
3147 return;
3148 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3149 SelectLoad(Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
3150 return;
3151 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3152 SelectLoad(Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
3153 return;
3154 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3155 SelectLoad(Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
3156 return;
3157 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3158 SelectLoad(Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
3159 return;
3160 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3161 SelectLoad(Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
3162 return;
3163 }
3164 break;
3165 case Intrinsic::aarch64_neon_ld4r:
3166 if (VT == MVT::v8i8) {
3167 SelectLoad(Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
3168 return;
3169 } else if (VT == MVT::v16i8) {
3170 SelectLoad(Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
3171 return;
3172 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3173 SelectLoad(Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
3174 return;
3175 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3176 SelectLoad(Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
3177 return;
3178 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3179 SelectLoad(Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
3180 return;
3181 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3182 SelectLoad(Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
3183 return;
3184 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3185 SelectLoad(Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
3186 return;
3187 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3188 SelectLoad(Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
3189 return;
3190 }
3191 break;
3192 case Intrinsic::aarch64_neon_ld2lane:
3193 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3194 SelectLoadLane(Node, 2, AArch64::LD2i8);
3195 return;
3196 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3197 VT == MVT::v8f16) {
3198 SelectLoadLane(Node, 2, AArch64::LD2i16);
3199 return;
3200 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3201 VT == MVT::v2f32) {
3202 SelectLoadLane(Node, 2, AArch64::LD2i32);
3203 return;
3204 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3205 VT == MVT::v1f64) {
3206 SelectLoadLane(Node, 2, AArch64::LD2i64);
3207 return;
3208 }
3209 break;
3210 case Intrinsic::aarch64_neon_ld3lane:
3211 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3212 SelectLoadLane(Node, 3, AArch64::LD3i8);
3213 return;
3214 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3215 VT == MVT::v8f16) {
3216 SelectLoadLane(Node, 3, AArch64::LD3i16);
3217 return;
3218 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3219 VT == MVT::v2f32) {
3220 SelectLoadLane(Node, 3, AArch64::LD3i32);
3221 return;
3222 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3223 VT == MVT::v1f64) {
3224 SelectLoadLane(Node, 3, AArch64::LD3i64);
3225 return;
3226 }
3227 break;
3228 case Intrinsic::aarch64_neon_ld4lane:
3229 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3230 SelectLoadLane(Node, 4, AArch64::LD4i8);
3231 return;
3232 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3233 VT == MVT::v8f16) {
3234 SelectLoadLane(Node, 4, AArch64::LD4i16);
3235 return;
3236 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3237 VT == MVT::v2f32) {
3238 SelectLoadLane(Node, 4, AArch64::LD4i32);
3239 return;
3240 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3241 VT == MVT::v1f64) {
3242 SelectLoadLane(Node, 4, AArch64::LD4i64);
3243 return;
3244 }
3245 break;
3246 }
3247 } break;
3248 case ISD::INTRINSIC_WO_CHAIN: {
3249 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(0))->getZExtValue();
3250 switch (IntNo) {
3251 default:
3252 break;
3253 case Intrinsic::aarch64_neon_tbl2:
3254 SelectTable(Node, 2,
3255 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
3256 false);
3257 return;
3258 case Intrinsic::aarch64_neon_tbl3:
3259 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
3260 : AArch64::TBLv16i8Three,
3261 false);
3262 return;
3263 case Intrinsic::aarch64_neon_tbl4:
3264 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
3265 : AArch64::TBLv16i8Four,
3266 false);
3267 return;
3268 case Intrinsic::aarch64_neon_tbx2:
3269 SelectTable(Node, 2,
3270 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
3271 true);
3272 return;
3273 case Intrinsic::aarch64_neon_tbx3:
3274 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
3275 : AArch64::TBXv16i8Three,
3276 true);
3277 return;
3278 case Intrinsic::aarch64_neon_tbx4:
3279 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
3280 : AArch64::TBXv16i8Four,
3281 true);
3282 return;
3283 case Intrinsic::aarch64_neon_smull:
3284 case Intrinsic::aarch64_neon_umull:
3285 if (tryMULLV64LaneV128(IntNo, Node))
3286 return;
3287 break;
3288 }
3289 break;
3290 }
3291 case ISD::INTRINSIC_VOID: {
3292 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
3293 if (Node->getNumOperands() >= 3)
3294 VT = Node->getOperand(2)->getValueType(0);
3295 switch (IntNo) {
3296 default:
3297 break;
3298 case Intrinsic::aarch64_neon_st1x2: {
3299 if (VT == MVT::v8i8) {
3300 SelectStore(Node, 2, AArch64::ST1Twov8b);
3301 return;
3302 } else if (VT == MVT::v16i8) {
3303 SelectStore(Node, 2, AArch64::ST1Twov16b);
3304 return;
3305 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3306 SelectStore(Node, 2, AArch64::ST1Twov4h);
3307 return;
3308 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3309 SelectStore(Node, 2, AArch64::ST1Twov8h);
3310 return;
3311 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3312 SelectStore(Node, 2, AArch64::ST1Twov2s);
3313 return;
3314 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3315 SelectStore(Node, 2, AArch64::ST1Twov4s);
3316 return;
3317 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3318 SelectStore(Node, 2, AArch64::ST1Twov2d);
3319 return;
3320 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3321 SelectStore(Node, 2, AArch64::ST1Twov1d);
3322 return;
3323 }
3324 break;
3325 }
3326 case Intrinsic::aarch64_neon_st1x3: {
3327 if (VT == MVT::v8i8) {
3328 SelectStore(Node, 3, AArch64::ST1Threev8b);
3329 return;
3330 } else if (VT == MVT::v16i8) {
3331 SelectStore(Node, 3, AArch64::ST1Threev16b);
3332 return;
3333 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3334 SelectStore(Node, 3, AArch64::ST1Threev4h);
3335 return;
3336 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3337 SelectStore(Node, 3, AArch64::ST1Threev8h);
3338 return;
3339 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3340 SelectStore(Node, 3, AArch64::ST1Threev2s);
3341 return;
3342 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3343 SelectStore(Node, 3, AArch64::ST1Threev4s);
3344 return;
3345 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3346 SelectStore(Node, 3, AArch64::ST1Threev2d);
3347 return;
3348 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3349 SelectStore(Node, 3, AArch64::ST1Threev1d);
3350 return;
3351 }
3352 break;
3353 }
3354 case Intrinsic::aarch64_neon_st1x4: {
3355 if (VT == MVT::v8i8) {
3356 SelectStore(Node, 4, AArch64::ST1Fourv8b);
3357 return;
3358 } else if (VT == MVT::v16i8) {
3359 SelectStore(Node, 4, AArch64::ST1Fourv16b);
3360 return;
3361 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3362 SelectStore(Node, 4, AArch64::ST1Fourv4h);
3363 return;
3364 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3365 SelectStore(Node, 4, AArch64::ST1Fourv8h);
3366 return;
3367 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3368 SelectStore(Node, 4, AArch64::ST1Fourv2s);
3369 return;
3370 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3371 SelectStore(Node, 4, AArch64::ST1Fourv4s);
3372 return;
3373 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3374 SelectStore(Node, 4, AArch64::ST1Fourv2d);
3375 return;
3376 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3377 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3378 return;
3379 }
3380 break;
3381 }
3382 case Intrinsic::aarch64_neon_st2: {
3383 if (VT == MVT::v8i8) {
3384 SelectStore(Node, 2, AArch64::ST2Twov8b);
3385 return;
3386 } else if (VT == MVT::v16i8) {
3387 SelectStore(Node, 2, AArch64::ST2Twov16b);
3388 return;
3389 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3390 SelectStore(Node, 2, AArch64::ST2Twov4h);
3391 return;
3392 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3393 SelectStore(Node, 2, AArch64::ST2Twov8h);
3394 return;
3395 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3396 SelectStore(Node, 2, AArch64::ST2Twov2s);
3397 return;
3398 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3399 SelectStore(Node, 2, AArch64::ST2Twov4s);
3400 return;
3401 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3402 SelectStore(Node, 2, AArch64::ST2Twov2d);
3403 return;
3404 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3405 SelectStore(Node, 2, AArch64::ST1Twov1d);
3406 return;
3407 }
3408 break;
3409 }
3410 case Intrinsic::aarch64_neon_st3: {
3411 if (VT == MVT::v8i8) {
3412 SelectStore(Node, 3, AArch64::ST3Threev8b);
3413 return;
3414 } else if (VT == MVT::v16i8) {
3415 SelectStore(Node, 3, AArch64::ST3Threev16b);
3416 return;
3417 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3418 SelectStore(Node, 3, AArch64::ST3Threev4h);
3419 return;
3420 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3421 SelectStore(Node, 3, AArch64::ST3Threev8h);
3422 return;
3423 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3424 SelectStore(Node, 3, AArch64::ST3Threev2s);
3425 return;
3426 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3427 SelectStore(Node, 3, AArch64::ST3Threev4s);
3428 return;
3429 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3430 SelectStore(Node, 3, AArch64::ST3Threev2d);
3431 return;
3432 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3433 SelectStore(Node, 3, AArch64::ST1Threev1d);
3434 return;
3435 }
3436 break;
3437 }
3438 case Intrinsic::aarch64_neon_st4: {
3439 if (VT == MVT::v8i8) {
3440 SelectStore(Node, 4, AArch64::ST4Fourv8b);
3441 return;
3442 } else if (VT == MVT::v16i8) {
3443 SelectStore(Node, 4, AArch64::ST4Fourv16b);
3444 return;
3445 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3446 SelectStore(Node, 4, AArch64::ST4Fourv4h);
3447 return;
3448 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3449 SelectStore(Node, 4, AArch64::ST4Fourv8h);
3450 return;
3451 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3452 SelectStore(Node, 4, AArch64::ST4Fourv2s);
3453 return;
3454 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3455 SelectStore(Node, 4, AArch64::ST4Fourv4s);
3456 return;
3457 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3458 SelectStore(Node, 4, AArch64::ST4Fourv2d);
3459 return;
3460 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3461 SelectStore(Node, 4, AArch64::ST1Fourv1d);
3462 return;
3463 }
3464 break;
3465 }
3466 case Intrinsic::aarch64_neon_st2lane: {
3467 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3468 SelectStoreLane(Node, 2, AArch64::ST2i8);
3469 return;
3470 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3471 VT == MVT::v8f16) {
3472 SelectStoreLane(Node, 2, AArch64::ST2i16);
3473 return;
3474 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3475 VT == MVT::v2f32) {
3476 SelectStoreLane(Node, 2, AArch64::ST2i32);
3477 return;
3478 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3479 VT == MVT::v1f64) {
3480 SelectStoreLane(Node, 2, AArch64::ST2i64);
3481 return;
3482 }
3483 break;
3484 }
3485 case Intrinsic::aarch64_neon_st3lane: {
3486 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3487 SelectStoreLane(Node, 3, AArch64::ST3i8);
3488 return;
3489 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3490 VT == MVT::v8f16) {
3491 SelectStoreLane(Node, 3, AArch64::ST3i16);
3492 return;
3493 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3494 VT == MVT::v2f32) {
3495 SelectStoreLane(Node, 3, AArch64::ST3i32);
3496 return;
3497 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3498 VT == MVT::v1f64) {
3499 SelectStoreLane(Node, 3, AArch64::ST3i64);
3500 return;
3501 }
3502 break;
3503 }
3504 case Intrinsic::aarch64_neon_st4lane: {
3505 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3506 SelectStoreLane(Node, 4, AArch64::ST4i8);
3507 return;
3508 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3509 VT == MVT::v8f16) {
3510 SelectStoreLane(Node, 4, AArch64::ST4i16);
3511 return;
3512 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3513 VT == MVT::v2f32) {
3514 SelectStoreLane(Node, 4, AArch64::ST4i32);
3515 return;
3516 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3517 VT == MVT::v1f64) {
3518 SelectStoreLane(Node, 4, AArch64::ST4i64);
3519 return;
3520 }
3521 break;
3522 }
3523 }
3524 break;
3525 }
3526 case AArch64ISD::LD2post: {
3527 if (VT == MVT::v8i8) {
3528 SelectPostLoad(Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
3529 return;
3530 } else if (VT == MVT::v16i8) {
3531 SelectPostLoad(Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
3532 return;
3533 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3534 SelectPostLoad(Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
3535 return;
3536 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3537 SelectPostLoad(Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
3538 return;
3539 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3540 SelectPostLoad(Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
3541 return;
3542 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3543 SelectPostLoad(Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
3544 return;
3545 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3546 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3547 return;
3548 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3549 SelectPostLoad(Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
3550 return;
3551 }
3552 break;
3553 }
3554 case AArch64ISD::LD3post: {
3555 if (VT == MVT::v8i8) {
3556 SelectPostLoad(Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
3557 return;
3558 } else if (VT == MVT::v16i8) {
3559 SelectPostLoad(Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
3560 return;
3561 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3562 SelectPostLoad(Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
3563 return;
3564 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3565 SelectPostLoad(Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
3566 return;
3567 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3568 SelectPostLoad(Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
3569 return;
3570 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3571 SelectPostLoad(Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
3572 return;
3573 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3574 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3575 return;
3576 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3577 SelectPostLoad(Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
3578 return;
3579 }
3580 break;
3581 }
3582 case AArch64ISD::LD4post: {
3583 if (VT == MVT::v8i8) {
3584 SelectPostLoad(Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
3585 return;
3586 } else if (VT == MVT::v16i8) {
3587 SelectPostLoad(Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
3588 return;
3589 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3590 SelectPostLoad(Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
3591 return;
3592 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3593 SelectPostLoad(Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
3594 return;
3595 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3596 SelectPostLoad(Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
3597 return;
3598 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3599 SelectPostLoad(Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
3600 return;
3601 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3602 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
3603 return;
3604 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3605 SelectPostLoad(Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
3606 return;
3607 }
3608 break;
3609 }
3610 case AArch64ISD::LD1x2post: {
3611 if (VT == MVT::v8i8) {
3612 SelectPostLoad(Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
3613 return;
3614 } else if (VT == MVT::v16i8) {
3615 SelectPostLoad(Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
3616 return;
3617 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3618 SelectPostLoad(Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
3619 return;
3620 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3621 SelectPostLoad(Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
3622 return;
3623 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3624 SelectPostLoad(Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
3625 return;
3626 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3627 SelectPostLoad(Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
3628 return;
3629 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3630 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
3631 return;
3632 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3633 SelectPostLoad(Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
3634 return;
3635 }
3636 break;
3637 }
3638 case AArch64ISD::LD1x3post: {
3639 if (VT == MVT::v8i8) {
3640 SelectPostLoad(Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
3641 return;
3642 } else if (VT == MVT::v16i8) {
3643 SelectPostLoad(Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
3644 return;
3645 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3646 SelectPostLoad(Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
3647 return;
3648 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3649 SelectPostLoad(Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
3650 return;
3651 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3652 SelectPostLoad(Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
3653 return;
3654 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3655 SelectPostLoad(Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
3656 return;
3657 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3658 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
3659 return;
3660 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3661 SelectPostLoad(Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
3662 return;
3663 }
3664 break;
3665 }
3666 case AArch64ISD::LD1x4post: {
3667 if (VT == MVT::v8i8) {
3668 SelectPostLoad(Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
3669 return;
3670 } else if (VT == MVT::v16i8) {
3671 SelectPostLoad(Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
3672 return;
3673 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3674 SelectPostLoad(Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
3675 return;
3676 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3677 SelectPostLoad(Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
3678 return;
3679 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3680 SelectPostLoad(Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
3681 return;
3682 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3683 SelectPostLoad(Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
3684 return;
3685 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3686 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
3687 return;
3688 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3689 SelectPostLoad(Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
3690 return;
3691 }
3692 break;
3693 }
3694 case AArch64ISD::LD1DUPpost: {
3695 if (VT == MVT::v8i8) {
3696 SelectPostLoad(Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
3697 return;
3698 } else if (VT == MVT::v16i8) {
3699 SelectPostLoad(Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
3700 return;
3701 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3702 SelectPostLoad(Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
3703 return;
3704 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3705 SelectPostLoad(Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
3706 return;
3707 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3708 SelectPostLoad(Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
3709 return;
3710 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3711 SelectPostLoad(Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
3712 return;
3713 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3714 SelectPostLoad(Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
3715 return;
3716 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3717 SelectPostLoad(Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
3718 return;
3719 }
3720 break;
3721 }
3722 case AArch64ISD::LD2DUPpost: {
3723 if (VT == MVT::v8i8) {
3724 SelectPostLoad(Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
3725 return;
3726 } else if (VT == MVT::v16i8) {
3727 SelectPostLoad(Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
3728 return;
3729 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3730 SelectPostLoad(Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
3731 return;
3732 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3733 SelectPostLoad(Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
3734 return;
3735 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3736 SelectPostLoad(Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
3737 return;
3738 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3739 SelectPostLoad(Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
3740 return;
3741 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3742 SelectPostLoad(Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
3743 return;
3744 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3745 SelectPostLoad(Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
3746 return;
3747 }
3748 break;
3749 }
3750 case AArch64ISD::LD3DUPpost: {
3751 if (VT == MVT::v8i8) {
3752 SelectPostLoad(Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
3753 return;
3754 } else if (VT == MVT::v16i8) {
3755 SelectPostLoad(Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
3756 return;
3757 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3758 SelectPostLoad(Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
3759 return;
3760 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3761 SelectPostLoad(Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
3762 return;
3763 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3764 SelectPostLoad(Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
3765 return;
3766 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3767 SelectPostLoad(Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
3768 return;
3769 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3770 SelectPostLoad(Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
3771 return;
3772 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3773 SelectPostLoad(Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
3774 return;
3775 }
3776 break;
3777 }
3778 case AArch64ISD::LD4DUPpost: {
3779 if (VT == MVT::v8i8) {
3780 SelectPostLoad(Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
3781 return;
3782 } else if (VT == MVT::v16i8) {
3783 SelectPostLoad(Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
3784 return;
3785 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3786 SelectPostLoad(Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
3787 return;
3788 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3789 SelectPostLoad(Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
3790 return;
3791 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3792 SelectPostLoad(Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
3793 return;
3794 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3795 SelectPostLoad(Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
3796 return;
3797 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3798 SelectPostLoad(Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
3799 return;
3800 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3801 SelectPostLoad(Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
3802 return;
3803 }
3804 break;
3805 }
3806 case AArch64ISD::LD1LANEpost: {
3807 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3808 SelectPostLoadLane(Node, 1, AArch64::LD1i8_POST);
3809 return;
3810 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3811 VT == MVT::v8f16) {
3812 SelectPostLoadLane(Node, 1, AArch64::LD1i16_POST);
3813 return;
3814 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3815 VT == MVT::v2f32) {
3816 SelectPostLoadLane(Node, 1, AArch64::LD1i32_POST);
3817 return;
3818 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3819 VT == MVT::v1f64) {
3820 SelectPostLoadLane(Node, 1, AArch64::LD1i64_POST);
3821 return;
3822 }
3823 break;
3824 }
3825 case AArch64ISD::LD2LANEpost: {
3826 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3827 SelectPostLoadLane(Node, 2, AArch64::LD2i8_POST);
3828 return;
3829 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3830 VT == MVT::v8f16) {
3831 SelectPostLoadLane(Node, 2, AArch64::LD2i16_POST);
3832 return;
3833 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3834 VT == MVT::v2f32) {
3835 SelectPostLoadLane(Node, 2, AArch64::LD2i32_POST);
3836 return;
3837 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3838 VT == MVT::v1f64) {
3839 SelectPostLoadLane(Node, 2, AArch64::LD2i64_POST);
3840 return;
3841 }
3842 break;
3843 }
3844 case AArch64ISD::LD3LANEpost: {
3845 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3846 SelectPostLoadLane(Node, 3, AArch64::LD3i8_POST);
3847 return;
3848 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3849 VT == MVT::v8f16) {
3850 SelectPostLoadLane(Node, 3, AArch64::LD3i16_POST);
3851 return;
3852 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3853 VT == MVT::v2f32) {
3854 SelectPostLoadLane(Node, 3, AArch64::LD3i32_POST);
3855 return;
3856 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3857 VT == MVT::v1f64) {
3858 SelectPostLoadLane(Node, 3, AArch64::LD3i64_POST);
3859 return;
3860 }
3861 break;
3862 }
3863 case AArch64ISD::LD4LANEpost: {
3864 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
3865 SelectPostLoadLane(Node, 4, AArch64::LD4i8_POST);
3866 return;
3867 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
3868 VT == MVT::v8f16) {
3869 SelectPostLoadLane(Node, 4, AArch64::LD4i16_POST);
3870 return;
3871 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
3872 VT == MVT::v2f32) {
3873 SelectPostLoadLane(Node, 4, AArch64::LD4i32_POST);
3874 return;
3875 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
3876 VT == MVT::v1f64) {
3877 SelectPostLoadLane(Node, 4, AArch64::LD4i64_POST);
3878 return;
3879 }
3880 break;
3881 }
3882 case AArch64ISD::ST2post: {
3883 VT = Node->getOperand(1).getValueType();
3884 if (VT == MVT::v8i8) {
3885 SelectPostStore(Node, 2, AArch64::ST2Twov8b_POST);
3886 return;
3887 } else if (VT == MVT::v16i8) {
3888 SelectPostStore(Node, 2, AArch64::ST2Twov16b_POST);
3889 return;
3890 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3891 SelectPostStore(Node, 2, AArch64::ST2Twov4h_POST);
3892 return;
3893 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3894 SelectPostStore(Node, 2, AArch64::ST2Twov8h_POST);
3895 return;
3896 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3897 SelectPostStore(Node, 2, AArch64::ST2Twov2s_POST);
3898 return;
3899 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3900 SelectPostStore(Node, 2, AArch64::ST2Twov4s_POST);
3901 return;
3902 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3903 SelectPostStore(Node, 2, AArch64::ST2Twov2d_POST);
3904 return;
3905 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3906 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
3907 return;
3908 }
3909 break;
3910 }
3911 case AArch64ISD::ST3post: {
3912 VT = Node->getOperand(1).getValueType();
3913 if (VT == MVT::v8i8) {
3914 SelectPostStore(Node, 3, AArch64::ST3Threev8b_POST);
3915 return;
3916 } else if (VT == MVT::v16i8) {
3917 SelectPostStore(Node, 3, AArch64::ST3Threev16b_POST);
3918 return;
3919 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3920 SelectPostStore(Node, 3, AArch64::ST3Threev4h_POST);
3921 return;
3922 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3923 SelectPostStore(Node, 3, AArch64::ST3Threev8h_POST);
3924 return;
3925 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3926 SelectPostStore(Node, 3, AArch64::ST3Threev2s_POST);
3927 return;
3928 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3929 SelectPostStore(Node, 3, AArch64::ST3Threev4s_POST);
3930 return;
3931 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3932 SelectPostStore(Node, 3, AArch64::ST3Threev2d_POST);
3933 return;
3934 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3935 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
3936 return;
3937 }
3938 break;
3939 }
3940 case AArch64ISD::ST4post: {
3941 VT = Node->getOperand(1).getValueType();
3942 if (VT == MVT::v8i8) {
3943 SelectPostStore(Node, 4, AArch64::ST4Fourv8b_POST);
3944 return;
3945 } else if (VT == MVT::v16i8) {
3946 SelectPostStore(Node, 4, AArch64::ST4Fourv16b_POST);
3947 return;
3948 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3949 SelectPostStore(Node, 4, AArch64::ST4Fourv4h_POST);
3950 return;
3951 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3952 SelectPostStore(Node, 4, AArch64::ST4Fourv8h_POST);
3953 return;
3954 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3955 SelectPostStore(Node, 4, AArch64::ST4Fourv2s_POST);
3956 return;
3957 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3958 SelectPostStore(Node, 4, AArch64::ST4Fourv4s_POST);
3959 return;
3960 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3961 SelectPostStore(Node, 4, AArch64::ST4Fourv2d_POST);
3962 return;
3963 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3964 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
3965 return;
3966 }
3967 break;
3968 }
3969 case AArch64ISD::ST1x2post: {
3970 VT = Node->getOperand(1).getValueType();
3971 if (VT == MVT::v8i8) {
3972 SelectPostStore(Node, 2, AArch64::ST1Twov8b_POST);
3973 return;
3974 } else if (VT == MVT::v16i8) {
3975 SelectPostStore(Node, 2, AArch64::ST1Twov16b_POST);
3976 return;
3977 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
3978 SelectPostStore(Node, 2, AArch64::ST1Twov4h_POST);
3979 return;
3980 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
3981 SelectPostStore(Node, 2, AArch64::ST1Twov8h_POST);
3982 return;
3983 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
3984 SelectPostStore(Node, 2, AArch64::ST1Twov2s_POST);
3985 return;
3986 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
3987 SelectPostStore(Node, 2, AArch64::ST1Twov4s_POST);
3988 return;
3989 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
3990 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
3991 return;
3992 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
3993 SelectPostStore(Node, 2, AArch64::ST1Twov2d_POST);
3994 return;
3995 }
3996 break;
3997 }
3998 case AArch64ISD::ST1x3post: {
3999 VT = Node->getOperand(1).getValueType();
4000 if (VT == MVT::v8i8) {
4001 SelectPostStore(Node, 3, AArch64::ST1Threev8b_POST);
4002 return;
4003 } else if (VT == MVT::v16i8) {
4004 SelectPostStore(Node, 3, AArch64::ST1Threev16b_POST);
4005 return;
4006 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4007 SelectPostStore(Node, 3, AArch64::ST1Threev4h_POST);
4008 return;
4009 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4010 SelectPostStore(Node, 3, AArch64::ST1Threev8h_POST);
4011 return;
4012 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4013 SelectPostStore(Node, 3, AArch64::ST1Threev2s_POST);
4014 return;
4015 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4016 SelectPostStore(Node, 3, AArch64::ST1Threev4s_POST);
4017 return;
4018 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4019 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
4020 return;
4021 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4022 SelectPostStore(Node, 3, AArch64::ST1Threev2d_POST);
4023 return;
4024 }
4025 break;
4026 }
4027 case AArch64ISD::ST1x4post: {
4028 VT = Node->getOperand(1).getValueType();
4029 if (VT == MVT::v8i8) {
4030 SelectPostStore(Node, 4, AArch64::ST1Fourv8b_POST);
4031 return;
4032 } else if (VT == MVT::v16i8) {
4033 SelectPostStore(Node, 4, AArch64::ST1Fourv16b_POST);
4034 return;
4035 } else if (VT == MVT::v4i16 || VT == MVT::v4f16) {
4036 SelectPostStore(Node, 4, AArch64::ST1Fourv4h_POST);
4037 return;
4038 } else if (VT == MVT::v8i16 || VT == MVT::v8f16) {
4039 SelectPostStore(Node, 4, AArch64::ST1Fourv8h_POST);
4040 return;
4041 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
4042 SelectPostStore(Node, 4, AArch64::ST1Fourv2s_POST);
4043 return;
4044 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
4045 SelectPostStore(Node, 4, AArch64::ST1Fourv4s_POST);
4046 return;
4047 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
4048 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
4049 return;
4050 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
4051 SelectPostStore(Node, 4, AArch64::ST1Fourv2d_POST);
4052 return;
4053 }
4054 break;
4055 }
4056 case AArch64ISD::ST2LANEpost: {
4057 VT = Node->getOperand(1).getValueType();
4058 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4059 SelectPostStoreLane(Node, 2, AArch64::ST2i8_POST);
4060 return;
4061 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4062 VT == MVT::v8f16) {
4063 SelectPostStoreLane(Node, 2, AArch64::ST2i16_POST);
4064 return;
4065 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4066 VT == MVT::v2f32) {
4067 SelectPostStoreLane(Node, 2, AArch64::ST2i32_POST);
4068 return;
4069 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4070 VT == MVT::v1f64) {
4071 SelectPostStoreLane(Node, 2, AArch64::ST2i64_POST);
4072 return;
4073 }
4074 break;
4075 }
4076 case AArch64ISD::ST3LANEpost: {
4077 VT = Node->getOperand(1).getValueType();
4078 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4079 SelectPostStoreLane(Node, 3, AArch64::ST3i8_POST);
4080 return;
4081 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4082 VT == MVT::v8f16) {
4083 SelectPostStoreLane(Node, 3, AArch64::ST3i16_POST);
4084 return;
4085 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4086 VT == MVT::v2f32) {
4087 SelectPostStoreLane(Node, 3, AArch64::ST3i32_POST);
4088 return;
4089 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4090 VT == MVT::v1f64) {
4091 SelectPostStoreLane(Node, 3, AArch64::ST3i64_POST);
4092 return;
4093 }
4094 break;
4095 }
4096 case AArch64ISD::ST4LANEpost: {
4097 VT = Node->getOperand(1).getValueType();
4098 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
4099 SelectPostStoreLane(Node, 4, AArch64::ST4i8_POST);
4100 return;
4101 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
4102 VT == MVT::v8f16) {
4103 SelectPostStoreLane(Node, 4, AArch64::ST4i16_POST);
4104 return;
4105 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
4106 VT == MVT::v2f32) {
4107 SelectPostStoreLane(Node, 4, AArch64::ST4i32_POST);
4108 return;
4109 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
4110 VT == MVT::v1f64) {
4111 SelectPostStoreLane(Node, 4, AArch64::ST4i64_POST);
4112 return;
4113 }
4114 break;
4115 }
4116 }
4117
4118 // Select the default instruction
4119 SelectCode(Node);
4120}
4121
4122/// createAArch64ISelDag - This pass converts a legalized DAG into a
4123/// AArch64-specific DAG, ready for instruction scheduling.
4124FunctionPass *llvm::createAArch64ISelDag(AArch64TargetMachine &TM,
4125 CodeGenOpt::Level OptLevel) {
4126 return new AArch64DAGToDAGISel(TM, OptLevel);
4127}

/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file declares the SDNode class and derived classes, which are used to
11// represent the nodes and operations present in a SelectionDAG. These nodes
12// and operations are machine code level operations, with some similarities to
13// the GCC RTL representation.
14//
15// Clients should include the SelectionDAG.h file instead of this file directly.
16//
17//===----------------------------------------------------------------------===//
18
19#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
20#define LLVM_CODEGEN_SELECTIONDAGNODES_H
21
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/ArrayRef.h"
24#include "llvm/ADT/BitVector.h"
25#include "llvm/ADT/FoldingSet.h"
26#include "llvm/ADT/GraphTraits.h"
27#include "llvm/ADT/SmallPtrSet.h"
28#include "llvm/ADT/SmallVector.h"
29#include "llvm/ADT/ilist_node.h"
30#include "llvm/ADT/iterator.h"
31#include "llvm/ADT/iterator_range.h"
32#include "llvm/CodeGen/ISDOpcodes.h"
33#include "llvm/CodeGen/MachineMemOperand.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include <algorithm>
47#include <cassert>
48#include <climits>
49#include <cstddef>
50#include <cstdint>
51#include <cstring>
52#include <iterator>
53#include <string>
54#include <tuple>
55
56namespace llvm {
57
58class APInt;
59class Constant;
60template <typename T> struct DenseMapInfo;
61class GlobalValue;
62class MachineBasicBlock;
63class MachineConstantPoolValue;
64class MCSymbol;
65class raw_ostream;
66class SDNode;
67class SelectionDAG;
68class Type;
69class Value;
70
71void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
72 bool force = false);
73
74/// This represents a list of ValueType's that has been intern'd by
75/// a SelectionDAG. Instances of this simple value class are returned by
76/// SelectionDAG::getVTList(...).
77///
78struct SDVTList {
79 const EVT *VTs;
80 unsigned int NumVTs;
81};
82
83namespace ISD {
84
85 /// Node predicates
86
87 /// If N is a BUILD_VECTOR node whose elements are all the same constant or
88 /// undefined, return true and return the constant value in \p SplatValue.
89 bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
90
91 /// Return true if the specified node is a BUILD_VECTOR where all of the
92 /// elements are ~0 or undef.
93 bool isBuildVectorAllOnes(const SDNode *N);
94
95 /// Return true if the specified node is a BUILD_VECTOR where all of the
96 /// elements are 0 or undef.
97 bool isBuildVectorAllZeros(const SDNode *N);
98
99 /// Return true if the specified node is a BUILD_VECTOR node of all
100 /// ConstantSDNode or undef.
101 bool isBuildVectorOfConstantSDNodes(const SDNode *N);
102
103 /// Return true if the specified node is a BUILD_VECTOR node of all
104 /// ConstantFPSDNode or undef.
105 bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
106
107 /// Return true if the node has at least one operand and all operands of the
108 /// specified node are ISD::UNDEF.
109 bool allOperandsUndef(const SDNode *N);
110
111} // end namespace ISD
112
113//===----------------------------------------------------------------------===//
114/// Unlike LLVM values, Selection DAG nodes may return multiple
115/// values as the result of a computation. Many nodes return multiple values,
116/// from loads (which define a token and a return value) to ADDC (which returns
117/// a result and a carry value), to calls (which may return an arbitrary number
118/// of values).
119///
120/// As such, each use of a SelectionDAG computation must indicate the node that
121/// computes it as well as which return value to use from that node. This pair
122/// of information is represented with the SDValue value type.
123///
124class SDValue {
125 friend struct DenseMapInfo<SDValue>;
126
127 SDNode *Node = nullptr; // The node defining the value we are using.
128 unsigned ResNo = 0; // Which return value of the node we are using.
129
130public:
131 SDValue() = default;
132 SDValue(SDNode *node, unsigned resno);
133
134 /// get the index which selects a specific result in the SDNode
135 unsigned getResNo() const { return ResNo; }
136
137 /// get the SDNode which holds the desired result
138 SDNode *getNode() const { return Node; }
139
140 /// set the SDNode
141 void setNode(SDNode *N) { Node = N; }
142
143 inline SDNode *operator->() const { return Node; }
144
145 bool operator==(const SDValue &O) const {
146 return Node == O.Node && ResNo == O.ResNo;
147 }
148 bool operator!=(const SDValue &O) const {
149 return !operator==(O);
150 }
151 bool operator<(const SDValue &O) const {
152 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
153 }
154 explicit operator bool() const {
155 return Node != nullptr;
156 }
157
158 SDValue getValue(unsigned R) const {
159 return SDValue(Node, R);
160 }
161
162 /// Return true if this node is an operand of N.
163 bool isOperandOf(const SDNode *N) const;
164
165 /// Return the ValueType of the referenced return value.
166 inline EVT getValueType() const;
167
168 /// Return the simple ValueType of the referenced return value.
169 MVT getSimpleValueType() const {
170 return getValueType().getSimpleVT();
171 }
172
173 /// Returns the size of the value in bits.
174 unsigned getValueSizeInBits() const {
175 return getValueType().getSizeInBits();
176 }
177
178 unsigned getScalarValueSizeInBits() const {
179 return getValueType().getScalarType().getSizeInBits();
180 }
181
182 // Forwarding methods - These forward to the corresponding methods in SDNode.
183 inline unsigned getOpcode() const;
184 inline unsigned getNumOperands() const;
185 inline const SDValue &getOperand(unsigned i) const;
186 inline uint64_t getConstantOperandVal(unsigned i) const;
187 inline bool isTargetMemoryOpcode() const;
188 inline bool isTargetOpcode() const;
189 inline bool isMachineOpcode() const;
190 inline bool isUndef() const;
191 inline unsigned getMachineOpcode() const;
192 inline const DebugLoc &getDebugLoc() const;
193 inline void dump() const;
194 inline void dump(const SelectionDAG *G) const;
195 inline void dumpr() const;
196 inline void dumpr(const SelectionDAG *G) const;
197
198 /// Return true if this operand (which must be a chain) reaches the
199 /// specified operand without crossing any side-effecting instructions.
200 /// In practice, this looks through token factors and non-volatile loads.
201 /// In order to remain efficient, this only
202 /// looks a couple of nodes in, it does not do an exhaustive search.
203 bool reachesChainWithoutSideEffects(SDValue Dest,
204 unsigned Depth = 2) const;
205
206 /// Return true if there are no nodes using value ResNo of Node.
207 inline bool use_empty() const;
208
209 /// Return true if there is exactly one node using value ResNo of Node.
210 inline bool hasOneUse() const;
211};
212
213template<> struct DenseMapInfo<SDValue> {
214 static inline SDValue getEmptyKey() {
215 SDValue V;
216 V.ResNo = -1U;
217 return V;
218 }
219
220 static inline SDValue getTombstoneKey() {
221 SDValue V;
222 V.ResNo = -2U;
223 return V;
224 }
225
226 static unsigned getHashValue(const SDValue &Val) {
227 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
228 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
229 }
230
231 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
232 return LHS == RHS;
233 }
234};
235template <> struct isPodLike<SDValue> { static const bool value = true; };
236
237/// Allow casting operators to work directly on
238/// SDValues as if they were SDNode*'s.
239template<> struct simplify_type<SDValue> {
240 using SimpleType = SDNode *;
241
242 static SimpleType getSimplifiedValue(SDValue &Val) {
243 return Val.getNode();
244 }
245};
246template<> struct simplify_type<const SDValue> {
247 using SimpleType = /*const*/ SDNode *;
248
249 static SimpleType getSimplifiedValue(const SDValue &Val) {
250 return Val.getNode();
251 }
252};
253
254/// Represents a use of a SDNode. This class holds an SDValue,
255/// which records the SDNode being used and the result number, a
256/// pointer to the SDNode using the value, and Next and Prev pointers,
257/// which link together all the uses of an SDNode.
258///
259class SDUse {
260 /// Val - The value being used.
261 SDValue Val;
262 /// User - The user of this value.
263 SDNode *User = nullptr;
264 /// Prev, Next - Pointers to the uses list of the SDNode referred by
265 /// this operand.
266 SDUse **Prev = nullptr;
267 SDUse *Next = nullptr;
268
269public:
270 SDUse() = default;
271 SDUse(const SDUse &U) = delete;
272 SDUse &operator=(const SDUse &) = delete;
273
274 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
275 operator const SDValue&() const { return Val; }
276
277 /// If implicit conversion to SDValue doesn't work, the get() method returns
278 /// the SDValue.
279 const SDValue &get() const { return Val; }
280
281 /// This returns the SDNode that contains this Use.
282 SDNode *getUser() { return User; }
283
284 /// Get the next SDUse in the use list.
285 SDUse *getNext() const { return Next; }
286
287 /// Convenience function for get().getNode().
288 SDNode *getNode() const { return Val.getNode(); }
289 /// Convenience function for get().getResNo().
290 unsigned getResNo() const { return Val.getResNo(); }
291 /// Convenience function for get().getValueType().
292 EVT getValueType() const { return Val.getValueType(); }
293
294 /// Convenience function for get().operator==
295 bool operator==(const SDValue &V) const {
296 return Val == V;
297 }
298
299 /// Convenience function for get().operator!=
300 bool operator!=(const SDValue &V) const {
301 return Val != V;
302 }
303
304 /// Convenience function for get().operator<
305 bool operator<(const SDValue &V) const {
306 return Val < V;
307 }
308
309private:
310 friend class SelectionDAG;
311 friend class SDNode;
312 // TODO: unfriend HandleSDNode once we fix its operand handling.
313 friend class HandleSDNode;
314
315 void setUser(SDNode *p) { User = p; }
316
317 /// Remove this use from its existing use list, assign it the
318 /// given value, and add it to the new value's node's use list.
319 inline void set(const SDValue &V);
320 /// Like set, but only supports initializing a newly-allocated
321 /// SDUse with a non-null value.
322 inline void setInitial(const SDValue &V);
323 /// Like set, but only sets the Node portion of the value,
324 /// leaving the ResNo portion unmodified.
325 inline void setNode(SDNode *N);
326
327 void addToList(SDUse **List) {
328 Next = *List;
329 if (Next) Next->Prev = &Next;
330 Prev = List;
331 *List = this;
332 }
333
334 void removeFromList() {
335 *Prev = Next;
336 if (Next) Next->Prev = Prev;
337 }
338};
339
340/// simplify_type specializations - Allow casting operators to work directly on
341/// SDValues as if they were SDNode*'s.
342template<> struct simplify_type<SDUse> {
343 using SimpleType = SDNode *;
344
345 static SimpleType getSimplifiedValue(SDUse &Val) {
346 return Val.getNode();
347 }
348};
349
350/// These are IR-level optimization flags that may be propagated to SDNodes.
351/// TODO: This data structure should be shared by the IR optimizer and the
352/// the backend.
353struct SDNodeFlags {
354private:
355 // This bit is used to determine if the flags are in a defined state.
356 // Flag bits can only be masked out during intersection if the masking flags
357 // are defined.
358 bool AnyDefined : 1;
359
360 bool NoUnsignedWrap : 1;
361 bool NoSignedWrap : 1;
362 bool Exact : 1;
363 bool NoNaNs : 1;
364 bool NoInfs : 1;
365 bool NoSignedZeros : 1;
366 bool AllowReciprocal : 1;
367 bool VectorReduction : 1;
368 bool AllowContract : 1;
369 bool ApproximateFuncs : 1;
370 bool AllowReassociation : 1;
371
372public:
373 /// Default constructor turns off all optimization flags.
374 SDNodeFlags()
375 : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
376 Exact(false), NoNaNs(false), NoInfs(false),
377 NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
378 AllowContract(false), ApproximateFuncs(false),
379 AllowReassociation(false) {}
380
381 /// Propagate the fast-math-flags from an IR FPMathOperator.
382 void copyFMF(const FPMathOperator &FPMO) {
383 setNoNaNs(FPMO.hasNoNaNs());
384 setNoInfs(FPMO.hasNoInfs());
385 setNoSignedZeros(FPMO.hasNoSignedZeros());
386 setAllowReciprocal(FPMO.hasAllowReciprocal());
387 setAllowContract(FPMO.hasAllowContract());
388 setApproximateFuncs(FPMO.hasApproxFunc());
389 setAllowReassociation(FPMO.hasAllowReassoc());
390 }
391
392 /// Sets the state of the flags to the defined state.
393 void setDefined() { AnyDefined = true; }
394 /// Returns true if the flags are in a defined state.
395 bool isDefined() const { return AnyDefined; }
396
397 // These are mutators for each flag.
398 void setNoUnsignedWrap(bool b) {
399 setDefined();
400 NoUnsignedWrap = b;
401 }
402 void setNoSignedWrap(bool b) {
403 setDefined();
404 NoSignedWrap = b;
405 }
406 void setExact(bool b) {
407 setDefined();
408 Exact = b;
409 }
410 void setNoNaNs(bool b) {
411 setDefined();
412 NoNaNs = b;
413 }
414 void setNoInfs(bool b) {
415 setDefined();
416 NoInfs = b;
417 }
418 void setNoSignedZeros(bool b) {
419 setDefined();
420 NoSignedZeros = b;
421 }
422 void setAllowReciprocal(bool b) {
423 setDefined();
424 AllowReciprocal = b;
425 }
426 void setVectorReduction(bool b) {
427 setDefined();
428 VectorReduction = b;
429 }
430 void setAllowContract(bool b) {
431 setDefined();
432 AllowContract = b;
433 }
434 void setApproximateFuncs(bool b) {
435 setDefined();
436 ApproximateFuncs = b;
437 }
438 void setAllowReassociation(bool b) {
439 setDefined();
440 AllowReassociation = b;
441 }
442
443 // These are accessors for each flag.
444 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
445 bool hasNoSignedWrap() const { return NoSignedWrap; }
446 bool hasExact() const { return Exact; }
447 bool hasNoNaNs() const { return NoNaNs; }
448 bool hasNoInfs() const { return NoInfs; }
449 bool hasNoSignedZeros() const { return NoSignedZeros; }
450 bool hasAllowReciprocal() const { return AllowReciprocal; }
451 bool hasVectorReduction() const { return VectorReduction; }
452 bool hasAllowContract() const { return AllowContract; }
453 bool hasApproximateFuncs() const { return ApproximateFuncs; }
454 bool hasAllowReassociation() const { return AllowReassociation; }
455
456 bool isFast() const {
457 return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs &&
458 AllowContract && ApproximateFuncs && AllowReassociation;
459 }
460
461 /// Clear any flags in this flag set that aren't also set in Flags.
462 /// If the given Flags are undefined then don't do anything.
463 void intersectWith(const SDNodeFlags Flags) {
464 if (!Flags.isDefined())
465 return;
466 NoUnsignedWrap &= Flags.NoUnsignedWrap;
467 NoSignedWrap &= Flags.NoSignedWrap;
468 Exact &= Flags.Exact;
469 NoNaNs &= Flags.NoNaNs;
470 NoInfs &= Flags.NoInfs;
471 NoSignedZeros &= Flags.NoSignedZeros;
472 AllowReciprocal &= Flags.AllowReciprocal;
473 VectorReduction &= Flags.VectorReduction;
474 AllowContract &= Flags.AllowContract;
475 ApproximateFuncs &= Flags.ApproximateFuncs;
476 AllowReassociation &= Flags.AllowReassociation;
477 }
478};
479
480/// Represents one node in the SelectionDAG.
481///
482class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
483private:
484 /// The operation that this node performs.
485 int16_t NodeType;
486
487protected:
488 // We define a set of mini-helper classes to help us interpret the bits in our
489 // SubclassData. These are designed to fit within a uint16_t so they pack
490 // with NodeType.
491
492 class SDNodeBitfields {
493 friend class SDNode;
494 friend class MemIntrinsicSDNode;
495 friend class MemSDNode;
496 friend class SelectionDAG;
497
498 uint16_t HasDebugValue : 1;
499 uint16_t IsMemIntrinsic : 1;
500 uint16_t IsDivergent : 1;
501 };
502 enum { NumSDNodeBits = 3 };
503
504 class ConstantSDNodeBitfields {
505 friend class ConstantSDNode;
506
507 uint16_t : NumSDNodeBits;
508
509 uint16_t IsOpaque : 1;
510 };
511
512 class MemSDNodeBitfields {
513 friend class MemSDNode;
514 friend class MemIntrinsicSDNode;
515 friend class AtomicSDNode;
516
517 uint16_t : NumSDNodeBits;
518
519 uint16_t IsVolatile : 1;
520 uint16_t IsNonTemporal : 1;
521 uint16_t IsDereferenceable : 1;
522 uint16_t IsInvariant : 1;
523 };
524 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
525
526 class LSBaseSDNodeBitfields {
527 friend class LSBaseSDNode;
528
529 uint16_t : NumMemSDNodeBits;
530
531 uint16_t AddressingMode : 3; // enum ISD::MemIndexedMode
532 };
533 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
534
535 class LoadSDNodeBitfields {
536 friend class LoadSDNode;
537 friend class MaskedLoadSDNode;
538
539 uint16_t : NumLSBaseSDNodeBits;
540
541 uint16_t ExtTy : 2; // enum ISD::LoadExtType
542 uint16_t IsExpanding : 1;
543 };
544
545 class StoreSDNodeBitfields {
546 friend class StoreSDNode;
547 friend class MaskedStoreSDNode;
548
549 uint16_t : NumLSBaseSDNodeBits;
550
551 uint16_t IsTruncating : 1;
552 uint16_t IsCompressing : 1;
553 };
554
555 union {
556 char RawSDNodeBits[sizeof(uint16_t)];
557 SDNodeBitfields SDNodeBits;
558 ConstantSDNodeBitfields ConstantSDNodeBits;
559 MemSDNodeBitfields MemSDNodeBits;
560 LSBaseSDNodeBitfields LSBaseSDNodeBits;
561 LoadSDNodeBitfields LoadSDNodeBits;
562 StoreSDNodeBitfields StoreSDNodeBits;
563 };
564
565 // RawSDNodeBits must cover the entirety of the union. This means that all of
566 // the union's members must have size <= RawSDNodeBits. We write the RHS as
567 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
568 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
569 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
570 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
571 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
572 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
573 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
574
575private:
576 friend class SelectionDAG;
577 // TODO: unfriend HandleSDNode once we fix its operand handling.
578 friend class HandleSDNode;
579
580 /// Unique id per SDNode in the DAG.
581 int NodeId = -1;
582
583 /// The values that are used by this operation.
584 SDUse *OperandList = nullptr;
585
586 /// The types of the values this node defines. SDNode's may
587 /// define multiple values simultaneously.
588 const EVT *ValueList;
589
590 /// List of uses for this SDNode.
591 SDUse *UseList = nullptr;
592
593 /// The number of entries in the Operand/Value list.
594 unsigned short NumOperands = 0;
595 unsigned short NumValues;
596
597 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
598 // original LLVM instructions.
599 // This is used for turning off scheduling, because we'll forgo
600 // the normal scheduling algorithms and output the instructions according to
601 // this ordering.
602 unsigned IROrder;
603
604 /// Source line information.
605 DebugLoc debugLoc;
606
607 /// Return a pointer to the specified value type.
608 static const EVT *getValueTypeList(EVT VT);
609
610 SDNodeFlags Flags;
611
612public:
613 /// Unique and persistent id per SDNode in the DAG.
614 /// Used for debug printing.
615 uint16_t PersistentId;
616
617 //===--------------------------------------------------------------------===//
618 // Accessors
619 //
620
621 /// Return the SelectionDAG opcode value for this node. For
622 /// pre-isel nodes (those for which isMachineOpcode returns false), these
623 /// are the opcode values in the ISD and <target>ISD namespaces. For
624 /// post-isel opcodes, see getMachineOpcode.
625 unsigned getOpcode() const { return (unsigned short)NodeType; }
626
627 /// Test if this node has a target-specific opcode (in the
628 /// \<target\>ISD namespace).
629 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
630
631 /// Test if this node has a target-specific
632 /// memory-referencing opcode (in the \<target\>ISD namespace and
633 /// greater than FIRST_TARGET_MEMORY_OPCODE).
634 bool isTargetMemoryOpcode() const {
635 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
636 }
637
638 /// Return true if the type of the node type undefined.
639 bool isUndef() const { return NodeType == ISD::UNDEF; }
640
641 /// Test if this node is a memory intrinsic (with valid pointer information).
642 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
643 /// non-memory intrinsics (with chains) that are not really instances of
644 /// MemSDNode. For such nodes, we need some extra state to determine the
645 /// proper classof relationship.
646 bool isMemIntrinsic() const {
647 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
648 NodeType == ISD::INTRINSIC_VOID) &&
649 SDNodeBits.IsMemIntrinsic;
650 }
651
652 /// Test if this node is a strict floating point pseudo-op.
653 bool isStrictFPOpcode() {
654 switch (NodeType) {
655 default:
656 return false;
657 case ISD::STRICT_FADD:
658 case ISD::STRICT_FSUB:
659 case ISD::STRICT_FMUL:
660 case ISD::STRICT_FDIV:
661 case ISD::STRICT_FREM:
662 case ISD::STRICT_FMA:
663 case ISD::STRICT_FSQRT:
664 case ISD::STRICT_FPOW:
665 case ISD::STRICT_FPOWI:
666 case ISD::STRICT_FSIN:
667 case ISD::STRICT_FCOS:
668 case ISD::STRICT_FEXP:
669 case ISD::STRICT_FEXP2:
670 case ISD::STRICT_FLOG:
671 case ISD::STRICT_FLOG10:
672 case ISD::STRICT_FLOG2:
673 case ISD::STRICT_FRINT:
674 case ISD::STRICT_FNEARBYINT:
675 return true;
676 }
677 }
678
679 /// Test if this node has a post-isel opcode, directly
680 /// corresponding to a MachineInstr opcode.
681 bool isMachineOpcode() const { return NodeType < 0; }
682
683 /// This may only be called if isMachineOpcode returns
684 /// true. It returns the MachineInstr opcode value that the node's opcode
685 /// corresponds to.
686 unsigned getMachineOpcode() const {
687 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ?
static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 687, __PRETTY_FUNCTION__))
;
688 return ~NodeType;
689 }
690
691 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
692 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
693
694 bool isDivergent() const { return SDNodeBits.IsDivergent; }
695
696 /// Return true if there are no uses of this node.
697 bool use_empty() const { return UseList == nullptr; }
698
699 /// Return true if there is exactly one use of this node.
700 bool hasOneUse() const {
701 return !use_empty() && std::next(use_begin()) == use_end();
702 }
703
704 /// Return the number of uses of this node. This method takes
705 /// time proportional to the number of uses.
706 size_t use_size() const { return std::distance(use_begin(), use_end()); }
707
708 /// Return the unique node id.
709 int getNodeId() const { return NodeId; }
710
711 /// Set unique node id.
712 void setNodeId(int Id) { NodeId = Id; }
713
714 /// Return the node ordering.
715 unsigned getIROrder() const { return IROrder; }
716
717 /// Set the node ordering.
718 void setIROrder(unsigned Order) { IROrder = Order; }
719
720 /// Return the source location info.
721 const DebugLoc &getDebugLoc() const { return debugLoc; }
722
723 /// Set source location info. Try to avoid this, putting
724 /// it in the constructor is preferable.
725 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
726
727 /// This class provides iterator support for SDUse
728 /// operands that use a specific SDNode.
729 class use_iterator
730 : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
731 friend class SDNode;
732
733 SDUse *Op = nullptr;
734
735 explicit use_iterator(SDUse *op) : Op(op) {}
736
737 public:
738 using reference = std::iterator<std::forward_iterator_tag,
739 SDUse, ptrdiff_t>::reference;
740 using pointer = std::iterator<std::forward_iterator_tag,
741 SDUse, ptrdiff_t>::pointer;
742
743 use_iterator() = default;
744 use_iterator(const use_iterator &I) : Op(I.Op) {}
745
746 bool operator==(const use_iterator &x) const {
747 return Op == x.Op;
748 }
749 bool operator!=(const use_iterator &x) const {
750 return !operator==(x);
751 }
752
753 /// Return true if this iterator is at the end of uses list.
754 bool atEnd() const { return Op == nullptr; }
755
756 // Iterator traversal: forward iteration only.
757 use_iterator &operator++() { // Preincrement
758 assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 758, __PRETTY_FUNCTION__))
;
759 Op = Op->getNext();
760 return *this;
761 }
762
763 use_iterator operator++(int) { // Postincrement
764 use_iterator tmp = *this; ++*this; return tmp;
765 }
766
767 /// Retrieve a pointer to the current user node.
768 SDNode *operator*() const {
769 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 769, __PRETTY_FUNCTION__))
;
770 return Op->getUser();
771 }
772
773 SDNode *operator->() const { return operator*(); }
774
775 SDUse &getUse() const { return *Op; }
776
777 /// Retrieve the operand # of this use in its user.
778 unsigned getOperandNo() const {
779 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 779, __PRETTY_FUNCTION__))
;
780 return (unsigned)(Op - Op->getUser()->OperandList);
781 }
782 };
783
784 /// Provide iteration support to walk over all uses of an SDNode.
785 use_iterator use_begin() const {
786 return use_iterator(UseList);
787 }
788
789 static use_iterator use_end() { return use_iterator(nullptr); }
790
791 inline iterator_range<use_iterator> uses() {
792 return make_range(use_begin(), use_end());
793 }
794 inline iterator_range<use_iterator> uses() const {
795 return make_range(use_begin(), use_end());
796 }
797
798 /// Return true if there are exactly NUSES uses of the indicated value.
799 /// This method ignores uses of other values defined by this operation.
800 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
801
802 /// Return true if there are any use of the indicated value.
803 /// This method ignores uses of other values defined by this operation.
804 bool hasAnyUseOfValue(unsigned Value) const;
805
806 /// Return true if this node is the only use of N.
807 bool isOnlyUserOf(const SDNode *N) const;
808
809 /// Return true if this node is an operand of N.
810 bool isOperandOf(const SDNode *N) const;
811
812 /// Return true if this node is a predecessor of N.
813 /// NOTE: Implemented on top of hasPredecessor and every bit as
814 /// expensive. Use carefully.
815 bool isPredecessorOf(const SDNode *N) const {
816 return N->hasPredecessor(this);
817 }
818
819 /// Return true if N is a predecessor of this node.
820 /// N is either an operand of this node, or can be reached by recursively
821 /// traversing up the operands.
822 /// NOTE: This is an expensive method. Use it carefully.
823 bool hasPredecessor(const SDNode *N) const;
824
825 /// Returns true if N is a predecessor of any node in Worklist. This
826 /// helper keeps Visited and Worklist sets externally to allow unions
827 /// searches to be performed in parallel, caching of results across
828 /// queries and incremental addition to Worklist. Stops early if N is
829 /// found but will resume. Remember to clear Visited and Worklists
830 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
831 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
832 /// topologically ordered (Operands have strictly smaller node id) and search
833 /// can be pruned leveraging this.
834 static bool hasPredecessorHelper(const SDNode *N,
835 SmallPtrSetImpl<const SDNode *> &Visited,
836 SmallVectorImpl<const SDNode *> &Worklist,
837 unsigned int MaxSteps = 0,
838 bool TopologicalPrune = false) {
839 SmallVector<const SDNode *, 8> DeferredNodes;
840 if (Visited.count(N))
841 return true;
842
843 // Node Id's are assigned in three places: As a topological
844 // ordering (> 0), during legalization (results in values set to
845 // 0), new nodes (set to -1). If N has a topolgical id then we
846 // know that all nodes with ids smaller than it cannot be
847 // successors and we need not check them. Filter out all node
848 // that can't be matches. We add them to the worklist before exit
849 // in case of multiple calls. Note that during selection the topological id
850 // may be violated if a node's predecessor is selected before it. We mark
851 // this at selection negating the id of unselected successors and
852 // restricting topological pruning to positive ids.
853
854 int NId = N->getNodeId();
855 // If we Invalidated the Id, reconstruct original NId.
856 if (NId < -1)
857 NId = -(NId + 1);
858
859 bool Found = false;
860 while (!Worklist.empty()) {
861 const SDNode *M = Worklist.pop_back_val();
862 int MId = M->getNodeId();
863 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
864 (MId > 0) && (MId < NId)) {
865 DeferredNodes.push_back(M);
866 continue;
867 }
868 for (const SDValue &OpV : M->op_values()) {
869 SDNode *Op = OpV.getNode();
870 if (Visited.insert(Op).second)
871 Worklist.push_back(Op);
872 if (Op == N)
873 Found = true;
874 }
875 if (Found)
876 break;
877 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
878 break;
879 }
880 // Push deferred nodes back on worklist.
881 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
882 // If we bailed early, conservatively return found.
883 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
884 return true;
885 return Found;
886 }
887
888 /// Return true if all the users of N are contained in Nodes.
889 /// NOTE: Requires at least one match, but doesn't require them all.
890 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
891
892 /// Return the number of values used by this operation.
893 unsigned getNumOperands() const { return NumOperands; }
894
895 /// Helper method returns the integer value of a ConstantSDNode operand.
896 inline uint64_t getConstantOperandVal(unsigned Num) const;
897
898 const SDValue &getOperand(unsigned Num) const {
899 assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!"
) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 899, __PRETTY_FUNCTION__))
;
900 return OperandList[Num];
901 }
902
903 using op_iterator = SDUse *;
904
905 op_iterator op_begin() const { return OperandList; }
906 op_iterator op_end() const { return OperandList+NumOperands; }
907 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
908
909 /// Iterator for directly iterating over the operand SDValue's.
910 struct value_op_iterator
911 : iterator_adaptor_base<value_op_iterator, op_iterator,
912 std::random_access_iterator_tag, SDValue,
913 ptrdiff_t, value_op_iterator *,
914 value_op_iterator *> {
915 explicit value_op_iterator(SDUse *U = nullptr)
916 : iterator_adaptor_base(U) {}
917
918 const SDValue &operator*() const { return I->get(); }
919 };
920
921 iterator_range<value_op_iterator> op_values() const {
922 return make_range(value_op_iterator(op_begin()),
923 value_op_iterator(op_end()));
924 }
925
926 SDVTList getVTList() const {
927 SDVTList X = { ValueList, NumValues };
928 return X;
929 }
930
931 /// If this node has a glue operand, return the node
932 /// to which the glue operand points. Otherwise return NULL.
933 SDNode *getGluedNode() const {
934 if (getNumOperands() != 0 &&
935 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
936 return getOperand(getNumOperands()-1).getNode();
937 return nullptr;
938 }
939
940 /// If this node has a glue value with a user, return
941 /// the user (there is at most one). Otherwise return NULL.
942 SDNode *getGluedUser() const {
943 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
944 if (UI.getUse().get().getValueType() == MVT::Glue)
945 return *UI;
946 return nullptr;
947 }
948
949 const SDNodeFlags getFlags() const { return Flags; }
950 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
951 bool isFast() { return Flags.isFast(); }
952
953 /// Clear any flags in this node that aren't also set in Flags.
954 /// If Flags is not in a defined state then this has no effect.
955 void intersectFlagsWith(const SDNodeFlags Flags);
956
957 /// Return the number of values defined/returned by this operator.
958 unsigned getNumValues() const { return NumValues; }
959
960 /// Return the type of a specified result.
961 EVT getValueType(unsigned ResNo) const {
962 assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ?
static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 962, __PRETTY_FUNCTION__))
;
963 return ValueList[ResNo];
964 }
965
966 /// Return the type of a specified result as a simple type.
967 MVT getSimpleValueType(unsigned ResNo) const {
968 return getValueType(ResNo).getSimpleVT();
969 }
970
971 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
972 unsigned getValueSizeInBits(unsigned ResNo) const {
973 return getValueType(ResNo).getSizeInBits();
974 }
975
976 using value_iterator = const EVT *;
977
978 value_iterator value_begin() const { return ValueList; }
979 value_iterator value_end() const { return ValueList+NumValues; }
980
981 /// Return the opcode of this operation for printing.
982 std::string getOperationName(const SelectionDAG *G = nullptr) const;
983 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
984 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
985 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
986 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
987 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
988
989 /// Print a SelectionDAG node and all children down to
990 /// the leaves. The given SelectionDAG allows target-specific nodes
991 /// to be printed in human-readable form. Unlike printr, this will
992 /// print the whole DAG, including children that appear multiple
993 /// times.
994 ///
995 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
996
997 /// Print a SelectionDAG node and children up to
998 /// depth "depth." The given SelectionDAG allows target-specific
999 /// nodes to be printed in human-readable form. Unlike printr, this
1000 /// will print children that appear multiple times wherever they are
1001 /// used.
1002 ///
1003 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1004 unsigned depth = 100) const;
1005
1006 /// Dump this node, for debugging.
1007 void dump() const;
1008
1009 /// Dump (recursively) this node and its use-def subgraph.
1010 void dumpr() const;
1011
1012 /// Dump this node, for debugging.
1013 /// The given SelectionDAG allows target-specific nodes to be printed
1014 /// in human-readable form.
1015 void dump(const SelectionDAG *G) const;
1016
1017 /// Dump (recursively) this node and its use-def subgraph.
1018 /// The given SelectionDAG allows target-specific nodes to be printed
1019 /// in human-readable form.
1020 void dumpr(const SelectionDAG *G) const;
1021
1022 /// printrFull to dbgs(). The given SelectionDAG allows
1023 /// target-specific nodes to be printed in human-readable form.
1024 /// Unlike dumpr, this will print the whole DAG, including children
1025 /// that appear multiple times.
1026 void dumprFull(const SelectionDAG *G = nullptr) const;
1027
1028 /// printrWithDepth to dbgs(). The given
1029 /// SelectionDAG allows target-specific nodes to be printed in
1030 /// human-readable form. Unlike dumpr, this will print children
1031 /// that appear multiple times wherever they are used.
1032 ///
1033 void dumprWithDepth(const SelectionDAG *G = nullptr,
1034 unsigned depth = 100) const;
1035
1036 /// Gather unique data for the node.
1037 void Profile(FoldingSetNodeID &ID) const;
1038
1039 /// This method should only be used by the SDUse class.
1040 void addUse(SDUse &U) { U.addToList(&UseList); }
1041
1042protected:
1043 static SDVTList getSDVTList(EVT VT) {
1044 SDVTList Ret = { getValueTypeList(VT), 1 };
1045 return Ret;
1046 }
1047
1048 /// Create an SDNode.
1049 ///
1050 /// SDNodes are created without any operands, and never own the operand
1051 /// storage. To add operands, see SelectionDAG::createOperands.
1052 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1053 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1054 IROrder(Order), debugLoc(std::move(dl)) {
1055 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1056 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1056, __PRETTY_FUNCTION__))
;
1057 assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1058, __PRETTY_FUNCTION__))
1058 "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1058, __PRETTY_FUNCTION__))
;
1059 }
1060
1061 /// Release the operands and set this node to have zero operands.
1062 void DropOperands();
1063};
1064
1065/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1066/// into SDNode creation functions.
1067/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1068/// from the original Instruction, and IROrder is the ordinal position of
1069/// the instruction.
1070/// When an SDNode is created after the DAG is being built, both DebugLoc and
1071/// the IROrder are propagated from the original SDNode.
1072/// So SDLoc class provides two constructors besides the default one, one to
1073/// be used by the DAGBuilder, the other to be used by others.
1074class SDLoc {
1075private:
1076 DebugLoc DL;
1077 int IROrder = 0;
1078
1079public:
1080 SDLoc() = default;
1081 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1082 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1083 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1084 assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void
> (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1084, __PRETTY_FUNCTION__))
;
1085 if (I)
1086 DL = I->getDebugLoc();
1087 }
1088
1089 unsigned getIROrder() const { return IROrder; }
1090 const DebugLoc &getDebugLoc() const { return DL; }
1091};
1092
1093// Define inline functions from the SDValue class.
1094
1095inline SDValue::SDValue(SDNode *node, unsigned resno)
1096 : Node(node), ResNo(resno) {
1097 // Explicitly check for !ResNo to avoid use-after-free, because there are
1098 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1099 // combines.
1100 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1101, __PRETTY_FUNCTION__))
1101 "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1101, __PRETTY_FUNCTION__))
;
1102 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1102, __PRETTY_FUNCTION__))
;
1103}
1104
1105inline unsigned SDValue::getOpcode() const {
1106 return Node->getOpcode();
1107}
1108
1109inline EVT SDValue::getValueType() const {
1110 return Node->getValueType(ResNo);
9
Called C++ object pointer is null
1111}
1112
1113inline unsigned SDValue::getNumOperands() const {
1114 return Node->getNumOperands();
1115}
1116
1117inline const SDValue &SDValue::getOperand(unsigned i) const {
1118 return Node->getOperand(i);
1119}
1120
1121inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1122 return Node->getConstantOperandVal(i);
1123}
1124
1125inline bool SDValue::isTargetOpcode() const {
1126 return Node->isTargetOpcode();
1127}
1128
1129inline bool SDValue::isTargetMemoryOpcode() const {
1130 return Node->isTargetMemoryOpcode();
1131}
1132
1133inline bool SDValue::isMachineOpcode() const {
1134 return Node->isMachineOpcode();
1135}
1136
1137inline unsigned SDValue::getMachineOpcode() const {
1138 return Node->getMachineOpcode();
1139}
1140
1141inline bool SDValue::isUndef() const {
1142 return Node->isUndef();
1143}
1144
1145inline bool SDValue::use_empty() const {
1146 return !Node->hasAnyUseOfValue(ResNo);
1147}
1148
1149inline bool SDValue::hasOneUse() const {
1150 return Node->hasNUsesOfValue(1, ResNo);
1151}
1152
1153inline const DebugLoc &SDValue::getDebugLoc() const {
1154 return Node->getDebugLoc();
1155}
1156
1157inline void SDValue::dump() const {
1158 return Node->dump();
1159}
1160
1161inline void SDValue::dump(const SelectionDAG *G) const {
1162 return Node->dump(G);
1163}
1164
1165inline void SDValue::dumpr() const {
1166 return Node->dumpr();
1167}
1168
1169inline void SDValue::dumpr(const SelectionDAG *G) const {
1170 return Node->dumpr(G);
1171}
1172
1173// Define inline functions from the SDUse class.
1174
1175inline void SDUse::set(const SDValue &V) {
1176 if (Val.getNode()) removeFromList();
1177 Val = V;
1178 if (V.getNode()) V.getNode()->addUse(*this);
1179}
1180
1181inline void SDUse::setInitial(const SDValue &V) {
1182 Val = V;
1183 V.getNode()->addUse(*this);
1184}
1185
1186inline void SDUse::setNode(SDNode *N) {
1187 if (Val.getNode()) removeFromList();
1188 Val.setNode(N);
1189 if (N) N->addUse(*this);
1190}
1191
1192/// This class is used to form a handle around another node that
1193/// is persistent and is updated across invocations of replaceAllUsesWith on its
1194/// operand. This node should be directly created by end-users and not added to
1195/// the AllNodes list.
1196class HandleSDNode : public SDNode {
1197 SDUse Op;
1198
1199public:
1200 explicit HandleSDNode(SDValue X)
1201 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1202 // HandleSDNodes are never inserted into the DAG, so they won't be
1203 // auto-numbered. Use ID 65535 as a sentinel.
1204 PersistentId = 0xffff;
1205
1206 // Manually set up the operand list. This node type is special in that it's
1207 // always stack allocated and SelectionDAG does not manage its operands.
1208 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1209 // be so special.
1210 Op.setUser(this);
1211 Op.setInitial(X);
1212 NumOperands = 1;
1213 OperandList = &Op;
1214 }
1215 ~HandleSDNode();
1216
1217 const SDValue &getValue() const { return Op; }
1218};
1219
1220class AddrSpaceCastSDNode : public SDNode {
1221private:
1222 unsigned SrcAddrSpace;
1223 unsigned DestAddrSpace;
1224
1225public:
1226 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1227 unsigned SrcAS, unsigned DestAS);
1228
1229 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1230 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1231
1232 static bool classof(const SDNode *N) {
1233 return N->getOpcode() == ISD::ADDRSPACECAST;
1234 }
1235};
1236
1237/// This is an abstract virtual class for memory operations.
1238class MemSDNode : public SDNode {
1239private:
1240 // VT of in-memory value.
1241 EVT MemoryVT;
1242
1243protected:
1244 /// Memory reference information.
1245 MachineMemOperand *MMO;
1246
1247public:
1248 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1249 EVT memvt, MachineMemOperand *MMO);
1250
1251 bool readMem() const { return MMO->isLoad(); }
1252 bool writeMem() const { return MMO->isStore(); }
1253
1254 /// Returns alignment and volatility of the memory access
1255 unsigned getOriginalAlignment() const {
1256 return MMO->getBaseAlignment();
1257 }
1258 unsigned getAlignment() const {
1259 return MMO->getAlignment();
1260 }
1261
1262 /// Return the SubclassData value, without HasDebugValue. This contains an
1263 /// encoding of the volatile flag, as well as bits used by subclasses. This
1264 /// function should only be used to compute a FoldingSetNodeID value.
1265 /// The HasDebugValue bit is masked out because CSE map needs to match
1266 /// nodes with debug info with nodes without debug info. Same is about
1267 /// isDivergent bit.
1268 unsigned getRawSubclassData() const {
1269 uint16_t Data;
1270 union {
1271 char RawSDNodeBits[sizeof(uint16_t)];
1272 SDNodeBitfields SDNodeBits;
1273 };
1274 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1275 SDNodeBits.HasDebugValue = 0;
1276 SDNodeBits.IsDivergent = false;
1277 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1278 return Data;
1279 }
1280
1281 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1282 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1283 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1284 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1285
1286 // Returns the offset from the location of the access.
1287 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1288
1289 /// Returns the AA info that describes the dereference.
1290 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1291
1292 /// Returns the Ranges that describes the dereference.
1293 const MDNode *getRanges() const { return MMO->getRanges(); }
1294
1295 /// Returns the synchronization scope ID for this memory operation.
1296 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1297
1298 /// Return the atomic ordering requirements for this memory operation. For
1299 /// cmpxchg atomic operations, return the atomic ordering requirements when
1300 /// store occurs.
1301 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
1302
1303 /// Return the type of the in-memory value.
1304 EVT getMemoryVT() const { return MemoryVT; }
1305
1306 /// Return a MachineMemOperand object describing the memory
1307 /// reference performed by operation.
1308 MachineMemOperand *getMemOperand() const { return MMO; }
1309
1310 const MachinePointerInfo &getPointerInfo() const {
1311 return MMO->getPointerInfo();
1312 }
1313
1314 /// Return the address space for the associated pointer
1315 unsigned getAddressSpace() const {
1316 return getPointerInfo().getAddrSpace();
1317 }
1318
1319 /// Update this MemSDNode's MachineMemOperand information
1320 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1321 /// This must only be used when the new alignment applies to all users of
1322 /// this MachineMemOperand.
1323 void refineAlignment(const MachineMemOperand *NewMMO) {
1324 MMO->refineAlignment(NewMMO);
1325 }
1326
1327 const SDValue &getChain() const { return getOperand(0); }
1328 const SDValue &getBasePtr() const {
1329 return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
1330 }
1331
1332 // Methods to support isa and dyn_cast
1333 static bool classof(const SDNode *N) {
1334 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1335 // with either an intrinsic or a target opcode.
1336 return N->getOpcode() == ISD::LOAD ||
1337 N->getOpcode() == ISD::STORE ||
1338 N->getOpcode() == ISD::PREFETCH ||
1339 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1340 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1341 N->getOpcode() == ISD::ATOMIC_SWAP ||
1342 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1343 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1344 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1345 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1346 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1347 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1348 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1349 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1350 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1351 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1352 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1353 N->getOpcode() == ISD::ATOMIC_LOAD ||
1354 N->getOpcode() == ISD::ATOMIC_STORE ||
1355 N->getOpcode() == ISD::MLOAD ||
1356 N->getOpcode() == ISD::MSTORE ||
1357 N->getOpcode() == ISD::MGATHER ||
1358 N->getOpcode() == ISD::MSCATTER ||
1359 N->isMemIntrinsic() ||
1360 N->isTargetMemoryOpcode();
1361 }
1362};
1363
1364/// This is an SDNode representing atomic operations.
1365class AtomicSDNode : public MemSDNode {
1366public:
1367 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1368 EVT MemVT, MachineMemOperand *MMO)
1369 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {}
1370
1371 const SDValue &getBasePtr() const { return getOperand(1); }
1372 const SDValue &getVal() const { return getOperand(2); }
1373
1374 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1375 /// otherwise.
1376 bool isCompareAndSwap() const {
1377 unsigned Op = getOpcode();
1378 return Op == ISD::ATOMIC_CMP_SWAP ||
1379 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1380 }
1381
1382 /// For cmpxchg atomic operations, return the atomic ordering requirements
1383 /// when store does not occur.
1384 AtomicOrdering getFailureOrdering() const {
1385 assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ?
static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1385, __PRETTY_FUNCTION__))
;
1386 return MMO->getFailureOrdering();
1387 }
1388
1389 // Methods to support isa and dyn_cast
1390 static bool classof(const SDNode *N) {
1391 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1392 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1393 N->getOpcode() == ISD::ATOMIC_SWAP ||
1394 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1395 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1396 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1397 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1398 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1399 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1400 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1401 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1402 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1403 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1404 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1405 N->getOpcode() == ISD::ATOMIC_LOAD ||
1406 N->getOpcode() == ISD::ATOMIC_STORE;
1407 }
1408};
1409
1410/// This SDNode is used for target intrinsics that touch
1411/// memory and need an associated MachineMemOperand. Its opcode may be
1412/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1413/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1414class MemIntrinsicSDNode : public MemSDNode {
1415public:
1416 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1417 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1418 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1419 SDNodeBits.IsMemIntrinsic = true;
1420 }
1421
1422 // Methods to support isa and dyn_cast
1423 static bool classof(const SDNode *N) {
1424 // We lower some target intrinsics to their target opcode
1425 // early a node with a target opcode can be of this class
1426 return N->isMemIntrinsic() ||
1427 N->getOpcode() == ISD::PREFETCH ||
1428 N->isTargetMemoryOpcode();
1429 }
1430};
1431
1432/// This SDNode is used to implement the code generator
1433/// support for the llvm IR shufflevector instruction. It combines elements
1434/// from two input vectors into a new input vector, with the selection and
1435/// ordering of elements determined by an array of integers, referred to as
1436/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1437/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1438/// An index of -1 is treated as undef, such that the code generator may put
1439/// any value in the corresponding element of the result.
1440class ShuffleVectorSDNode : public SDNode {
1441 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1442 // is freed when the SelectionDAG object is destroyed.
1443 const int *Mask;
1444
1445protected:
1446 friend class SelectionDAG;
1447
1448 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1449 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1450
1451public:
1452 ArrayRef<int> getMask() const {
1453 EVT VT = getValueType(0);
1454 return makeArrayRef(Mask, VT.getVectorNumElements());
1455 }
1456
1457 int getMaskElt(unsigned Idx) const {
1458 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"
) ? static_cast<void> (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1458, __PRETTY_FUNCTION__))
;
1459 return Mask[Idx];
1460 }
1461
1462 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1463
1464 int getSplatIndex() const {
1465 assert(isSplat() && "Cannot get splat index for non-splat!")((isSplat() && "Cannot get splat index for non-splat!"
) ? static_cast<void> (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1465, __PRETTY_FUNCTION__))
;
1466 EVT VT = getValueType(0);
1467 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
1468 if (Mask[i] >= 0)
1469 return Mask[i];
1470 }
1471 llvm_unreachable("Splat with all undef indices?")::llvm::llvm_unreachable_internal("Splat with all undef indices?"
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1471)
;
1472 }
1473
1474 static bool isSplatMask(const int *Mask, EVT VT);
1475
1476 /// Change values in a shuffle permute mask assuming
1477 /// the two vector operands have swapped position.
1478 static void commuteMask(MutableArrayRef<int> Mask) {
1479 unsigned NumElems = Mask.size();
1480 for (unsigned i = 0; i != NumElems; ++i) {
1481 int idx = Mask[i];
1482 if (idx < 0)
1483 continue;
1484 else if (idx < (int)NumElems)
1485 Mask[i] = idx + NumElems;
1486 else
1487 Mask[i] = idx - NumElems;
1488 }
1489 }
1490
1491 static bool classof(const SDNode *N) {
1492 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1493 }
1494};
1495
1496class ConstantSDNode : public SDNode {
1497 friend class SelectionDAG;
1498
1499 const ConstantInt *Value;
1500
1501 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1502 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1503 getSDVTList(VT)),
1504 Value(val) {
1505 ConstantSDNodeBits.IsOpaque = isOpaque;
1506 }
1507
1508public:
1509 const ConstantInt *getConstantIntValue() const { return Value; }
1510 const APInt &getAPIntValue() const { return Value->getValue(); }
1511 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1512 int64_t getSExtValue() const { return Value->getSExtValue(); }
1513 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1514 return Value->getLimitedValue(Limit);
1515 }
1516
1517 bool isOne() const { return Value->isOne(); }
1518 bool isNullValue() const { return Value->isZero(); }
1519 bool isAllOnesValue() const { return Value->isMinusOne(); }
1520
1521 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1522
1523 static bool classof(const SDNode *N) {
1524 return N->getOpcode() == ISD::Constant ||
1525 N->getOpcode() == ISD::TargetConstant;
1526 }
1527};
1528
1529uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1530 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1531}
1532
1533class ConstantFPSDNode : public SDNode {
1534 friend class SelectionDAG;
1535
1536 const ConstantFP *Value;
1537
1538 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1539 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1540 DebugLoc(), getSDVTList(VT)),
1541 Value(val) {}
1542
1543public:
1544 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1545 const ConstantFP *getConstantFPValue() const { return Value; }
1546
1547 /// Return true if the value is positive or negative zero.
1548 bool isZero() const { return Value->isZero(); }
1549
1550 /// Return true if the value is a NaN.
1551 bool isNaN() const { return Value->isNaN(); }
1552
1553 /// Return true if the value is an infinity
1554 bool isInfinity() const { return Value->isInfinity(); }
1555
1556 /// Return true if the value is negative.
1557 bool isNegative() const { return Value->isNegative(); }
1558
1559 /// We don't rely on operator== working on double values, as
1560 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1561 /// As such, this method can be used to do an exact bit-for-bit comparison of
1562 /// two floating point values.
1563
1564 /// We leave the version with the double argument here because it's just so
1565 /// convenient to write "2.0" and the like. Without this function we'd
1566 /// have to duplicate its logic everywhere it's called.
1567 bool isExactlyValue(double V) const {
1568 return Value->getValueAPF().isExactlyValue(V);
1569 }
1570 bool isExactlyValue(const APFloat& V) const;
1571
1572 static bool isValueValidForType(EVT VT, const APFloat& Val);
1573
1574 static bool classof(const SDNode *N) {
1575 return N->getOpcode() == ISD::ConstantFP ||
1576 N->getOpcode() == ISD::TargetConstantFP;
1577 }
1578};
1579
1580/// Returns true if \p V is a constant integer zero.
1581bool isNullConstant(SDValue V);
1582
1583/// Returns true if \p V is an FP constant with a value of positive zero.
1584bool isNullFPConstant(SDValue V);
1585
1586/// Returns true if \p V is an integer constant with all bits set.
1587bool isAllOnesConstant(SDValue V);
1588
1589/// Returns true if \p V is a constant integer one.
1590bool isOneConstant(SDValue V);
1591
1592/// Return the non-bitcasted source operand of \p V if it exists.
1593/// If \p V is not a bitcasted value, it is returned as-is.
1594SDValue peekThroughBitcasts(SDValue V);
1595
1596/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1597/// If \p V is not a bitcasted one-use value, it is returned as-is.
1598SDValue peekThroughOneUseBitcasts(SDValue V);
1599
1600/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1601/// constant is canonicalized to be operand 1.
1602bool isBitwiseNot(SDValue V);
1603
1604/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1605ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false);
1606
1607/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1608ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1609
1610class GlobalAddressSDNode : public SDNode {
1611 friend class SelectionDAG;
1612
1613 const GlobalValue *TheGlobal;
1614 int64_t Offset;
1615 unsigned char TargetFlags;
1616
1617 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1618 const GlobalValue *GA, EVT VT, int64_t o,
1619 unsigned char TF);
1620
1621public:
1622 const GlobalValue *getGlobal() const { return TheGlobal; }
1623 int64_t getOffset() const { return Offset; }
1624 unsigned char getTargetFlags() const { return TargetFlags; }
1625 // Return the address space this GlobalAddress belongs to.
1626 unsigned getAddressSpace() const;
1627
1628 static bool classof(const SDNode *N) {
1629 return N->getOpcode() == ISD::GlobalAddress ||
1630 N->getOpcode() == ISD::TargetGlobalAddress ||
1631 N->getOpcode() == ISD::GlobalTLSAddress ||
1632 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1633 }
1634};
1635
1636class FrameIndexSDNode : public SDNode {
1637 friend class SelectionDAG;
1638
1639 int FI;
1640
1641 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1642 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1643 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1644 }
1645
1646public:
1647 int getIndex() const { return FI; }
1648
1649 static bool classof(const SDNode *N) {
1650 return N->getOpcode() == ISD::FrameIndex ||
1651 N->getOpcode() == ISD::TargetFrameIndex;
1652 }
1653};
1654
1655class JumpTableSDNode : public SDNode {
1656 friend class SelectionDAG;
1657
1658 int JTI;
1659 unsigned char TargetFlags;
1660
1661 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned char TF)
1662 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1663 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1664 }
1665
1666public:
1667 int getIndex() const { return JTI; }
1668 unsigned char getTargetFlags() const { return TargetFlags; }
1669
1670 static bool classof(const SDNode *N) {
1671 return N->getOpcode() == ISD::JumpTable ||
1672 N->getOpcode() == ISD::TargetJumpTable;
1673 }
1674};
1675
1676class ConstantPoolSDNode : public SDNode {
1677 friend class SelectionDAG;
1678
1679 union {
1680 const Constant *ConstVal;
1681 MachineConstantPoolValue *MachineCPVal;
1682 } Val;
1683 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1684 unsigned Alignment; // Minimum alignment requirement of CP (not log2 value).
1685 unsigned char TargetFlags;
1686
1687 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1688 unsigned Align, unsigned char TF)
1689 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1690 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1691 TargetFlags(TF) {
1692 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1692, __PRETTY_FUNCTION__))
;
1693 Val.ConstVal = c;
1694 }
1695
1696 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v,
1697 EVT VT, int o, unsigned Align, unsigned char TF)
1698 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1699 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1700 TargetFlags(TF) {
1701 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1701, __PRETTY_FUNCTION__))
;
1702 Val.MachineCPVal = v;
1703 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1704 }
1705
1706public:
1707 bool isMachineConstantPoolEntry() const {
1708 return Offset < 0;
1709 }
1710
1711 const Constant *getConstVal() const {
1712 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((!isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1712, __PRETTY_FUNCTION__))
;
1713 return Val.ConstVal;
1714 }
1715
1716 MachineConstantPoolValue *getMachineCPVal() const {
1717 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1717, __PRETTY_FUNCTION__))
;
1718 return Val.MachineCPVal;
1719 }
1720
1721 int getOffset() const {
1722 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1723 }
1724
1725 // Return the alignment of this constant pool object, which is either 0 (for
1726 // default alignment) or the desired value.
1727 unsigned getAlignment() const { return Alignment; }
1728 unsigned char getTargetFlags() const { return TargetFlags; }
1729
1730 Type *getType() const;
1731
1732 static bool classof(const SDNode *N) {
1733 return N->getOpcode() == ISD::ConstantPool ||
1734 N->getOpcode() == ISD::TargetConstantPool;
1735 }
1736};
1737
1738/// Completely target-dependent object reference.
1739class TargetIndexSDNode : public SDNode {
1740 friend class SelectionDAG;
1741
1742 unsigned char TargetFlags;
1743 int Index;
1744 int64_t Offset;
1745
1746public:
1747 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned char TF)
1748 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1749 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1750
1751 unsigned char getTargetFlags() const { return TargetFlags; }
1752 int getIndex() const { return Index; }
1753 int64_t getOffset() const { return Offset; }
1754
1755 static bool classof(const SDNode *N) {
1756 return N->getOpcode() == ISD::TargetIndex;
1757 }
1758};
1759
1760class BasicBlockSDNode : public SDNode {
1761 friend class SelectionDAG;
1762
1763 MachineBasicBlock *MBB;
1764
1765 /// Debug info is meaningful and potentially useful here, but we create
1766 /// blocks out of order when they're jumped to, which makes it a bit
1767 /// harder. Let's see if we need it first.
1768 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1769 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1770 {}
1771
1772public:
1773 MachineBasicBlock *getBasicBlock() const { return MBB; }
1774
1775 static bool classof(const SDNode *N) {
1776 return N->getOpcode() == ISD::BasicBlock;
1777 }
1778};
1779
1780/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1781class BuildVectorSDNode : public SDNode {
1782public:
1783 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1784 explicit BuildVectorSDNode() = delete;
1785
1786 /// Check if this is a constant splat, and if so, find the
1787 /// smallest element size that splats the vector. If MinSplatBits is
1788 /// nonzero, the element size must be at least that large. Note that the
1789 /// splat element may be the entire vector (i.e., a one element vector).
1790 /// Returns the splat element value in SplatValue. Any undefined bits in
1791 /// that value are zero, and the corresponding bits in the SplatUndef mask
1792 /// are set. The SplatBitSize value is set to the splat element size in
1793 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1794 /// undefined. isBigEndian describes the endianness of the target.
1795 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1796 unsigned &SplatBitSize, bool &HasAnyUndefs,
1797 unsigned MinSplatBits = 0,
1798 bool isBigEndian = false) const;
1799
1800 /// Returns the splatted value or a null value if this is not a splat.
1801 ///
1802 /// If passed a non-null UndefElements bitvector, it will resize it to match
1803 /// the vector width and set the bits where elements are undef.
1804 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1805
1806 /// Returns the splatted constant or null if this is not a constant
1807 /// splat.
1808 ///
1809 /// If passed a non-null UndefElements bitvector, it will resize it to match
1810 /// the vector width and set the bits where elements are undef.
1811 ConstantSDNode *
1812 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
1813
1814 /// Returns the splatted constant FP or null if this is not a constant
1815 /// FP splat.
1816 ///
1817 /// If passed a non-null UndefElements bitvector, it will resize it to match
1818 /// the vector width and set the bits where elements are undef.
1819 ConstantFPSDNode *
1820 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
1821
1822 /// If this is a constant FP splat and the splatted constant FP is an
1823 /// exact power or 2, return the log base 2 integer value. Otherwise,
1824 /// return -1.
1825 ///
1826 /// The BitWidth specifies the necessary bit precision.
1827 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
1828 uint32_t BitWidth) const;
1829
1830 bool isConstant() const;
1831
1832 static bool classof(const SDNode *N) {
1833 return N->getOpcode() == ISD::BUILD_VECTOR;
1834 }
1835};
1836
1837/// An SDNode that holds an arbitrary LLVM IR Value. This is
1838/// used when the SelectionDAG needs to make a simple reference to something
1839/// in the LLVM IR representation.
1840///
1841class SrcValueSDNode : public SDNode {
1842 friend class SelectionDAG;
1843
1844 const Value *V;
1845
1846 /// Create a SrcValue for a general value.
1847 explicit SrcValueSDNode(const Value *v)
1848 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
1849
1850public:
1851 /// Return the contained Value.
1852 const Value *getValue() const { return V; }
1853
1854 static bool classof(const SDNode *N) {
1855 return N->getOpcode() == ISD::SRCVALUE;
1856 }
1857};
1858
1859class MDNodeSDNode : public SDNode {
1860 friend class SelectionDAG;
1861
1862 const MDNode *MD;
1863
1864 explicit MDNodeSDNode(const MDNode *md)
1865 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
1866 {}
1867
1868public:
1869 const MDNode *getMD() const { return MD; }
1870
1871 static bool classof(const SDNode *N) {
1872 return N->getOpcode() == ISD::MDNODE_SDNODE;
1873 }
1874};
1875
1876class RegisterSDNode : public SDNode {
1877 friend class SelectionDAG;
1878
1879 unsigned Reg;
1880
1881 RegisterSDNode(unsigned reg, EVT VT)
1882 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
1883
1884public:
1885 unsigned getReg() const { return Reg; }
1886
1887 static bool classof(const SDNode *N) {
1888 return N->getOpcode() == ISD::Register;
1889 }
1890};
1891
1892class RegisterMaskSDNode : public SDNode {
1893 friend class SelectionDAG;
1894
1895 // The memory for RegMask is not owned by the node.
1896 const uint32_t *RegMask;
1897
1898