LLVM 24.0.0git
AArch64ISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- AArch64ISelDAGToDAG.cpp - A dag to dag inst selector for AArch64 --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the AArch64 target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64ExpandImm.h"
18#include "llvm/ADT/APSInt.h"
22#include "llvm/IR/Function.h" // To access function attributes.
23#include "llvm/IR/GlobalValue.h"
24#include "llvm/IR/Intrinsics.h"
25#include "llvm/IR/IntrinsicsAArch64.h"
27#include "llvm/Support/Debug.h"
32
33using namespace llvm;
34using namespace llvm::SDPatternMatch;
35
36#define DEBUG_TYPE "aarch64-isel"
37#define PASS_NAME "AArch64 Instruction Selection"
38
39// https://github.com/llvm/llvm-project/issues/114425
40#if defined(_MSC_VER) && !defined(__clang__) && !defined(NDEBUG)
41#pragma inline_depth(0)
42#endif
43
44//===--------------------------------------------------------------------===//
45/// AArch64DAGToDAGISel - AArch64 specific code to select AArch64 machine
46/// instructions for SelectionDAG operations.
47///
48namespace {
49
50class AArch64DAGToDAGISel : public SelectionDAGISel {
51
52 /// Subtarget - Keep a pointer to the AArch64Subtarget around so that we can
53 /// make the right decision when generating code for different targets.
55
56public:
57 AArch64DAGToDAGISel() = delete;
58
59 explicit AArch64DAGToDAGISel(AArch64TargetMachine &tm,
60 CodeGenOptLevel OptLevel)
61 : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr) {}
62
63 bool runOnMachineFunction(MachineFunction &MF) override {
66 }
67
68 void Select(SDNode *Node) override;
69 void PreprocessISelDAG() override;
70
71 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
72 /// inline asm expressions.
73 bool SelectInlineAsmMemoryOperand(const SDValue &Op,
74 InlineAsm::ConstraintCode ConstraintID,
75 std::vector<SDValue> &OutOps) override;
76
77 template <signed Low, signed High, signed Scale>
78 bool SelectRDVLImm(SDValue N, SDValue &Imm);
79
80 template <signed Low, signed High>
81 bool SelectRDSVLShiftImm(SDValue N, SDValue &Imm);
82
83 bool SelectArithExtendedRegister(SDValue N, SDValue &Reg, SDValue &Shift);
84 bool SelectArithUXTXRegister(SDValue N, SDValue &Reg, SDValue &Shift);
85 bool SelectArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
86 bool SelectNegArithImmed(SDValue N, SDValue &Val, SDValue &Shift);
87 bool SelectArithShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
88 return SelectShiftedRegister(N, false, Reg, Shift);
89 }
90 bool SelectLogicalShiftedRegister(SDValue N, SDValue &Reg, SDValue &Shift) {
91 return SelectShiftedRegister(N, true, Reg, Shift);
92 }
93 template <unsigned ShiftWidth>
94 bool SelectShiftMask(SDValue N, SDValue &ShAmt);
95
96 bool SelectAddrModeIndexed7S8(SDValue N, SDValue &Base, SDValue &OffImm) {
97 return SelectAddrModeIndexed7S(N, 1, Base, OffImm);
98 }
99 bool SelectAddrModeIndexed7S16(SDValue N, SDValue &Base, SDValue &OffImm) {
100 return SelectAddrModeIndexed7S(N, 2, Base, OffImm);
101 }
102 bool SelectAddrModeIndexed7S32(SDValue N, SDValue &Base, SDValue &OffImm) {
103 return SelectAddrModeIndexed7S(N, 4, Base, OffImm);
104 }
105 bool SelectAddrModeIndexed7S64(SDValue N, SDValue &Base, SDValue &OffImm) {
106 return SelectAddrModeIndexed7S(N, 8, Base, OffImm);
107 }
108 bool SelectAddrModeIndexed7S128(SDValue N, SDValue &Base, SDValue &OffImm) {
109 return SelectAddrModeIndexed7S(N, 16, Base, OffImm);
110 }
111 bool SelectAddrModeIndexedS9S128(SDValue N, SDValue &Base, SDValue &OffImm) {
112 return SelectAddrModeIndexedBitWidth(N, true, 9, 16, Base, OffImm);
113 }
114 bool SelectAddrModeIndexedU6S128(SDValue N, SDValue &Base, SDValue &OffImm) {
115 return SelectAddrModeIndexedBitWidth(N, false, 6, 16, Base, OffImm);
116 }
117 bool SelectAddrModeIndexed8(SDValue N, SDValue &Base, SDValue &OffImm) {
118 return SelectAddrModeIndexed(N, 1, Base, OffImm);
119 }
120 bool SelectAddrModeIndexed16(SDValue N, SDValue &Base, SDValue &OffImm) {
121 return SelectAddrModeIndexed(N, 2, Base, OffImm);
122 }
123 bool SelectAddrModeIndexed32(SDValue N, SDValue &Base, SDValue &OffImm) {
124 return SelectAddrModeIndexed(N, 4, Base, OffImm);
125 }
126 bool SelectAddrModeIndexed64(SDValue N, SDValue &Base, SDValue &OffImm) {
127 return SelectAddrModeIndexed(N, 8, Base, OffImm);
128 }
129 bool SelectAddrModeIndexed128(SDValue N, SDValue &Base, SDValue &OffImm) {
130 return SelectAddrModeIndexed(N, 16, Base, OffImm);
131 }
132 bool SelectAddrModeUnscaled8(SDValue N, SDValue &Base, SDValue &OffImm) {
133 return SelectAddrModeUnscaled(N, 1, Base, OffImm);
134 }
135 bool SelectAddrModeUnscaled16(SDValue N, SDValue &Base, SDValue &OffImm) {
136 return SelectAddrModeUnscaled(N, 2, Base, OffImm);
137 }
138 bool SelectAddrModeUnscaled32(SDValue N, SDValue &Base, SDValue &OffImm) {
139 return SelectAddrModeUnscaled(N, 4, Base, OffImm);
140 }
141 bool SelectAddrModeUnscaled64(SDValue N, SDValue &Base, SDValue &OffImm) {
142 return SelectAddrModeUnscaled(N, 8, Base, OffImm);
143 }
144 bool SelectAddrModeUnscaled128(SDValue N, SDValue &Base, SDValue &OffImm) {
145 return SelectAddrModeUnscaled(N, 16, Base, OffImm);
146 }
147 template <unsigned Size, unsigned Max>
148 bool SelectAddrModeIndexedUImm(SDValue N, SDValue &Base, SDValue &OffImm) {
149 // Test if there is an appropriate addressing mode and check if the
150 // immediate fits.
151 bool Found = SelectAddrModeIndexed(N, Size, Base, OffImm);
152 if (Found) {
153 if (auto *CI = dyn_cast<ConstantSDNode>(OffImm)) {
154 int64_t C = CI->getSExtValue();
155 if (C <= Max)
156 return true;
157 }
158 }
159
160 // Otherwise, base only, materialize address in register.
161 Base = N;
162 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
163 return true;
164 }
165
166 template<int Width>
167 bool SelectAddrModeWRO(SDValue N, SDValue &Base, SDValue &Offset,
168 SDValue &SignExtend, SDValue &DoShift) {
169 return SelectAddrModeWRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
170 }
171
172 template<int Width>
173 bool SelectAddrModeXRO(SDValue N, SDValue &Base, SDValue &Offset,
174 SDValue &SignExtend, SDValue &DoShift) {
175 return SelectAddrModeXRO(N, Width / 8, Base, Offset, SignExtend, DoShift);
176 }
177
178 bool SelectExtractHigh(SDValue N, SDValue &Res) {
179 if (Subtarget->isLittleEndian() && N->getOpcode() == ISD::BITCAST)
180 N = N->getOperand(0);
181 if (N->getOpcode() != ISD::EXTRACT_SUBVECTOR ||
182 !isa<ConstantSDNode>(N->getOperand(1)))
183 return false;
184 EVT VT = N->getValueType(0);
185 EVT LVT = N->getOperand(0).getValueType();
186 unsigned Index = N->getConstantOperandVal(1);
187 if (!VT.is64BitVector() || !LVT.is128BitVector() ||
188 Index != VT.getVectorNumElements())
189 return false;
190 Res = N->getOperand(0);
191 return true;
192 }
193
194 bool SelectRoundingVLShr(SDValue N, SDValue &Res1, SDValue &Res2) {
195 if (N.getOpcode() != AArch64ISD::VLSHR)
196 return false;
197 SDValue Op = N->getOperand(0);
198 EVT VT = Op.getValueType();
199 unsigned ShtAmt = N->getConstantOperandVal(1);
200 if (ShtAmt > VT.getScalarSizeInBits() / 2 || Op.getOpcode() != ISD::ADD)
201 return false;
202
203 APInt Imm;
204 if (Op.getOperand(1).getOpcode() == AArch64ISD::MOVIshift)
206 Op.getOperand(1).getConstantOperandVal(0)
207 << Op.getOperand(1).getConstantOperandVal(1));
208 else if (Op.getOperand(1).getOpcode() == AArch64ISD::DUP &&
209 isa<ConstantSDNode>(Op.getOperand(1).getOperand(0)))
211 Op.getOperand(1).getConstantOperandVal(0));
212 else
213 return false;
214
215 if (Imm != 1ULL << (ShtAmt - 1))
216 return false;
217
218 Res1 = Op.getOperand(0);
219 Res2 = CurDAG->getTargetConstant(ShtAmt, SDLoc(N), MVT::i32);
220 return true;
221 }
222
223 bool SelectDupZeroOrUndef(SDValue N) {
224 switch(N->getOpcode()) {
225 case ISD::UNDEF:
226 case ISD::POISON:
227 return true;
228 case AArch64ISD::DUP:
229 case ISD::SPLAT_VECTOR: {
230 auto Opnd0 = N->getOperand(0);
231 if (isNullConstant(Opnd0))
232 return true;
233 if (isNullFPConstant(Opnd0))
234 return true;
235 break;
236 }
237 default:
238 break;
239 }
240
241 return false;
242 }
243
244 bool SelectAny(SDValue) { return true; }
245
246 bool SelectDupZero(SDValue N) {
247 switch(N->getOpcode()) {
248 case AArch64ISD::DUP:
249 case ISD::SPLAT_VECTOR: {
250 auto Opnd0 = N->getOperand(0);
251 if (isNullConstant(Opnd0))
252 return true;
253 if (isNullFPConstant(Opnd0))
254 return true;
255 break;
256 }
257 }
258
259 return false;
260 }
261
262 template <MVT::SimpleValueType VT, bool Negate>
263 bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
264 return SelectSVEAddSubImm(N, VT, Imm, Shift, Negate);
265 }
266
267 template <MVT::SimpleValueType VT, bool Negate>
268 bool SelectSVEAddSubSSatImm(SDValue N, SDValue &Imm, SDValue &Shift) {
269 return SelectSVEAddSubSSatImm(N, VT, Imm, Shift, Negate);
270 }
271
272 template <MVT::SimpleValueType VT>
273 bool SelectSVECpyDupImm(SDValue N, SDValue &Imm, SDValue &Shift) {
274 return SelectSVECpyDupImm(N, VT, Imm, Shift);
275 }
276
277 template <MVT::SimpleValueType VT, bool Invert = false>
278 bool SelectSVELogicalImm(SDValue N, SDValue &Imm) {
279 return SelectSVELogicalImm(N, VT, Imm, Invert);
280 }
281
282 template <MVT::SimpleValueType VT>
283 bool SelectSVEArithImm(SDValue N, SDValue &Imm) {
284 return SelectSVEArithImm(N, VT, Imm);
285 }
286
287 template <unsigned Low, unsigned High, bool AllowSaturation = false>
288 bool SelectSVEShiftImm(SDValue N, SDValue &Imm) {
289 return SelectSVEShiftImm(N, Low, High, AllowSaturation, Imm);
290 }
291
292 bool SelectSVEShiftSplatImmR(SDValue N, SDValue &Imm) {
293 if (N->getOpcode() != ISD::SPLAT_VECTOR)
294 return false;
295
296 EVT EltVT = N->getValueType(0).getVectorElementType();
297 return SelectSVEShiftImm(N->getOperand(0), /* Low */ 1,
298 /* High */ EltVT.getFixedSizeInBits(),
299 /* AllowSaturation */ true, Imm);
300 }
301
302 // Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
303 template<signed Min, signed Max, signed Scale, bool Shift>
304 bool SelectCntImm(SDValue N, SDValue &Imm) {
306 return false;
307
308 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
309 if (Shift)
310 MulImm = 1LL << MulImm;
311
312 if ((MulImm % std::abs(Scale)) != 0)
313 return false;
314
315 MulImm /= Scale;
316 if ((MulImm >= Min) && (MulImm <= Max)) {
317 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32);
318 return true;
319 }
320
321 return false;
322 }
323
324 template <signed Max, signed Scale>
325 bool SelectEXTImm(SDValue N, SDValue &Imm) {
327 return false;
328
329 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
330
331 if (MulImm >= 0 && MulImm <= Max) {
332 MulImm *= Scale;
333 Imm = CurDAG->getTargetConstant(MulImm, SDLoc(N), MVT::i32);
334 return true;
335 }
336
337 return false;
338 }
339
340 template <unsigned BaseReg, unsigned Max>
341 bool ImmToReg(SDValue N, SDValue &Imm) {
342 if (auto *CI = dyn_cast<ConstantSDNode>(N)) {
343 uint64_t C = CI->getZExtValue();
344
345 if (C > Max)
346 return false;
347
348 Imm = CurDAG->getRegister(BaseReg + C, MVT::Other);
349 return true;
350 }
351 return false;
352 }
353
354 /// Form sequences of consecutive 64/128-bit registers for use in NEON
355 /// instructions making use of a vector-list (e.g. ldN, tbl). Vecs must have
356 /// between 1 and 4 elements. If it contains a single element that is returned
357 /// unchanged; otherwise a REG_SEQUENCE value is returned.
360 // Form a sequence of SVE registers for instructions using list of vectors,
361 // e.g. structured loads and stores (ldN, stN).
362 SDValue createZTuple(ArrayRef<SDValue> Vecs);
363
364 // Similar to above, except the register must start at a multiple of the
365 // tuple, e.g. z2 for a 2-tuple, or z8 for a 4-tuple.
366 SDValue createZMulTuple(ArrayRef<SDValue> Regs);
367
368 /// Generic helper for the createDTuple/createQTuple
369 /// functions. Those should almost always be called instead.
370 SDValue createTuple(ArrayRef<SDValue> Vecs, const unsigned RegClassIDs[],
371 const unsigned SubRegs[]);
372
373 void SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc, bool isExt);
374
375 bool tryIndexedLoad(SDNode *N);
376
377 void SelectPtrauthAuth(SDNode *N);
378 void SelectPtrauthResign(SDNode *N);
379 void SelectPtrauthResignWithPC(SDNode *N);
380
381 bool trySelectStackSlotTagP(SDNode *N);
382 void SelectTagP(SDNode *N);
383
384 void SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
385 unsigned SubRegIdx);
386 void SelectPostLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
387 unsigned SubRegIdx);
388 void SelectLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
389 void SelectPostLoadLane(SDNode *N, unsigned NumVecs, unsigned Opc);
390 void SelectPredicatedLoad(SDNode *N, unsigned NumVecs, unsigned Scale,
391 unsigned Opc_rr, unsigned Opc_ri,
392 bool IsIntr = false);
393 void SelectContiguousMultiVectorLoad(SDNode *N, unsigned NumVecs,
394 unsigned Scale, unsigned Opc_ri,
395 unsigned Opc_rr);
396 void SelectDestructiveMultiIntrinsic(SDNode *N, unsigned NumVecs,
397 bool IsZmMulti, unsigned Opcode,
398 bool HasPred = false);
399 void SelectPExtPair(SDNode *N, unsigned Opc);
400 void SelectWhilePair(SDNode *N, unsigned Opc);
401 void SelectCVTIntrinsic(SDNode *N, unsigned NumVecs, unsigned Opcode);
402 void SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs, unsigned Opcode);
403 void SelectClamp(SDNode *N, unsigned NumVecs, unsigned Opcode);
404 void SelectUnaryMultiIntrinsic(SDNode *N, unsigned NumOutVecs,
405 bool IsTupleInput, unsigned Opc);
406 void SelectFrintFromVT(SDNode *N, unsigned NumVecs, unsigned Opcode);
407
408 template <unsigned MaxIdx, unsigned Scale>
409 void SelectMultiVectorMove(SDNode *N, unsigned NumVecs, unsigned BaseReg,
410 unsigned Op);
411 void SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
412 unsigned Op, unsigned MaxIdx, unsigned Scale,
413 unsigned BaseReg = 0);
414 /// SVE Reg+Imm addressing mode.
415 template <int64_t Min, int64_t Max>
416 bool SelectAddrModeIndexedSVE(SDNode *Root, SDValue N, SDValue &Base,
417 SDValue &OffImm);
418 /// SVE Reg+Reg address mode.
419 template <unsigned Scale>
420 bool SelectSVERegRegAddrMode(SDValue N, SDValue &Base, SDValue &Offset) {
421 return SelectSVERegRegAddrMode(N, Scale, Base, Offset);
422 }
423
424 void SelectMultiVectorLutiLane(SDNode *Node, unsigned NumOutVecs,
425 unsigned Opc, uint32_t MaxImm);
426 void SelectMultiVectorLuti6LaneX4(SDNode *Node, unsigned NumIndexVecs);
427
428 void SelectMultiVectorLuti(SDNode *Node, unsigned NumOutVecs, unsigned Opc,
429 unsigned NumInVecs);
430
431 template <unsigned MaxIdx, unsigned Scale>
432 bool SelectSMETileSlice(SDValue N, SDValue &Vector, SDValue &Offset) {
433 return SelectSMETileSlice(N, MaxIdx, Vector, Offset, Scale);
434 }
435
436 void SelectStore(SDNode *N, unsigned NumVecs, unsigned Opc);
437 void SelectPostStore(SDNode *N, unsigned NumVecs, unsigned Opc);
438 void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
439 void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
440 void SelectPredicatedStore(SDNode *N, unsigned NumVecs, unsigned Scale,
441 unsigned Opc_rr, unsigned Opc_ri);
442 std::tuple<unsigned, SDValue, SDValue>
443 findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr, unsigned Opc_ri,
444 const SDValue &OldBase, const SDValue &OldOffset,
445 unsigned Scale);
446
447 bool tryBitfieldExtractOp(SDNode *N);
448 bool tryBitfieldInsertOp(SDNode *N);
449 bool tryBitfieldInsertInZeroOp(SDNode *N);
450
451 bool tryReadRegister(SDNode *N);
452 bool tryWriteRegister(SDNode *N);
453
454 bool trySelectCastFixedLengthToScalableVector(SDNode *N);
455 bool trySelectCastScalableToFixedLengthVector(SDNode *N);
456
457 bool trySelectXAR(SDNode *N);
458
459 bool tryFoldCselToFMaxMin(SDNode *N);
460
461// Include the pieces autogenerated from the target description.
462#include "AArch64GenDAGISel.inc"
463
464private:
465 bool SelectShiftedRegister(SDValue N, bool AllowROR, SDValue &Reg,
466 SDValue &Shift);
467 bool SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg, SDValue &Shift);
468 bool SelectAddrModeIndexed7S(SDValue N, unsigned Size, SDValue &Base,
469 SDValue &OffImm) {
470 return SelectAddrModeIndexedBitWidth(N, true, 7, Size, Base, OffImm);
471 }
472 bool SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm, unsigned BW,
473 unsigned Size, SDValue &Base,
474 SDValue &OffImm);
475 bool SelectAddrModeIndexed(SDValue N, unsigned Size, SDValue &Base,
476 SDValue &OffImm);
477 bool SelectAddrModeUnscaled(SDValue N, unsigned Size, SDValue &Base,
478 SDValue &OffImm);
479 bool SelectAddrModeWRO(SDValue N, unsigned Size, SDValue &Base,
480 SDValue &Offset, SDValue &SignExtend,
481 SDValue &DoShift);
482 bool SelectAddrModeXRO(SDValue N, unsigned Size, SDValue &Base,
483 SDValue &Offset, SDValue &SignExtend,
484 SDValue &DoShift);
485 bool isWorthNegatingImm(SDValue V) const;
486 bool isWorthFoldingALU(SDValue V, bool LSL = false) const;
487 bool isWorthFoldingAddr(SDValue V, unsigned Size) const;
488 bool SelectExtendedSHL(SDValue N, unsigned Size, bool WantExtend,
489 SDValue &Offset, SDValue &SignExtend);
490
491 template<unsigned RegWidth>
492 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos) {
493 return SelectCVTFixedPosOperand(N, FixedPos, RegWidth);
494 }
495 bool SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos, unsigned Width);
496
497 template <unsigned RegWidth>
498 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos) {
499 return SelectCVTFixedPointVec(N, FixedPos, RegWidth);
500 }
501 bool SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos, unsigned Width);
502
503 template<unsigned RegWidth>
504 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos) {
505 return SelectCVTFixedPosRecipOperand(N, FixedPos, RegWidth);
506 }
507
508 bool SelectCVTFixedPosRecipOperand(SDValue N, SDValue &FixedPos,
509 unsigned Width);
510
511 template <unsigned FloatWidth>
512 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos) {
513 return SelectCVTFixedPosRecipOperandVec(N, FixedPos, FloatWidth);
514 }
515
516 bool SelectCVTFixedPosRecipOperandVec(SDValue N, SDValue &FixedPos,
517 unsigned Width);
518
519 bool SelectCMP_SWAP(SDNode *N);
520
521 AArch64MemoryHint decodeMemoryHintFlags(MachineMemOperand *MMO) const;
522 bool isAtomicMemoryHint(SDNode *N, AArch64MemoryHint Hint) const;
523
524 bool SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
525 bool Negate);
526 bool SelectSVEAddSubImm(SDLoc DL, APInt Value, MVT VT, SDValue &Imm,
527 SDValue &Shift, bool Negate);
528 bool SelectSVEAddSubSSatImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift,
529 bool Negate);
530 bool SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm, SDValue &Shift);
531 bool SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm, bool Invert);
532
533 // Match `<NEON Splat> SVEImm` (where <NEON Splat> could be fmov, movi, etc).
534 bool SelectNEONSplatOfSVELogicalImm(SDValue N, SDValue &Imm);
535 bool SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift);
536 bool SelectNEONSplatOfSVEArithSImm(SDValue N, SDValue &Imm);
537 bool SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm);
538 bool SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm);
539
540 bool SelectSVESignedArithImm(SDLoc DL, APInt Value, SDValue &Imm);
541 bool SelectSVESignedArithImm(SDValue N, SDValue &Imm);
542 bool SelectSVEShiftImm(SDValue N, uint64_t Low, uint64_t High,
543 bool AllowSaturation, SDValue &Imm);
544
545 bool SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm);
546 bool SelectSVERegRegAddrMode(SDValue N, unsigned Scale, SDValue &Base,
547 SDValue &Offset);
548 bool SelectSMETileSlice(SDValue N, unsigned MaxSize, SDValue &Vector,
549 SDValue &Offset, unsigned Scale = 1);
550
551 bool SelectAllActivePredicate(SDValue N);
552 bool SelectAnyPredicate(SDValue N);
553
554 bool SelectCmpBranchUImm6Operand(SDNode *P, SDValue N, SDValue &Imm);
555
556 template <bool MatchCBB>
557 bool SelectCmpBranchExtOperand(SDValue N, SDValue &Reg, SDValue &ExtType);
558};
559
560class AArch64DAGToDAGISelLegacy : public SelectionDAGISelLegacy {
561public:
562 static char ID;
563 explicit AArch64DAGToDAGISelLegacy(AArch64TargetMachine &tm,
564 CodeGenOptLevel OptLevel)
566 ID, std::make_unique<AArch64DAGToDAGISel>(tm, OptLevel)) {}
567};
568} // end anonymous namespace
569
570char AArch64DAGToDAGISelLegacy::ID = 0;
571
572INITIALIZE_PASS(AArch64DAGToDAGISelLegacy, DEBUG_TYPE, PASS_NAME, false, false)
573
576 std::make_unique<AArch64DAGToDAGISel>(TM, TM.getOptLevel())) {}
577
578/// addBitcastHints - This method adds bitcast hints to the operands of a node
579/// to help instruction selector determine which operands are in Neon registers.
581 SDLoc DL(&N);
582 auto getFloatVT = [&](EVT VT) {
583 EVT ScalarVT = VT.getScalarType();
584 assert((ScalarVT == MVT::i32 || ScalarVT == MVT::i64) && "Unexpected VT");
585 return VT.changeElementType(*(DAG.getContext()),
586 ScalarVT == MVT::i32 ? MVT::f32 : MVT::f64);
587 };
589 NewOps.reserve(N.getNumOperands());
590
591 for (unsigned I = 0, E = N.getNumOperands(); I < E; ++I) {
592 auto bitcasted = DAG.getBitcast(getFloatVT(N.getOperand(I).getValueType()),
593 N.getOperand(I));
594 NewOps.push_back(bitcasted);
595 }
596 EVT OrigVT = N.getValueType(0);
597 SDValue OpNode = DAG.getNode(N.getOpcode(), DL, getFloatVT(OrigVT), NewOps);
598 return DAG.getBitcast(OrigVT, OpNode);
599}
600
601/// isIntImmediate - This method tests to see if the node is a constant
602/// operand. If so Imm will receive the 64-bit value.
603static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
605 Imm = C->getZExtValue();
606 return true;
607 }
608 return false;
609}
610
611// isIntImmediate - This method tests to see if a constant operand.
612// If so Imm will receive the value.
614 return isIntImmediate(N.getNode(), Imm);
615}
616
617// isOpcWithIntImmediate - This method tests to see if the node is a specific
618// opcode and that it has a immediate integer right operand.
619// If so Imm will receive the 32 bit value.
620static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
621 uint64_t &Imm) {
622 return N->getOpcode() == Opc &&
623 isIntImmediate(N->getOperand(1).getNode(), Imm);
624}
625
626// isIntImmediateEq - This method tests to see if N is a constant operand that
627// is equivalent to 'ImmExpected'.
628#ifndef NDEBUG
629static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected) {
631 if (!isIntImmediate(N.getNode(), Imm))
632 return false;
633 return Imm == ImmExpected;
634}
635#endif
636
637static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth) {
638 assert(RegWidth == 32 || RegWidth == 64);
639 if (RegWidth == 32)
640 return APInt(RegWidth,
643}
644
645// Decodes the raw integer splat value from a NEON splat operation.
646static std::optional<APInt> DecodeNEONSplat(SDValue N,
648 assert(N.getValueType().isInteger() && "Only integers are supported");
649 if (N->getOpcode() == AArch64ISD::NVCAST ||
650 (N->getOpcode() == ISD::BITCAST && Subtarget->isLittleEndian()))
651 N = N->getOperand(0);
652 unsigned SplatWidth = N.getScalarValueSizeInBits();
653 if (N.getOpcode() == AArch64ISD::FMOV)
654 return DecodeFMOVImm(N.getConstantOperandVal(0), SplatWidth);
655 if (N->getOpcode() == AArch64ISD::MOVI)
656 return APInt(SplatWidth, N.getConstantOperandVal(0));
657 if (N->getOpcode() == AArch64ISD::MOVIshift)
658 return APInt(SplatWidth, N.getConstantOperandVal(0)
659 << N.getConstantOperandVal(1));
660 if (N->getOpcode() == AArch64ISD::MVNIshift)
661 return ~APInt(SplatWidth, N.getConstantOperandVal(0)
662 << N.getConstantOperandVal(1));
663 if (N->getOpcode() == AArch64ISD::MOVIedit)
665 N.getConstantOperandVal(0)));
666 if (N->getOpcode() == AArch64ISD::DUP)
667 if (auto *Const = dyn_cast<ConstantSDNode>(N->getOperand(0)))
668 return Const->getAPIntValue().trunc(SplatWidth);
669 APInt SplatVal;
670 if (ISD::isConstantSplatVector(N, SplatVal))
671 return SplatVal.trunc(SplatWidth);
672 // TODO: Recognize more splat-like NEON operations. See ConstantBuildVector
673 // in AArch64ISelLowering.
674 return std::nullopt;
675}
676
677// If \p N is a NEON splat operation (movi, fmov, etc), return the splat value
678// matching the element size of N.
679static std::optional<APInt>
681 unsigned SplatWidth = N.getScalarValueSizeInBits();
682 if (std::optional<APInt> SplatVal = DecodeNEONSplat(N, Subtarget)) {
683 if (SplatVal->getBitWidth() <= SplatWidth)
684 return APInt::getSplat(SplatWidth, *SplatVal);
685 if (SplatVal->isSplat(SplatWidth))
686 return SplatVal->trunc(SplatWidth);
687 }
688 return std::nullopt;
689}
690
691bool AArch64DAGToDAGISel::SelectNEONSplatOfSVELogicalImm(SDValue N,
692 SDValue &Imm) {
693 std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget);
694 if (!ImmVal)
695 return false;
696 uint64_t Encoding;
697 if (!AArch64_AM::isSVELogicalImm(N.getScalarValueSizeInBits(),
698 ImmVal->getZExtValue(), Encoding))
699 return false;
700
701 Imm = CurDAG->getTargetConstant(Encoding, SDLoc(N), MVT::i64);
702 return true;
703}
704
705bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEAddSubImm(SDValue N, SDValue &Imm,
706 SDValue &Shift) {
707 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
708 return SelectSVEAddSubImm(SDLoc(N), *ImmVal,
709 N.getValueType().getScalarType().getSimpleVT(),
710 Imm, Shift,
711 /*Negate=*/false);
712 return false;
713}
714
715bool AArch64DAGToDAGISel::SelectNEONSplatOfSVEArithSImm(SDValue N,
716 SDValue &Imm) {
717 if (std::optional<APInt> ImmVal = GetNEONSplatValue(N, Subtarget))
718 return SelectSVESignedArithImm(SDLoc(N), *ImmVal, Imm);
719 return false;
720}
721
722bool AArch64DAGToDAGISel::SelectNEONSplatOfSImm8(SDValue N, SDValue &Imm) {
723 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
724 if (!ImmAPIntVal)
725 return false;
726
727 int64_t ImmVal = ImmAPIntVal->getSExtValue();
728 if (ImmVal < -128 || ImmVal > 127)
729 return false;
730
731 Imm = CurDAG->getSignedTargetConstant(ImmVal, SDLoc(N), MVT::i32);
732 return true;
733}
734
735bool AArch64DAGToDAGISel::SelectNEONSplatOfUImm8(SDValue N, SDValue &Imm) {
736 std::optional<APInt> ImmAPIntVal = GetNEONSplatValue(N, Subtarget);
737 if (!ImmAPIntVal)
738 return false;
739
740 uint64_t ImmVal = ImmAPIntVal->getZExtValue();
741 if (ImmVal > 255)
742 return false;
743
744 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32);
745 return true;
746}
747
748bool AArch64DAGToDAGISel::SelectInlineAsmMemoryOperand(
749 const SDValue &Op, const InlineAsm::ConstraintCode ConstraintID,
750 std::vector<SDValue> &OutOps) {
751 switch(ConstraintID) {
752 default:
753 llvm_unreachable("Unexpected asm memory constraint");
754 case InlineAsm::ConstraintCode::m:
755 case InlineAsm::ConstraintCode::o:
756 case InlineAsm::ConstraintCode::Q:
757 // We need to make sure that this one operand does not end up in XZR, thus
758 // require the address to be in a pointer register.
759 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
760 const TargetRegisterClass *TRC =
762 SDLoc dl(Op);
763 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), dl, MVT::i64);
764 SDValue NewOp =
765 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
766 dl, Op.getValueType(),
767 Op, RC), 0);
768 OutOps.push_back(NewOp);
769 return false;
770 }
771 return true;
772}
773
774// Returns true if every use of N is a variable shift (SHL/SRL/SRA/ROTR) that
775// uses N as its shift-amount operand. When that holds, folding N into the
776// shift amount (e.g. generating a NEG or MVN) is safe even with multiple
777// uses, because each shift only reads the low bits of the amount.
779 return all_of(N.getNode()->users(), [&](SDNode *User) {
780 switch (User->getOpcode()) {
781 case ISD::SHL:
782 case ISD::SRL:
783 case ISD::SRA:
784 case ISD::ROTR:
785 return User->getOperand(1).getNode() == N.getNode();
786 case ISD::ANY_EXTEND:
787 case ISD::ZERO_EXTEND:
788 case ISD::SIGN_EXTEND:
789 // A 32-bit shift amount may be widened to i64 before it reaches the
790 // shift; look through the extend to the ultimate users.
791 return allUsesAreShiftAmounts(SDValue(User, 0));
792 default:
793 return false;
794 }
795 });
796}
797
798template <unsigned ShiftWidth>
799bool AArch64DAGToDAGISel::SelectShiftMask(SDValue N, SDValue &ShAmt) {
800 // AArch64 shift instructions only use the low log2(ShiftWidth) bits of the
801 // shift amount. The matcher may run on a node wider or narrower than the
802 // shift result (an i32 amount widened to i64, or an i64 modulus reduced to
803 // an i32 subtract), so the redundancy threshold and modulus come from the
804 // ShiftWidth template parameter, while the NEG/NOT opcode follows the node's
805 // own type. Return false if nothing was combined so other patterns (e.g.
806 // zext/sext GPR32 -> SUBREG_TO_REG) can match.
807 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
808 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
809 uint64_t Mask = N.getConstantOperandVal(1);
810 // Remove AND if the mask covers at least the low log2(ShiftWidth) bits.
811 if ((unsigned)llvm::countr_one(Mask) >= Log2_32(ShiftWidth)) {
812 ShAmt = N.getOperand(0);
813 return true;
814 }
815 }
816 // If shifting by X+/-N where N == 0 mod ShiftWidth, then just shift by X
817 // to avoid the ADD/SUB. The low log2(ShiftWidth) bits are unchanged, so the
818 // shift can use X directly; the original ADD/SUB stays for any other users.
819 if ((N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::SUB) &&
820 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
822 if (isIntImmediate(N.getOperand(1).getNode(), Imm) &&
823 (Imm % ShiftWidth == 0)) {
824 ShAmt = N.getOperand(0);
825 return true;
826 }
827 }
828
829 // If shifting by N-X where N == 0 mod ShiftWidth, then just shift by -X
830 // to generate a NEG instead of a SUB from a constant.
831 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
832 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
834 if (isIntImmediate(N.getOperand(0).getNode(), Imm) && Imm != 0 &&
835 (Imm % ShiftWidth == 0)) {
836 SDLoc DL(N);
837 EVT VT = N.getValueType();
838 bool Is32 = VT == MVT::i32;
839 unsigned NegOpc = Is32 ? AArch64::SUBWrr : AArch64::SUBXrr;
840 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
841 SDValue Zero =
842 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, VT);
843 MachineSDNode *Neg =
844 CurDAG->getMachineNode(NegOpc, DL, VT, Zero, N.getOperand(1));
845 ShAmt = SDValue(Neg, 0);
846 return true;
847 }
848 }
849
850 // If shifting by N-X where N == -1 mod ShiftWidth, then just shift by ~X
851 // to generate a NOT (MVN) instead of a SUB from a constant.
852 if (N.getOpcode() == ISD::SUB && allUsesAreShiftAmounts(N) &&
853 (N.getValueType() == MVT::i32 || N.getValueType() == MVT::i64)) {
855 if (isIntImmediate(N.getOperand(0).getNode(), Imm) &&
856 (Imm % ShiftWidth == ShiftWidth - 1)) {
857 SDLoc DL(N);
858 EVT VT = N.getValueType();
859 bool Is32 = VT == MVT::i32;
860 unsigned NotOpc = Is32 ? AArch64::ORNWrr : AArch64::ORNXrr;
861 unsigned ZeroReg = Is32 ? AArch64::WZR : AArch64::XZR;
862 SDValue Zero =
863 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, ZeroReg, VT);
864 MachineSDNode *Not =
865 CurDAG->getMachineNode(NotOpc, DL, VT, Zero, N.getOperand(1));
866 ShAmt = SDValue(Not, 0);
867 return true;
868 }
869 }
870
871 return false;
872}
873
874/// SelectArithImmed - Select an immediate value that can be represented as
875/// a 12-bit value shifted left by either 0 or 12. If so, return true with
876/// Val set to the 12-bit value and Shift set to the shifter operand.
877bool AArch64DAGToDAGISel::SelectArithImmed(SDValue N, SDValue &Val,
878 SDValue &Shift) {
879 // This function is called from the addsub_shifted_imm ComplexPattern,
880 // which lists [imm] as the list of opcode it's interested in, however
881 // we still need to check whether the operand is actually an immediate
882 // here because the ComplexPattern opcode list is only used in
883 // root-level opcode matching.
884 if (!isa<ConstantSDNode>(N.getNode()))
885 return false;
886
887 uint64_t Immed = N.getNode()->getAsZExtVal();
888
890 return false;
891
892 unsigned ShiftAmt = AArch64_AM::getArithImmedShift(Immed);
893 Immed >>= ShiftAmt;
894
895 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShiftAmt);
896 SDLoc dl(N);
897 Val = CurDAG->getTargetConstant(Immed, dl, MVT::i32);
898 Shift = CurDAG->getTargetConstant(ShVal, dl, MVT::i32);
899 return true;
900}
901
902/// SelectNegArithImmed - As above, but negates the value before trying to
903/// select it.
904bool AArch64DAGToDAGISel::SelectNegArithImmed(SDValue N, SDValue &Val,
905 SDValue &Shift) {
906 // This function is called from the addsub_shifted_imm ComplexPattern,
907 // which lists [imm] as the list of opcode it's interested in, however
908 // we still need to check whether the operand is actually an immediate
909 // here because the ComplexPattern opcode list is only used in
910 // root-level opcode matching.
911 if (!isa<ConstantSDNode>(N.getNode()))
912 return false;
913
914 // The immediate operand must be a 24-bit zero-extended immediate.
915 uint64_t Immed = N.getNode()->getAsZExtVal();
916
917 // This negation is almost always valid, but "cmp wN, #0" and "cmn wN, #0"
918 // have the opposite effect on the C flag, so this pattern mustn't match under
919 // those circumstances.
920 if (Immed == 0)
921 return false;
922
923 if (N.getValueType() == MVT::i32)
924 Immed = ~((uint32_t)Immed) + 1;
925 else
926 Immed = ~Immed + 1ULL;
927 if (Immed & 0xFFFFFFFFFF000000ULL)
928 return false;
929
930 Immed &= 0xFFFFFFULL;
931 return SelectArithImmed(CurDAG->getConstant(Immed, SDLoc(N), MVT::i32), Val,
932 Shift);
933}
934
935/// getShiftTypeForNode - Translate a shift node to the corresponding
936/// ShiftType value.
938 switch (N.getOpcode()) {
939 default:
941 case ISD::SHL:
942 return AArch64_AM::LSL;
943 case ISD::SRL:
944 return AArch64_AM::LSR;
945 case ISD::SRA:
946 return AArch64_AM::ASR;
947 case ISD::ROTR:
948 return AArch64_AM::ROR;
949 }
950}
951
953 return isa<MemSDNode>(*N) || N->getOpcode() == AArch64ISD::PREFETCH;
954}
955
956/// Determine whether it is worth it to fold SHL into the addressing
957/// mode.
959 assert(V.getOpcode() == ISD::SHL && "invalid opcode");
960 // It is worth folding logical shift of up to three places.
961 auto *CSD = dyn_cast<ConstantSDNode>(V.getOperand(1));
962 if (!CSD)
963 return false;
964 unsigned ShiftVal = CSD->getZExtValue();
965 if (ShiftVal > 3)
966 return false;
967
968 // Check if this particular node is reused in any non-memory related
969 // operation. If yes, do not try to fold this node into the address
970 // computation, since the computation will be kept.
971 const SDNode *Node = V.getNode();
972 for (SDNode *UI : Node->users())
973 if (!isMemOpOrPrefetch(UI))
974 for (SDNode *UII : UI->users())
975 if (!isMemOpOrPrefetch(UII))
976 return false;
977 return true;
978}
979
980/// Determine whether it is worth to fold V into an extended register addressing
981/// mode.
982bool AArch64DAGToDAGISel::isWorthFoldingAddr(SDValue V, unsigned Size) const {
983 // Trivial if we are optimizing for code size or if there is only
984 // one use of the value.
985 if (CurDAG->shouldOptForSize() || V.hasOneUse())
986 return true;
987
988 // If a subtarget has a slow shift, folding a shift into multiple loads
989 // costs additional micro-ops.
990 if (Subtarget->hasAddrLSLSlow14() && (Size == 2 || Size == 16))
991 return false;
992
993 // Check whether we're going to emit the address arithmetic anyway because
994 // it's used by a non-address operation.
995 if (V.getOpcode() == ISD::SHL && isWorthFoldingSHL(V))
996 return true;
997 if (V.getOpcode() == ISD::ADD) {
998 const SDValue LHS = V.getOperand(0);
999 const SDValue RHS = V.getOperand(1);
1000 if (LHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(LHS))
1001 return true;
1002 if (RHS.getOpcode() == ISD::SHL && isWorthFoldingSHL(RHS))
1003 return true;
1004 }
1005
1006 // It hurts otherwise, since the value will be reused.
1007 return false;
1008}
1009
1010/// and (shl/srl/sra, x, c), mask --> shl (srl/sra, x, c1), c2
1011/// to select more shifted register
1012bool AArch64DAGToDAGISel::SelectShiftedRegisterFromAnd(SDValue N, SDValue &Reg,
1013 SDValue &Shift) {
1014 EVT VT = N.getValueType();
1015 if (VT != MVT::i32 && VT != MVT::i64)
1016 return false;
1017
1018 if (N->getOpcode() != ISD::AND || !N->hasOneUse())
1019 return false;
1020 SDValue LHS = N.getOperand(0);
1021 if (!LHS->hasOneUse())
1022 return false;
1023
1024 unsigned LHSOpcode = LHS->getOpcode();
1025 if (LHSOpcode != ISD::SHL && LHSOpcode != ISD::SRL && LHSOpcode != ISD::SRA)
1026 return false;
1027
1028 ConstantSDNode *ShiftAmtNode = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
1029 if (!ShiftAmtNode)
1030 return false;
1031
1032 uint64_t ShiftAmtC = ShiftAmtNode->getZExtValue();
1033 ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(N.getOperand(1));
1034 if (!RHSC)
1035 return false;
1036
1037 APInt AndMask = RHSC->getAPIntValue();
1038 unsigned LowZBits, MaskLen;
1039 if (!AndMask.isShiftedMask(LowZBits, MaskLen))
1040 return false;
1041
1042 unsigned BitWidth = N.getValueSizeInBits();
1043 SDLoc DL(LHS);
1044 uint64_t NewShiftC;
1045 unsigned NewShiftOp;
1046 if (LHSOpcode == ISD::SHL) {
1047 // LowZBits <= ShiftAmtC will fall into isBitfieldPositioningOp
1048 if (LowZBits <= ShiftAmtC)
1049 return false;
1050
1051 NewShiftC = LowZBits - ShiftAmtC;
1052 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1053 } else {
1054 if (LowZBits == 0)
1055 return false;
1056
1057 // NewShiftC >= BitWidth will fall into isBitfieldExtractOp
1058 NewShiftC = LowZBits + ShiftAmtC;
1059 if (NewShiftC >= BitWidth)
1060 return false;
1061
1062 // SRA need all high bits
1063 if (LHSOpcode == ISD::SRA && (BitWidth != (LowZBits + MaskLen)))
1064 return false;
1065
1066 if (LHSOpcode == ISD::SRL)
1067 NewShiftOp = VT == MVT::i64 ? AArch64::UBFMXri : AArch64::UBFMWri;
1068 else
1069 NewShiftOp = VT == MVT::i64 ? AArch64::SBFMXri : AArch64::SBFMWri;
1070 }
1071
1072 assert(NewShiftC < BitWidth && "Invalid shift amount");
1073 SDValue NewShiftAmt = CurDAG->getTargetConstant(NewShiftC, DL, VT);
1074 // If the mask doesn't extend to the MSB, extract only the masked field.
1075 unsigned MSB = BitWidth - 1;
1076 if (LowZBits + MaskLen < BitWidth)
1077 MSB = std::min<uint64_t>(NewShiftC + MaskLen, BitWidth) - 1;
1078 SDValue MSBC = CurDAG->getTargetConstant(MSB, DL, VT);
1079 Reg = SDValue(CurDAG->getMachineNode(NewShiftOp, DL, VT, LHS->getOperand(0),
1080 NewShiftAmt, MSBC),
1081 0);
1082 unsigned ShVal = AArch64_AM::getShifterImm(AArch64_AM::LSL, LowZBits);
1083 Shift = CurDAG->getTargetConstant(ShVal, DL, MVT::i32);
1084 return true;
1085}
1086
1087/// getExtendTypeForNode - Translate an extend node to the corresponding
1088/// ExtendType value.
1090getExtendTypeForNode(SDValue N, bool IsLoadStore = false) {
1091 if (N.getOpcode() == ISD::SIGN_EXTEND ||
1092 N.getOpcode() == ISD::SIGN_EXTEND_INREG) {
1093 EVT SrcVT;
1094 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG)
1095 SrcVT = cast<VTSDNode>(N.getOperand(1))->getVT();
1096 else
1097 SrcVT = N.getOperand(0).getValueType();
1098
1099 if (!IsLoadStore && SrcVT == MVT::i8)
1100 return AArch64_AM::SXTB;
1101 else if (!IsLoadStore && SrcVT == MVT::i16)
1102 return AArch64_AM::SXTH;
1103 else if (SrcVT == MVT::i32)
1104 return AArch64_AM::SXTW;
1105 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1106
1108 } else if (N.getOpcode() == ISD::ZERO_EXTEND ||
1109 N.getOpcode() == ISD::ANY_EXTEND) {
1110 EVT SrcVT = N.getOperand(0).getValueType();
1111 if (!IsLoadStore && SrcVT == MVT::i8)
1112 return AArch64_AM::UXTB;
1113 else if (!IsLoadStore && SrcVT == MVT::i16)
1114 return AArch64_AM::UXTH;
1115 else if (SrcVT == MVT::i32)
1116 return AArch64_AM::UXTW;
1117 assert(SrcVT != MVT::i64 && "extend from 64-bits?");
1118
1120 } else if (N.getOpcode() == ISD::AND) {
1121 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
1122 if (!CSD)
1124 uint64_t AndMask = CSD->getZExtValue();
1125
1126 switch (AndMask) {
1127 default:
1129 case 0xFF:
1130 return !IsLoadStore ? AArch64_AM::UXTB : AArch64_AM::InvalidShiftExtend;
1131 case 0xFFFF:
1132 return !IsLoadStore ? AArch64_AM::UXTH : AArch64_AM::InvalidShiftExtend;
1133 case 0xFFFFFFFF:
1134 return AArch64_AM::UXTW;
1135 }
1136 }
1137
1139}
1140
1141/// Determine whether constant -V is cheaper to materialise than V.
1142bool AArch64DAGToDAGISel::isWorthNegatingImm(SDValue V) const {
1143 assert(isa<ConstantSDNode>(V) && "invalid node");
1144
1145 EVT VT = V.getValueType();
1146 assert((VT == MVT::i32 || VT == MVT::i64) && "invalid type");
1147
1148 // It's only worth negating the constant if it doesn't have other uses.
1149 if (!V.hasOneUse())
1150 return false;
1151
1152 uint64_t Imm = cast<ConstantSDNode>(V)->getZExtValue();
1153 unsigned BitSize = VT.getSizeInBits();
1155 AArch64_IMM::expandMOVImm(Imm, BitSize, OrigCost);
1156 AArch64_IMM::expandMOVImm(-Imm, BitSize, NewCost);
1157 return NewCost.size() < OrigCost.size();
1158}
1159
1160/// Determine whether it is worth to fold V into an extended register of an
1161/// Add/Sub. LSL means we are folding into an `add w0, w1, w2, lsl #N`
1162/// instruction, and the shift should be treated as worth folding even if has
1163/// multiple uses.
1164bool AArch64DAGToDAGISel::isWorthFoldingALU(SDValue V, bool LSL) const {
1165 // Trivial if we are optimizing for code size or if there is only
1166 // one use of the value.
1167 if (CurDAG->shouldOptForSize() || V.hasOneUse())
1168 return true;
1169
1170 // If a subtarget has a fastpath LSL we can fold a logical shift into
1171 // the add/sub and save a cycle.
1172 if (LSL && Subtarget->hasALULSLFast() && V.getOpcode() == ISD::SHL &&
1173 V.getConstantOperandVal(1) <= 4 &&
1175 return true;
1176
1177 // It hurts otherwise, since the value will be reused.
1178 return false;
1179}
1180
1182 SDValue &Opd0, unsigned &LSB,
1183 unsigned &MSB) {
1184 // We are looking for the following pattern which basically extracts several
1185 // continuous bits from the source value and places it from the LSB of the
1186 // destination value, all other bits of the destination value or set to zero:
1187 //
1188 // Value2 = AND Value, MaskImm
1189 // SRL Value2, ShiftImm
1190 //
1191 // with MaskImm >> ShiftImm to search for the bit width.
1192 //
1193 // This gets selected into a single UBFM:
1194 //
1195 // UBFM Value, ShiftImm, Log2_64(MaskImm)
1196 //
1197
1198 if (N->getOpcode() != ISD::SRL)
1199 return false;
1200
1201 uint64_t AndMask = 0;
1202 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, AndMask))
1203 return false;
1204
1205 Opd0 = N->getOperand(0).getOperand(0);
1206
1207 uint64_t SrlImm = 0;
1208 if (!isIntImmediate(N->getOperand(1), SrlImm))
1209 return false;
1210
1211 // Check whether we really have several bits extract here.
1212 if (!isMask_64(AndMask >> SrlImm))
1213 return false;
1214
1215 Opc = N->getValueType(0) == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
1216 LSB = SrlImm;
1217 MSB = llvm::Log2_64(AndMask);
1218 return true;
1219}
1220
1221static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0,
1222 unsigned &Immr, unsigned &Imms,
1223 bool BiggerPattern) {
1224 assert((N->getOpcode() == ISD::SRA || N->getOpcode() == ISD::SRL) &&
1225 "N must be a SHR/SRA operation to call this function");
1226
1227 EVT VT = N->getValueType(0);
1228
1229 // Here we can test the type of VT and return false when the type does not
1230 // match, but since it is done prior to that call in the current context
1231 // we turned that into an assert to avoid redundant code.
1232 assert((VT == MVT::i32 || VT == MVT::i64) &&
1233 "Type checking must have been done before calling this function");
1234
1235 // Check for AND + SRL doing several bits extract.
1236 if (isSeveralBitsExtractOpFromShr(N, Opc, Opd0, Immr, Imms))
1237 return true;
1238
1239 // We're looking for a shift of a shift.
1240 uint64_t ShlImm = 0;
1241 uint64_t TruncBits = 0;
1242 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, ShlImm)) {
1243 Opd0 = N->getOperand(0).getOperand(0);
1244 } else if (VT == MVT::i32 && N->getOpcode() == ISD::SRL &&
1245 N->getOperand(0).getNode()->getOpcode() == ISD::TRUNCATE) {
1246 // We are looking for a shift of truncate. Truncate from i64 to i32 could
1247 // be considered as setting high 32 bits as zero. Our strategy here is to
1248 // always generate 64bit UBFM. This consistency will help the CSE pass
1249 // later find more redundancy.
1250 Opd0 = N->getOperand(0).getOperand(0);
1251 TruncBits = Opd0->getValueType(0).getSizeInBits() - VT.getSizeInBits();
1252 VT = Opd0.getValueType();
1253 assert(VT == MVT::i64 && "the promoted type should be i64");
1254 } else if (BiggerPattern) {
1255 // Let's pretend a 0 shift left has been performed.
1256 // FIXME: Currently we limit this to the bigger pattern case,
1257 // because some optimizations expect AND and not UBFM
1258 Opd0 = N->getOperand(0);
1259 } else
1260 return false;
1261
1262 // Missing combines/constant folding may have left us with strange
1263 // constants.
1264 if (ShlImm >= VT.getSizeInBits()) {
1265 LLVM_DEBUG(
1266 (dbgs() << N
1267 << ": Found large shift immediate, this should not happen\n"));
1268 return false;
1269 }
1270
1271 uint64_t SrlImm = 0;
1272 if (!isIntImmediate(N->getOperand(1), SrlImm))
1273 return false;
1274
1275 assert(SrlImm > 0 && SrlImm < VT.getSizeInBits() &&
1276 "bad amount in shift node!");
1277 int immr = SrlImm - ShlImm;
1278 Immr = immr < 0 ? immr + VT.getSizeInBits() : immr;
1279 Imms = VT.getSizeInBits() - ShlImm - TruncBits - 1;
1280 // SRA requires a signed extraction
1281 if (VT == MVT::i32)
1282 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMWri : AArch64::UBFMWri;
1283 else
1284 Opc = N->getOpcode() == ISD::SRA ? AArch64::SBFMXri : AArch64::UBFMXri;
1285 return true;
1286}
1287
1288/// SelectShiftedRegister - Select a "shifted register" operand. If the value
1289/// is not shifted, set the Shift operand to default of "LSL 0". The logical
1290/// instructions allow the shifted register to be rotated, but the arithmetic
1291/// instructions do not. The AllowROR parameter specifies whether ROR is
1292/// supported.
1293bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR,
1294 SDValue &Reg, SDValue &Shift) {
1295 if (SelectShiftedRegisterFromAnd(N, Reg, Shift))
1296 return true;
1297
1299 if (ShType == AArch64_AM::InvalidShiftExtend)
1300 return false;
1301 if (!AllowROR && ShType == AArch64_AM::ROR)
1302 return false;
1303
1304 // Don't match if the SRA/SRL can be folded with an SHL or AND to form a
1305 // SBFM/UBFM.
1306 if (N.getOpcode() == ISD::SRA || N.getOpcode() == ISD::SRL) {
1307 SDValue Inner = N.getOperand(0);
1308 unsigned Opc, Immr, Imms;
1309 SDValue Opd0;
1310 if ((Inner.getOpcode() == ISD::SHL ||
1311 (Inner.getOpcode() == ISD::AND &&
1312 !isBitwiseNot(Inner.getOperand(0)))) &&
1313 Inner.hasOneUse() &&
1314 isBitfieldExtractOpFromShr(N.getNode(), Opc, Opd0, Immr, Imms, false))
1315 return false;
1316 }
1317
1318 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1319 unsigned BitSize = N.getValueSizeInBits();
1320 unsigned Val = RHS->getZExtValue() & (BitSize - 1);
1321 unsigned ShVal = AArch64_AM::getShifterImm(ShType, Val);
1322
1323 Reg = N.getOperand(0);
1324 Shift = CurDAG->getTargetConstant(ShVal, SDLoc(N), MVT::i32);
1325 return isWorthFoldingALU(N, true);
1326 }
1327
1328 return false;
1329}
1330
1331/// Instructions that accept extend modifiers like UXTW expect the register
1332/// being extended to be a GPR32, but the incoming DAG might be acting on a
1333/// GPR64 (either via SEXT_INREG or AND). Extract the appropriate low bits if
1334/// this is the case.
1336 if (N.getValueType() == MVT::i32)
1337 return N;
1338
1339 SDLoc dl(N);
1340 return CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl, MVT::i32, N);
1341}
1342
1343// Returns a suitable CNT/INC/DEC/RDVL multiplier to calculate VSCALE*N.
1344template<signed Low, signed High, signed Scale>
1345bool AArch64DAGToDAGISel::SelectRDVLImm(SDValue N, SDValue &Imm) {
1346 if (!isa<ConstantSDNode>(N))
1347 return false;
1348
1349 int64_t MulImm = cast<ConstantSDNode>(N)->getSExtValue();
1350 if ((MulImm % std::abs(Scale)) == 0) {
1351 int64_t RDVLImm = MulImm / Scale;
1352 if ((RDVLImm >= Low) && (RDVLImm <= High)) {
1353 Imm = CurDAG->getSignedTargetConstant(RDVLImm, SDLoc(N), MVT::i32);
1354 return true;
1355 }
1356 }
1357
1358 return false;
1359}
1360
1361// Returns a suitable RDSVL multiplier from a left shift.
1362template <signed Low, signed High>
1363bool AArch64DAGToDAGISel::SelectRDSVLShiftImm(SDValue N, SDValue &Imm) {
1364 if (!isa<ConstantSDNode>(N))
1365 return false;
1366
1367 int64_t MulImm = 1LL << cast<ConstantSDNode>(N)->getSExtValue();
1368 if (MulImm >= Low && MulImm <= High) {
1369 Imm = CurDAG->getSignedTargetConstant(MulImm, SDLoc(N), MVT::i32);
1370 return true;
1371 }
1372
1373 return false;
1374}
1375
1376/// SelectArithExtendedRegister - Select a "extended register" operand. This
1377/// operand folds in an extend followed by an optional left shift.
1378bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg,
1379 SDValue &Shift) {
1380 unsigned ShiftVal = 0;
1382
1383 if (N.getOpcode() == ISD::SHL) {
1384 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
1385 if (!CSD)
1386 return false;
1387 ShiftVal = CSD->getZExtValue();
1388 if (ShiftVal > 4)
1389 return false;
1390
1391 Ext = getExtendTypeForNode(N.getOperand(0));
1393 return false;
1394
1395 Reg = N.getOperand(0).getOperand(0);
1396 } else {
1397 Ext = getExtendTypeForNode(N);
1399 return false;
1400
1401 // Don't match sext of vector extracts. These can use SMOV, but if we match
1402 // this as an extended register, we'll always fold the extend into an ALU op
1403 // user of the extend (which results in a UMOV).
1405 SDValue Op = N.getOperand(0);
1406 if (Op->getOpcode() == ISD::ANY_EXTEND)
1407 Op = Op->getOperand(0);
1408 if (Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
1409 Op.getOperand(0).getValueType().isFixedLengthVector())
1410 return false;
1411 }
1412
1413 Reg = N.getOperand(0);
1414
1415 // Don't match if free 32-bit -> 64-bit zext can be used instead. Use the
1416 // isDef32 as a heuristic for when the operand is likely to be a 32bit def.
1417 auto isDef32 = [](SDValue N) {
1418 unsigned Opc = N.getOpcode();
1419 return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
1422 Opc != ISD::FREEZE;
1423 };
1424 if (Ext == AArch64_AM::UXTW && Reg->getValueType(0).getSizeInBits() == 32 &&
1425 isDef32(Reg))
1426 return false;
1427 }
1428
1429 // Don't match if the sext can be folded with an asr to form an SBFX.
1430 if (Ext == AArch64_AM::SXTW && Reg.getOpcode() == ISD::SRA &&
1431 Reg.getValueType() == MVT::i32 &&
1432 isa<ConstantSDNode>(Reg.getOperand(1)) && Reg.hasOneUse())
1433 return false;
1434
1435 // AArch64 mandates that the RHS of the operation must use the smallest
1436 // register class that could contain the size being extended from. Thus,
1437 // if we're folding a (sext i8), we need the RHS to be a GPR32, even though
1438 // there might not be an actual 32-bit value in the program. We can
1439 // (harmlessly) synthesize one by injected an EXTRACT_SUBREG here.
1440 assert(Ext != AArch64_AM::UXTX && Ext != AArch64_AM::SXTX);
1441 Reg = narrowIfNeeded(CurDAG, Reg);
1442 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
1443 MVT::i32);
1444 return isWorthFoldingALU(N);
1445}
1446
1447/// SelectArithUXTXRegister - Select a "UXTX register" operand. This
1448/// operand is referred by the instructions have SP operand
1449bool AArch64DAGToDAGISel::SelectArithUXTXRegister(SDValue N, SDValue &Reg,
1450 SDValue &Shift) {
1451 unsigned ShiftVal = 0;
1453
1454 if (N.getOpcode() != ISD::SHL)
1455 return false;
1456
1457 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
1458 if (!CSD)
1459 return false;
1460 ShiftVal = CSD->getZExtValue();
1461 if (ShiftVal > 4)
1462 return false;
1463
1464 Ext = AArch64_AM::UXTX;
1465 Reg = N.getOperand(0);
1466 Shift = CurDAG->getTargetConstant(getArithExtendImm(Ext, ShiftVal), SDLoc(N),
1467 MVT::i32);
1468 return isWorthFoldingALU(N);
1469}
1470
1471/// If there's a use of this ADDlow that's not itself a load/store then we'll
1472/// need to create a real ADD instruction from it anyway and there's no point in
1473/// folding it into the mem op. Theoretically, it shouldn't matter, but there's
1474/// a single pseudo-instruction for an ADRP/ADD pair so over-aggressive folding
1475/// leads to duplicated ADRP instructions.
1477 for (auto *User : N->users()) {
1478 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
1479 User->getOpcode() != ISD::ATOMIC_LOAD &&
1480 User->getOpcode() != ISD::ATOMIC_STORE)
1481 return false;
1482
1483 // ldar and stlr have much more restrictive addressing modes (just a
1484 // register).
1485 if (isStrongerThanMonotonic(cast<MemSDNode>(User)->getSuccessOrdering()))
1486 return false;
1487 }
1488
1489 return true;
1490}
1491
1492/// Check if the immediate offset is valid as a scaled immediate.
1493static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range,
1494 unsigned Size) {
1495 if ((Offset & (Size - 1)) == 0 && Offset >= 0 &&
1496 Offset < (Range << Log2_32(Size)))
1497 return true;
1498 return false;
1499}
1500
1501/// SelectAddrModeIndexedBitWidth - Select a "register plus scaled (un)signed BW-bit
1502/// immediate" address. The "Size" argument is the size in bytes of the memory
1503/// reference, which determines the scale.
1504bool AArch64DAGToDAGISel::SelectAddrModeIndexedBitWidth(SDValue N, bool IsSignedImm,
1505 unsigned BW, unsigned Size,
1506 SDValue &Base,
1507 SDValue &OffImm) {
1508 SDLoc dl(N);
1509 const DataLayout &DL = CurDAG->getDataLayout();
1510 const TargetLowering *TLI = getTargetLowering();
1511 if (N.getOpcode() == ISD::FrameIndex) {
1512 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1513 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1514 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1515 return true;
1516 }
1517
1518 // As opposed to the (12-bit) Indexed addressing mode below, the 7/9-bit signed
1519 // selected here doesn't support labels/immediates, only base+offset.
1520 if (CurDAG->isBaseWithConstantOffset(N)) {
1521 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1522 if (IsSignedImm) {
1523 int64_t RHSC = RHS->getSExtValue();
1524 unsigned Scale = Log2_32(Size);
1525 int64_t Range = 0x1LL << (BW - 1);
1526
1527 if ((RHSC & (Size - 1)) == 0 && RHSC >= -(Range << Scale) &&
1528 RHSC < (Range << Scale)) {
1529 Base = N.getOperand(0);
1530 if (Base.getOpcode() == ISD::FrameIndex) {
1531 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1532 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1533 }
1534 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1535 return true;
1536 }
1537 } else {
1538 // unsigned Immediate
1539 uint64_t RHSC = RHS->getZExtValue();
1540 unsigned Scale = Log2_32(Size);
1541 uint64_t Range = 0x1ULL << BW;
1542
1543 if ((RHSC & (Size - 1)) == 0 && RHSC < (Range << Scale)) {
1544 Base = N.getOperand(0);
1545 if (Base.getOpcode() == ISD::FrameIndex) {
1546 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1547 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1548 }
1549 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1550 return true;
1551 }
1552 }
1553 }
1554 }
1555 // Base only. The address will be materialized into a register before
1556 // the memory is accessed.
1557 // add x0, Xbase, #offset
1558 // stp x1, x2, [x0]
1559 Base = N;
1560 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1561 return true;
1562}
1563
1564/// SelectAddrModeIndexed - Select a "register plus scaled unsigned 12-bit
1565/// immediate" address. The "Size" argument is the size in bytes of the memory
1566/// reference, which determines the scale.
1567bool AArch64DAGToDAGISel::SelectAddrModeIndexed(SDValue N, unsigned Size,
1568 SDValue &Base, SDValue &OffImm) {
1569 SDLoc dl(N);
1570 const DataLayout &DL = CurDAG->getDataLayout();
1571 const TargetLowering *TLI = getTargetLowering();
1572 if (N.getOpcode() == ISD::FrameIndex) {
1573 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1574 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1575 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1576 return true;
1577 }
1578
1579 if (N.getOpcode() == AArch64ISD::ADDlow && isWorthFoldingADDlow(N)) {
1580 GlobalAddressSDNode *GAN =
1581 dyn_cast<GlobalAddressSDNode>(N.getOperand(1).getNode());
1582 Base = N.getOperand(0);
1583 OffImm = N.getOperand(1);
1584 if (!GAN)
1585 return true;
1586
1587 if (GAN->getOffset() % Size == 0 &&
1589 return true;
1590 }
1591
1592 if (CurDAG->isBaseWithConstantOffset(N)) {
1593 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1594 int64_t RHSC = (int64_t)RHS->getZExtValue();
1595 unsigned Scale = Log2_32(Size);
1596 if (isValidAsScaledImmediate(RHSC, 0x1000, Size)) {
1597 Base = N.getOperand(0);
1598 if (Base.getOpcode() == ISD::FrameIndex) {
1599 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1600 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
1601 }
1602 OffImm = CurDAG->getTargetConstant(RHSC >> Scale, dl, MVT::i64);
1603 return true;
1604 }
1605 }
1606 }
1607
1608 // Before falling back to our general case, check if the unscaled
1609 // instructions can handle this. If so, that's preferable.
1610 if (SelectAddrModeUnscaled(N, Size, Base, OffImm))
1611 return false;
1612
1613 // Base only. The address will be materialized into a register before
1614 // the memory is accessed.
1615 // add x0, Xbase, #offset
1616 // ldr x0, [x0]
1617 Base = N;
1618 OffImm = CurDAG->getTargetConstant(0, dl, MVT::i64);
1619 return true;
1620}
1621
1622/// SelectAddrModeUnscaled - Select a "register plus unscaled signed 9-bit
1623/// immediate" address. This should only match when there is an offset that
1624/// is not valid for a scaled immediate addressing mode. The "Size" argument
1625/// is the size in bytes of the memory reference, which is needed here to know
1626/// what is valid for a scaled immediate.
1627bool AArch64DAGToDAGISel::SelectAddrModeUnscaled(SDValue N, unsigned Size,
1628 SDValue &Base,
1629 SDValue &OffImm) {
1630 if (!CurDAG->isBaseWithConstantOffset(N))
1631 return false;
1632 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1633 int64_t RHSC = RHS->getSExtValue();
1634 if (RHSC >= -256 && RHSC < 256) {
1635 Base = N.getOperand(0);
1636 if (Base.getOpcode() == ISD::FrameIndex) {
1637 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1638 const TargetLowering *TLI = getTargetLowering();
1639 Base = CurDAG->getTargetFrameIndex(
1640 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1641 }
1642 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i64);
1643 return true;
1644 }
1645 }
1646 return false;
1647}
1648
1650 SDLoc dl(N);
1651 SDValue ImpDef = SDValue(
1652 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, MVT::i64), 0);
1653 return CurDAG->getTargetInsertSubreg(AArch64::sub_32, dl, MVT::i64, ImpDef,
1654 N);
1655}
1656
1657/// Check if the given SHL node (\p N), can be used to form an
1658/// extended register for an addressing mode.
1659bool AArch64DAGToDAGISel::SelectExtendedSHL(SDValue N, unsigned Size,
1660 bool WantExtend, SDValue &Offset,
1661 SDValue &SignExtend) {
1662 assert(N.getOpcode() == ISD::SHL && "Invalid opcode.");
1663 ConstantSDNode *CSD = dyn_cast<ConstantSDNode>(N.getOperand(1));
1664 if (!CSD || (CSD->getZExtValue() & 0x7) != CSD->getZExtValue())
1665 return false;
1666
1667 SDLoc dl(N);
1668 if (WantExtend) {
1670 getExtendTypeForNode(N.getOperand(0), true);
1672 return false;
1673
1674 Offset = narrowIfNeeded(CurDAG, N.getOperand(0).getOperand(0));
1675 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1676 MVT::i32);
1677 } else {
1678 Offset = N.getOperand(0);
1679 SignExtend = CurDAG->getTargetConstant(0, dl, MVT::i32);
1680 }
1681
1682 unsigned LegalShiftVal = Log2_32(Size);
1683 unsigned ShiftVal = CSD->getZExtValue();
1684
1685 if (ShiftVal != 0 && ShiftVal != LegalShiftVal)
1686 return false;
1687
1688 return isWorthFoldingAddr(N, Size);
1689}
1690
1691bool AArch64DAGToDAGISel::SelectAddrModeWRO(SDValue N, unsigned Size,
1693 SDValue &SignExtend,
1694 SDValue &DoShift) {
1695 if (N.getOpcode() != ISD::ADD)
1696 return false;
1697 SDValue LHS = N.getOperand(0);
1698 SDValue RHS = N.getOperand(1);
1699 SDLoc dl(N);
1700
1701 // We don't want to match immediate adds here, because they are better lowered
1702 // to the register-immediate addressing modes.
1704 return false;
1705
1706 // Check if this particular node is reused in any non-memory related
1707 // operation. If yes, do not try to fold this node into the address
1708 // computation, since the computation will be kept.
1709 const SDNode *Node = N.getNode();
1710 for (SDNode *UI : Node->users()) {
1711 if (!isMemOpOrPrefetch(UI))
1712 return false;
1713 }
1714
1715 // Remember if it is worth folding N when it produces extended register.
1716 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(N, Size);
1717
1718 // Try to match a shifted extend on the RHS.
1719 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1720 SelectExtendedSHL(RHS, Size, true, Offset, SignExtend)) {
1721 Base = LHS;
1722 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1723 return true;
1724 }
1725
1726 // Try to match a shifted extend on the LHS.
1727 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1728 SelectExtendedSHL(LHS, Size, true, Offset, SignExtend)) {
1729 Base = RHS;
1730 DoShift = CurDAG->getTargetConstant(true, dl, MVT::i32);
1731 return true;
1732 }
1733
1734 // There was no shift, whatever else we find.
1735 DoShift = CurDAG->getTargetConstant(false, dl, MVT::i32);
1736
1738 // Try to match an unshifted extend on the LHS.
1739 if (IsExtendedRegisterWorthFolding &&
1740 (Ext = getExtendTypeForNode(LHS, true)) !=
1742 Base = RHS;
1743 Offset = narrowIfNeeded(CurDAG, LHS.getOperand(0));
1744 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1745 MVT::i32);
1746 if (isWorthFoldingAddr(LHS, Size))
1747 return true;
1748 }
1749
1750 // Try to match an unshifted extend on the RHS.
1751 if (IsExtendedRegisterWorthFolding &&
1752 (Ext = getExtendTypeForNode(RHS, true)) !=
1754 Base = LHS;
1755 Offset = narrowIfNeeded(CurDAG, RHS.getOperand(0));
1756 SignExtend = CurDAG->getTargetConstant(Ext == AArch64_AM::SXTW, dl,
1757 MVT::i32);
1758 if (isWorthFoldingAddr(RHS, Size))
1759 return true;
1760 }
1761
1762 return false;
1763}
1764
1765// Check if the given immediate is preferred by ADD. If an immediate can be
1766// encoded in an ADD, or it can be encoded in an "ADD LSL #12" and can not be
1767// encoded by one MOVZ, return true.
1768static bool isPreferredADD(int64_t ImmOff) {
1769 // Constant in [0x0, 0xfff] can be encoded in ADD.
1770 if ((ImmOff & 0xfffffffffffff000LL) == 0x0LL)
1771 return true;
1772 // Check if it can be encoded in an "ADD LSL #12".
1773 if ((ImmOff & 0xffffffffff000fffLL) == 0x0LL)
1774 // As a single MOVZ is faster than a "ADD of LSL #12", ignore such constant.
1775 return (ImmOff & 0xffffffffff00ffffLL) != 0x0LL &&
1776 (ImmOff & 0xffffffffffff0fffLL) != 0x0LL;
1777 return false;
1778}
1779
1780bool AArch64DAGToDAGISel::SelectAddrModeXRO(SDValue N, unsigned Size,
1782 SDValue &SignExtend,
1783 SDValue &DoShift) {
1784 if (N.getOpcode() != ISD::ADD)
1785 return false;
1786 SDValue LHS = N.getOperand(0);
1787 SDValue RHS = N.getOperand(1);
1788 SDLoc DL(N);
1789
1790 // Check if this particular node is reused in any non-memory related
1791 // operation. If yes, do not try to fold this node into the address
1792 // computation, since the computation will be kept.
1793 const SDNode *Node = N.getNode();
1794 for (SDNode *UI : Node->users()) {
1795 if (!isMemOpOrPrefetch(UI))
1796 return false;
1797 }
1798
1799 // Watch out if RHS is a wide immediate, it can not be selected into
1800 // [BaseReg+Imm] addressing mode. Also it may not be able to be encoded into
1801 // ADD/SUB. Instead it will use [BaseReg + 0] address mode and generate
1802 // instructions like:
1803 // MOV X0, WideImmediate
1804 // ADD X1, BaseReg, X0
1805 // LDR X2, [X1, 0]
1806 // For such situation, using [BaseReg, XReg] addressing mode can save one
1807 // ADD/SUB:
1808 // MOV X0, WideImmediate
1809 // LDR X2, [BaseReg, X0]
1810 if (isa<ConstantSDNode>(RHS)) {
1811 int64_t ImmOff = (int64_t)RHS->getAsZExtVal();
1812 // Skip the immediate can be selected by load/store addressing mode.
1813 // Also skip the immediate can be encoded by a single ADD (SUB is also
1814 // checked by using -ImmOff).
1815 if (isValidAsScaledImmediate(ImmOff, 0x1000, Size) ||
1816 isPreferredADD(ImmOff) || isPreferredADD(-ImmOff))
1817 return false;
1818
1819 SDValue Ops[] = { RHS };
1820 SDNode *MOVI =
1821 CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
1822 SDValue MOVIV = SDValue(MOVI, 0);
1823 // This ADD of two X register will be selected into [Reg+Reg] mode.
1824 N = CurDAG->getNode(ISD::ADD, DL, MVT::i64, LHS, MOVIV);
1825 }
1826
1827 // Remember if it is worth folding N when it produces extended register.
1828 bool IsExtendedRegisterWorthFolding = isWorthFoldingAddr(N, Size);
1829
1830 // Try to match a shifted extend on the RHS.
1831 if (IsExtendedRegisterWorthFolding && RHS.getOpcode() == ISD::SHL &&
1832 SelectExtendedSHL(RHS, Size, false, Offset, SignExtend)) {
1833 Base = LHS;
1834 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1835 return true;
1836 }
1837
1838 // Try to match a shifted extend on the LHS.
1839 if (IsExtendedRegisterWorthFolding && LHS.getOpcode() == ISD::SHL &&
1840 SelectExtendedSHL(LHS, Size, false, Offset, SignExtend)) {
1841 Base = RHS;
1842 DoShift = CurDAG->getTargetConstant(true, DL, MVT::i32);
1843 return true;
1844 }
1845
1846 // Match any non-shifted, non-extend, non-immediate add expression.
1847 Base = LHS;
1848 Offset = RHS;
1849 SignExtend = CurDAG->getTargetConstant(false, DL, MVT::i32);
1850 DoShift = CurDAG->getTargetConstant(false, DL, MVT::i32);
1851 // Reg1 + Reg2 is free: no check needed.
1852 return true;
1853}
1854
1855SDValue AArch64DAGToDAGISel::createDTuple(ArrayRef<SDValue> Regs) {
1856 static const unsigned RegClassIDs[] = {
1857 AArch64::DDRegClassID, AArch64::DDDRegClassID, AArch64::DDDDRegClassID};
1858 static const unsigned SubRegs[] = {AArch64::dsub0, AArch64::dsub1,
1859 AArch64::dsub2, AArch64::dsub3};
1860
1861 return createTuple(Regs, RegClassIDs, SubRegs);
1862}
1863
1864SDValue AArch64DAGToDAGISel::createQTuple(ArrayRef<SDValue> Regs) {
1865 static const unsigned RegClassIDs[] = {
1866 AArch64::QQRegClassID, AArch64::QQQRegClassID, AArch64::QQQQRegClassID};
1867 static const unsigned SubRegs[] = {AArch64::qsub0, AArch64::qsub1,
1868 AArch64::qsub2, AArch64::qsub3};
1869
1870 return createTuple(Regs, RegClassIDs, SubRegs);
1871}
1872
1873SDValue AArch64DAGToDAGISel::createZTuple(ArrayRef<SDValue> Regs) {
1874 static const unsigned RegClassIDs[] = {AArch64::ZPR2RegClassID,
1875 AArch64::ZPR3RegClassID,
1876 AArch64::ZPR4RegClassID};
1877 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1878 AArch64::zsub2, AArch64::zsub3};
1879
1880 return createTuple(Regs, RegClassIDs, SubRegs);
1881}
1882
1883SDValue AArch64DAGToDAGISel::createZMulTuple(ArrayRef<SDValue> Regs) {
1884 assert(Regs.size() == 2 || Regs.size() == 4);
1885
1886 // The createTuple interface requires 3 RegClassIDs for each possible
1887 // tuple type even though we only have them for ZPR2 and ZPR4.
1888 static const unsigned RegClassIDs[] = {AArch64::ZPR2Mul2RegClassID, 0,
1889 AArch64::ZPR4Mul4RegClassID};
1890 static const unsigned SubRegs[] = {AArch64::zsub0, AArch64::zsub1,
1891 AArch64::zsub2, AArch64::zsub3};
1892 return createTuple(Regs, RegClassIDs, SubRegs);
1893}
1894
1895SDValue AArch64DAGToDAGISel::createTuple(ArrayRef<SDValue> Regs,
1896 const unsigned RegClassIDs[],
1897 const unsigned SubRegs[]) {
1898 // There's no special register-class for a vector-list of 1 element: it's just
1899 // a vector.
1900 if (Regs.size() == 1)
1901 return Regs[0];
1902
1903 assert(Regs.size() >= 2 && Regs.size() <= 4);
1904
1905 SDLoc DL(Regs[0]);
1906
1908
1909 // First operand of REG_SEQUENCE is the desired RegClass.
1910 Ops.push_back(
1911 CurDAG->getTargetConstant(RegClassIDs[Regs.size() - 2], DL, MVT::i32));
1912
1913 // Then we get pairs of source & subregister-position for the components.
1914 for (unsigned i = 0; i < Regs.size(); ++i) {
1915 Ops.push_back(Regs[i]);
1916 Ops.push_back(CurDAG->getTargetConstant(SubRegs[i], DL, MVT::i32));
1917 }
1918
1919 SDNode *N =
1920 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
1921 return SDValue(N, 0);
1922}
1923
1924void AArch64DAGToDAGISel::SelectTable(SDNode *N, unsigned NumVecs, unsigned Opc,
1925 bool isExt) {
1926 SDLoc dl(N);
1927 EVT VT = N->getValueType(0);
1928
1929 unsigned ExtOff = isExt;
1930
1931 // Form a REG_SEQUENCE to force register allocation.
1932 unsigned Vec0Off = ExtOff + 1;
1933 SmallVector<SDValue, 4> Regs(N->ops().slice(Vec0Off, NumVecs));
1934 SDValue RegSeq = createQTuple(Regs);
1935
1937 if (isExt)
1938 Ops.push_back(N->getOperand(1));
1939 Ops.push_back(RegSeq);
1940 Ops.push_back(N->getOperand(NumVecs + ExtOff + 1));
1941 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
1942}
1943
1944static std::tuple<SDValue, SDValue>
1946 SDLoc DL(Disc);
1947 SDValue AddrDisc;
1948 SDValue ConstDisc;
1949
1950 // If this is a blend, remember the constant and address discriminators.
1951 // Otherwise, it's either a constant discriminator, or a non-blended
1952 // address discriminator.
1953 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
1954 Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
1955 AddrDisc = Disc->getOperand(1);
1956 ConstDisc = Disc->getOperand(2);
1957 } else {
1958 ConstDisc = Disc;
1959 }
1960
1961 // If the constant discriminator (either the blend RHS, or the entire
1962 // discriminator value) isn't a 16-bit constant, bail out, and let the
1963 // discriminator be computed separately.
1964 auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
1965 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
1966 return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
1967
1968 // If there's no address discriminator, use XZR directly.
1969 if (!AddrDisc)
1970 AddrDisc = DAG->getRegister(AArch64::XZR, MVT::i64);
1971
1972 return std::make_tuple(
1973 DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
1974 AddrDisc);
1975}
1976
1977void AArch64DAGToDAGISel::SelectPtrauthAuth(SDNode *N) {
1978 SDLoc DL(N);
1979 // IntrinsicID is operand #0
1980 SDValue Val = N->getOperand(1);
1981 SDValue AUTKey = N->getOperand(2);
1982 SDValue AUTDisc = N->getOperand(3);
1983
1984 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
1985 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
1986
1987 SDValue AUTAddrDisc, AUTConstDisc;
1988 std::tie(AUTConstDisc, AUTAddrDisc) =
1989 extractPtrauthBlendDiscriminators(AUTDisc, CurDAG);
1990
1991 if (!Subtarget->isX16X17Safer()) {
1992 std::vector<SDValue> Ops = {Val, AUTKey, AUTConstDisc, AUTAddrDisc};
1993 // Copy deactivation symbol if present.
1994 if (N->getNumOperands() > 4)
1995 Ops.push_back(N->getOperand(4));
1996
1997 SDNode *AUT =
1998 CurDAG->getMachineNode(AArch64::AUTxMxN, DL, MVT::i64, MVT::i64, Ops);
1999 ReplaceNode(N, AUT);
2000 } else {
2001 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
2002 AArch64::X16, Val, SDValue());
2003 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, X16Copy.getValue(1)};
2004
2005 SDNode *AUT = CurDAG->getMachineNode(AArch64::AUTx16x17, DL, MVT::i64, Ops);
2006 ReplaceNode(N, AUT);
2007 }
2008}
2009
2010void AArch64DAGToDAGISel::SelectPtrauthResign(SDNode *N) {
2011 SDLoc DL(N);
2012 // IntrinsicID is operand #0, if W_CHAIN it is #1
2013 int OffsetBase = N->getOpcode() == ISD::INTRINSIC_W_CHAIN ? 1 : 0;
2014 SDValue Val = N->getOperand(OffsetBase + 1);
2015 SDValue AUTKey = N->getOperand(OffsetBase + 2);
2016 SDValue AUTDisc = N->getOperand(OffsetBase + 3);
2017 SDValue PACKey = N->getOperand(OffsetBase + 4);
2018 SDValue PACDisc = N->getOperand(OffsetBase + 5);
2019 uint32_t IntNum = N->getConstantOperandVal(OffsetBase + 0);
2020 bool HasLoad = IntNum == Intrinsic::ptrauth_resign_load_relative;
2021
2022 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
2023 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
2024
2025 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
2026 PACKey = CurDAG->getTargetConstant(PACKeyC, DL, MVT::i64);
2027
2028 SDValue AUTAddrDisc, AUTConstDisc;
2029 std::tie(AUTConstDisc, AUTAddrDisc) =
2030 extractPtrauthBlendDiscriminators(AUTDisc, CurDAG);
2031
2032 SDValue PACAddrDisc, PACConstDisc;
2033 std::tie(PACConstDisc, PACAddrDisc) =
2034 extractPtrauthBlendDiscriminators(PACDisc, CurDAG);
2035
2036 SDValue X16Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
2037 AArch64::X16, Val, SDValue());
2038
2039 if (HasLoad) {
2040 SDValue Addend = N->getOperand(OffsetBase + 6);
2041 SDValue IncomingChain = N->getOperand(0);
2042 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc,
2043 PACKey, PACConstDisc, PACAddrDisc,
2044 Addend, IncomingChain, X16Copy.getValue(1)};
2045
2046 SDNode *AUTRELLOADPAC = CurDAG->getMachineNode(AArch64::AUTRELLOADPAC, DL,
2047 MVT::i64, MVT::Other, Ops);
2048 ReplaceNode(N, AUTRELLOADPAC);
2049 } else {
2050 SDValue Ops[] = {AUTKey, AUTConstDisc, AUTAddrDisc, PACKey,
2051 PACConstDisc, PACAddrDisc, X16Copy.getValue(1)};
2052
2053 SDNode *AUTPAC = CurDAG->getMachineNode(AArch64::AUTPAC, DL, MVT::i64, Ops);
2054 ReplaceNode(N, AUTPAC);
2055 }
2056}
2057
2058void AArch64DAGToDAGISel::SelectPtrauthResignWithPC(SDNode *N) {
2059 SDLoc DL(N);
2060 SDValue Val = N->getOperand(1);
2061 SDValue AUTKey = N->getOperand(2);
2062 SDValue AUTDisc = N->getOperand(3);
2063 SDValue AUTPC = N->getOperand(4);
2064 SDValue PACKey = N->getOperand(5);
2065 SDValue PACDisc = N->getOperand(6);
2066
2067 unsigned AUTKeyC = cast<ConstantSDNode>(AUTKey)->getZExtValue();
2068 unsigned PACKeyC = cast<ConstantSDNode>(PACKey)->getZExtValue();
2069
2070 AUTKey = CurDAG->getTargetConstant(AUTKeyC, DL, MVT::i64);
2071 PACKey = CurDAG->getTargetConstant(PACKeyC, DL, MVT::i64);
2072
2073 SDValue PACAddrDisc, PACConstDisc;
2074 std::tie(PACConstDisc, PACAddrDisc) =
2075 extractPtrauthBlendDiscriminators(PACDisc, CurDAG);
2076
2077 SDValue X17Copy = CurDAG->getCopyToReg(CurDAG->getEntryNode(), DL,
2078 AArch64::X17, Val, SDValue());
2079 SDValue X16Copy = CurDAG->getCopyToReg(
2080 CurDAG->getEntryNode(), DL, AArch64::X16, AUTDisc, X17Copy.getValue(1));
2081 SDValue X15Copy = CurDAG->getCopyToReg(
2082 CurDAG->getEntryNode(), DL, AArch64::X15, AUTPC, X16Copy.getValue(1));
2083
2084 SDValue Ops[] = {AUTKey, PACKey, PACConstDisc, PACAddrDisc,
2085 X15Copy.getValue(1)};
2086 SDNode *AUTPCPAC =
2087 CurDAG->getMachineNode(AArch64::AUTPCPAC, DL, MVT::i64, Ops);
2088 ReplaceNode(N, AUTPCPAC);
2089}
2090
2091bool AArch64DAGToDAGISel::tryIndexedLoad(SDNode *N) {
2092 LoadSDNode *LD = cast<LoadSDNode>(N);
2093 if (LD->isUnindexed())
2094 return false;
2095 EVT VT = LD->getMemoryVT();
2096 EVT DstVT = N->getValueType(0);
2097 ISD::MemIndexedMode AM = LD->getAddressingMode();
2098 bool IsPre = AM == ISD::PRE_INC || AM == ISD::PRE_DEC;
2099 ConstantSDNode *OffsetOp = cast<ConstantSDNode>(LD->getOffset());
2100 int OffsetVal = (int)OffsetOp->getZExtValue();
2101
2102 // We're not doing validity checking here. That was done when checking
2103 // if we should mark the load as indexed or not. We're just selecting
2104 // the right instruction.
2105 unsigned Opcode = 0;
2106
2107 ISD::LoadExtType ExtType = LD->getExtensionType();
2108 bool InsertTo64 = false;
2109 bool UseLd1 =
2110 (VT.is64BitVector() || VT.is128BitVector()) &&
2111 (!Subtarget->isLittleEndian() || (Subtarget->requiresStrictAlign() &&
2112 LD->getAlign() < VT.getStoreSize()));
2113 if (VT == MVT::i64)
2114 Opcode = IsPre ? AArch64::LDRXpre : AArch64::LDRXpost;
2115 else if (VT == MVT::i32) {
2116 if (ExtType == ISD::NON_EXTLOAD)
2117 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2118 else if (ExtType == ISD::SEXTLOAD)
2119 Opcode = IsPre ? AArch64::LDRSWpre : AArch64::LDRSWpost;
2120 else {
2121 Opcode = IsPre ? AArch64::LDRWpre : AArch64::LDRWpost;
2122 InsertTo64 = true;
2123 // The result of the load is only i32. It's the subreg_to_reg that makes
2124 // it into an i64.
2125 DstVT = MVT::i32;
2126 }
2127 } else if (VT == MVT::i16) {
2128 if (ExtType == ISD::SEXTLOAD) {
2129 if (DstVT == MVT::i64)
2130 Opcode = IsPre ? AArch64::LDRSHXpre : AArch64::LDRSHXpost;
2131 else
2132 Opcode = IsPre ? AArch64::LDRSHWpre : AArch64::LDRSHWpost;
2133 } else {
2134 Opcode = IsPre ? AArch64::LDRHHpre : AArch64::LDRHHpost;
2135 InsertTo64 = DstVT == MVT::i64;
2136 // The result of the load is only i32. It's the subreg_to_reg that makes
2137 // it into an i64.
2138 DstVT = MVT::i32;
2139 }
2140 } else if (VT == MVT::i8) {
2141 if (ExtType == ISD::SEXTLOAD) {
2142 if (DstVT == MVT::i64)
2143 Opcode = IsPre ? AArch64::LDRSBXpre : AArch64::LDRSBXpost;
2144 else
2145 Opcode = IsPre ? AArch64::LDRSBWpre : AArch64::LDRSBWpost;
2146 } else {
2147 Opcode = IsPre ? AArch64::LDRBBpre : AArch64::LDRBBpost;
2148 InsertTo64 = DstVT == MVT::i64;
2149 // The result of the load is only i32. It's the subreg_to_reg that makes
2150 // it into an i64.
2151 DstVT = MVT::i32;
2152 }
2153 } else if (VT == MVT::f16) {
2154 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2155 } else if (VT == MVT::bf16) {
2156 Opcode = IsPre ? AArch64::LDRHpre : AArch64::LDRHpost;
2157 } else if (VT == MVT::f32) {
2158 Opcode = IsPre ? AArch64::LDRSpre : AArch64::LDRSpost;
2159 } else if (VT == MVT::f64 || (VT.is64BitVector() && !UseLd1)) {
2160 Opcode = IsPre ? AArch64::LDRDpre : AArch64::LDRDpost;
2161 } else if (VT.is128BitVector() && !UseLd1) {
2162 Opcode = IsPre ? AArch64::LDRQpre : AArch64::LDRQpost;
2163 } else if (VT.is64BitVector() && UseLd1) {
2164 if (IsPre || OffsetVal != 8)
2165 return false;
2166 switch (VT.getScalarSizeInBits()) {
2167 case 8:
2168 Opcode = AArch64::LD1Onev8b_POST;
2169 break;
2170 case 16:
2171 Opcode = AArch64::LD1Onev4h_POST;
2172 break;
2173 case 32:
2174 Opcode = AArch64::LD1Onev2s_POST;
2175 break;
2176 case 64:
2177 Opcode = AArch64::LD1Onev1d_POST;
2178 break;
2179 default:
2180 llvm_unreachable("Expected vector element to be a power of 2");
2181 }
2182 } else if (VT.is128BitVector() && UseLd1) {
2183 if (IsPre || OffsetVal != 16)
2184 return false;
2185 switch (VT.getScalarSizeInBits()) {
2186 case 8:
2187 Opcode = AArch64::LD1Onev16b_POST;
2188 break;
2189 case 16:
2190 Opcode = AArch64::LD1Onev8h_POST;
2191 break;
2192 case 32:
2193 Opcode = AArch64::LD1Onev4s_POST;
2194 break;
2195 case 64:
2196 Opcode = AArch64::LD1Onev2d_POST;
2197 break;
2198 default:
2199 llvm_unreachable("Expected vector element to be a power of 2");
2200 }
2201 } else
2202 return false;
2203 SDValue Chain = LD->getChain();
2204 SDValue Base = LD->getBasePtr();
2205 SDLoc dl(N);
2206 // LD1 encodes an immediate offset by using XZR as the offset register.
2207 SDValue Offset = UseLd1 ? CurDAG->getRegister(AArch64::XZR, MVT::i64)
2208 : CurDAG->getTargetConstant(OffsetVal, dl, MVT::i64);
2209 SDValue Ops[] = { Base, Offset, Chain };
2210 SDNode *Res = CurDAG->getMachineNode(Opcode, dl, MVT::i64, DstVT,
2211 MVT::Other, Ops);
2212
2213 // Transfer memoperands.
2214 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2215 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Res), {MemOp});
2216
2217 // Either way, we're replacing the node, so tell the caller that.
2218 SDValue LoadedVal = SDValue(Res, 1);
2219 if (InsertTo64) {
2220 SDValue SubReg = CurDAG->getTargetConstant(AArch64::sub_32, dl, MVT::i32);
2221 LoadedVal = SDValue(CurDAG->getMachineNode(AArch64::SUBREG_TO_REG, dl,
2222 MVT::i64, LoadedVal, SubReg),
2223 0);
2224 }
2225
2226 ReplaceUses(SDValue(N, 0), LoadedVal);
2227 ReplaceUses(SDValue(N, 1), SDValue(Res, 0));
2228 ReplaceUses(SDValue(N, 2), SDValue(Res, 2));
2229 CurDAG->RemoveDeadNode(N);
2230 return true;
2231}
2232
2233void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc,
2234 unsigned SubRegIdx) {
2235 SDLoc dl(N);
2236 EVT VT = N->getValueType(0);
2237 SDValue Chain = N->getOperand(0);
2238
2239 SDValue Ops[] = {N->getOperand(2), // Mem operand;
2240 Chain};
2241
2242 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2243
2244 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2245 SDValue SuperReg = SDValue(Ld, 0);
2246 for (unsigned i = 0; i < NumVecs; ++i)
2247 ReplaceUses(SDValue(N, i),
2248 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
2249
2250 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
2251
2252 // Transfer memoperands. In the case of AArch64::LD64B, there won't be one,
2253 // because it's too simple to have needed special treatment during lowering.
2254 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(N)) {
2255 MachineMemOperand *MemOp = MemIntr->getMemOperand();
2256 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
2257 }
2258
2259 CurDAG->RemoveDeadNode(N);
2260}
2261
2262void AArch64DAGToDAGISel::SelectPostLoad(SDNode *N, unsigned NumVecs,
2263 unsigned Opc, unsigned SubRegIdx) {
2264 SDLoc dl(N);
2265 EVT VT = N->getValueType(0);
2266 SDValue Chain = N->getOperand(0);
2267
2268 SDValue Ops[] = {N->getOperand(1), // Mem operand
2269 N->getOperand(2), // Incremental
2270 Chain};
2271
2272 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2273 MVT::Untyped, MVT::Other};
2274
2275 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2276
2277 // Update uses of write back register
2278 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
2279
2280 // Update uses of vector list
2281 SDValue SuperReg = SDValue(Ld, 1);
2282 if (NumVecs == 1)
2283 ReplaceUses(SDValue(N, 0), SuperReg);
2284 else
2285 for (unsigned i = 0; i < NumVecs; ++i)
2286 ReplaceUses(SDValue(N, i),
2287 CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg));
2288
2289 // Transfer memoperands.
2290 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2291 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
2292
2293 // Update the chain
2294 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
2295 CurDAG->RemoveDeadNode(N);
2296}
2297
2298/// Optimize \param OldBase and \param OldOffset selecting the best addressing
2299/// mode. Returns a tuple consisting of an Opcode, an SDValue representing the
2300/// new Base and an SDValue representing the new offset.
2301std::tuple<unsigned, SDValue, SDValue>
2302AArch64DAGToDAGISel::findAddrModeSVELoadStore(SDNode *N, unsigned Opc_rr,
2303 unsigned Opc_ri,
2304 const SDValue &OldBase,
2305 const SDValue &OldOffset,
2306 unsigned Scale) {
2307 SDValue NewBase = OldBase;
2308 SDValue NewOffset = OldOffset;
2309 // Detect a possible Reg+Imm addressing mode.
2310 const bool IsRegImm = SelectAddrModeIndexedSVE</*Min=*/-8, /*Max=*/7>(
2311 N, OldBase, NewBase, NewOffset);
2312
2313 // Detect a possible reg+reg addressing mode, but only if we haven't already
2314 // detected a Reg+Imm one.
2315 const bool IsRegReg =
2316 !IsRegImm && SelectSVERegRegAddrMode(OldBase, Scale, NewBase, NewOffset);
2317
2318 // Select the instruction.
2319 return std::make_tuple(IsRegReg ? Opc_rr : Opc_ri, NewBase, NewOffset);
2320}
2321
2322enum class SelectTypeKind {
2323 Int1 = 0,
2324 Int = 1,
2325 FP = 2,
2327};
2328
2329/// This function selects an opcode from a list of opcodes, which is
2330/// expected to be the opcode for { 8-bit, 16-bit, 32-bit, 64-bit }
2331/// element types, in this order.
2332template <SelectTypeKind Kind>
2333static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef<unsigned> Opcodes) {
2334 // Only match scalable vector VTs
2335 if (!VT.isScalableVector())
2336 return 0;
2337
2338 EVT EltVT = VT.getVectorElementType();
2339 unsigned Key = VT.getVectorMinNumElements();
2340 switch (Kind) {
2342 break;
2344 if (EltVT != MVT::i8 && EltVT != MVT::i16 && EltVT != MVT::i32 &&
2345 EltVT != MVT::i64)
2346 return 0;
2347 break;
2349 if (EltVT != MVT::i1)
2350 return 0;
2351 break;
2352 case SelectTypeKind::FP:
2353 if (EltVT == MVT::bf16)
2354 Key = 16;
2355 else if (EltVT != MVT::bf16 && EltVT != MVT::f16 && EltVT != MVT::f32 &&
2356 EltVT != MVT::f64)
2357 return 0;
2358 break;
2359 }
2360
2361 unsigned Offset;
2362 switch (Key) {
2363 case 16: // 8-bit or bf16
2364 Offset = 0;
2365 break;
2366 case 8: // 16-bit
2367 Offset = 1;
2368 break;
2369 case 4: // 32-bit
2370 Offset = 2;
2371 break;
2372 case 2: // 64-bit
2373 Offset = 3;
2374 break;
2375 default:
2376 return 0;
2377 }
2378
2379 return (Opcodes.size() <= Offset) ? 0 : Opcodes[Offset];
2380}
2381
2382// This function is almost identical to SelectWhilePair, but has an
2383// extra check on the range of the immediate operand.
2384// TODO: Merge these two functions together at some point?
2385void AArch64DAGToDAGISel::SelectPExtPair(SDNode *N, unsigned Opc) {
2386 // Immediate can be either 0 or 1.
2387 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(N->getOperand(2)))
2388 if (Imm->getZExtValue() > 1)
2389 return;
2390
2391 SDLoc DL(N);
2392 EVT VT = N->getValueType(0);
2393 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)};
2394 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
2395 SDValue SuperReg = SDValue(WhilePair, 0);
2396
2397 for (unsigned I = 0; I < 2; ++I)
2398 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
2399 AArch64::psub0 + I, DL, VT, SuperReg));
2400
2401 CurDAG->RemoveDeadNode(N);
2402}
2403
2404void AArch64DAGToDAGISel::SelectWhilePair(SDNode *N, unsigned Opc) {
2405 SDLoc DL(N);
2406 EVT VT = N->getValueType(0);
2407
2408 SDValue Ops[] = {N->getOperand(1), N->getOperand(2)};
2409
2410 SDNode *WhilePair = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
2411 SDValue SuperReg = SDValue(WhilePair, 0);
2412
2413 for (unsigned I = 0; I < 2; ++I)
2414 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
2415 AArch64::psub0 + I, DL, VT, SuperReg));
2416
2417 CurDAG->RemoveDeadNode(N);
2418}
2419
2420void AArch64DAGToDAGISel::SelectCVTIntrinsic(SDNode *N, unsigned NumVecs,
2421 unsigned Opcode) {
2422 EVT VT = N->getValueType(0);
2423 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumVecs));
2424 SDValue Ops = createZTuple(Regs);
2425 SDLoc DL(N);
2426 SDNode *Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Ops);
2427 SDValue SuperReg = SDValue(Intrinsic, 0);
2428 for (unsigned i = 0; i < NumVecs; ++i)
2429 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2430 AArch64::zsub0 + i, DL, VT, SuperReg));
2431
2432 CurDAG->RemoveDeadNode(N);
2433}
2434
2435void AArch64DAGToDAGISel::SelectCVTIntrinsicFP8(SDNode *N, unsigned NumVecs,
2436 unsigned Opcode) {
2437 SDLoc DL(N);
2438 EVT VT = N->getValueType(0);
2439 SmallVector<SDValue, 4> Ops(N->op_begin() + 2, N->op_end());
2440 Ops.push_back(/*Chain*/ N->getOperand(0));
2441
2442 SDNode *Instruction =
2443 CurDAG->getMachineNode(Opcode, DL, {MVT::Untyped, MVT::Other}, Ops);
2444 SDValue SuperReg = SDValue(Instruction, 0);
2445
2446 for (unsigned i = 0; i < NumVecs; ++i)
2447 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2448 AArch64::zsub0 + i, DL, VT, SuperReg));
2449
2450 // Copy chain
2451 unsigned ChainIdx = NumVecs;
2452 ReplaceUses(SDValue(N, ChainIdx), SDValue(Instruction, 1));
2453 CurDAG->RemoveDeadNode(N);
2454}
2455
2456void AArch64DAGToDAGISel::SelectDestructiveMultiIntrinsic(SDNode *N,
2457 unsigned NumVecs,
2458 bool IsZmMulti,
2459 unsigned Opcode,
2460 bool HasPred) {
2461 assert(Opcode != 0 && "Unexpected opcode");
2462
2463 SDLoc DL(N);
2464 EVT VT = N->getValueType(0);
2465 SDUse *OpsIter = N->op_begin() + 1; // Skip intrinsic ID
2467
2468 auto GetMultiVecOperand = [&]() {
2469 SmallVector<SDValue, 4> Regs(OpsIter, OpsIter + NumVecs);
2470 OpsIter += NumVecs;
2471 return createZMulTuple(Regs);
2472 };
2473
2474 if (HasPred)
2475 Ops.push_back(*OpsIter++);
2476
2477 Ops.push_back(GetMultiVecOperand());
2478 if (IsZmMulti)
2479 Ops.push_back(GetMultiVecOperand());
2480 else
2481 Ops.push_back(*OpsIter++);
2482
2483 // Append any remaining operands.
2484 Ops.append(OpsIter, N->op_end());
2485 SDNode *Intrinsic;
2486 Intrinsic = CurDAG->getMachineNode(Opcode, DL, MVT::Untyped, Ops);
2487 SDValue SuperReg = SDValue(Intrinsic, 0);
2488 for (unsigned i = 0; i < NumVecs; ++i)
2489 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2490 AArch64::zsub0 + i, DL, VT, SuperReg));
2491
2492 CurDAG->RemoveDeadNode(N);
2493}
2494
2495void AArch64DAGToDAGISel::SelectPredicatedLoad(SDNode *N, unsigned NumVecs,
2496 unsigned Scale, unsigned Opc_ri,
2497 unsigned Opc_rr, bool IsIntr) {
2498 assert(Scale < 5 && "Invalid scaling value.");
2499 SDLoc DL(N);
2500 EVT VT = N->getValueType(0);
2501 SDValue Chain = N->getOperand(0);
2502
2503 // Optimize addressing mode.
2505 unsigned Opc;
2506 std::tie(Opc, Base, Offset) = findAddrModeSVELoadStore(
2507 N, Opc_rr, Opc_ri, N->getOperand(IsIntr ? 3 : 2),
2508 CurDAG->getTargetConstant(0, DL, MVT::i64), Scale);
2509
2510 SDValue Ops[] = {N->getOperand(IsIntr ? 2 : 1), // Predicate
2511 Base, // Memory operand
2512 Offset, Chain};
2513
2514 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2515
2516 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops);
2517 SDValue SuperReg = SDValue(Load, 0);
2518 for (unsigned i = 0; i < NumVecs; ++i)
2519 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2520 AArch64::zsub0 + i, DL, VT, SuperReg));
2521
2522 // Copy chain
2523 unsigned ChainIdx = NumVecs;
2524 ReplaceUses(SDValue(N, ChainIdx), SDValue(Load, 1));
2525 CurDAG->RemoveDeadNode(N);
2526}
2527
2528void AArch64DAGToDAGISel::SelectContiguousMultiVectorLoad(SDNode *N,
2529 unsigned NumVecs,
2530 unsigned Scale,
2531 unsigned Opc_ri,
2532 unsigned Opc_rr) {
2533 assert(Scale < 4 && "Invalid scaling value.");
2534 SDLoc DL(N);
2535 EVT VT = N->getValueType(0);
2536 SDValue Chain = N->getOperand(0);
2537
2538 SDValue PNg = N->getOperand(2);
2539 SDValue Base = N->getOperand(3);
2540 SDValue Offset = CurDAG->getTargetConstant(0, DL, MVT::i64);
2541 unsigned Opc;
2542 std::tie(Opc, Base, Offset) =
2543 findAddrModeSVELoadStore(N, Opc_rr, Opc_ri, Base, Offset, Scale);
2544
2545 SDValue Ops[] = {PNg, // Predicate-as-counter
2546 Base, // Memory operand
2547 Offset, Chain};
2548
2549 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2550
2551 SDNode *Load = CurDAG->getMachineNode(Opc, DL, ResTys, Ops);
2552 SDValue SuperReg = SDValue(Load, 0);
2553 for (unsigned i = 0; i < NumVecs; ++i)
2554 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2555 AArch64::zsub0 + i, DL, VT, SuperReg));
2556
2557 // Copy chain
2558 unsigned ChainIdx = NumVecs;
2559 ReplaceUses(SDValue(N, ChainIdx), SDValue(Load, 1));
2560 CurDAG->RemoveDeadNode(N);
2561}
2562
2563void AArch64DAGToDAGISel::SelectFrintFromVT(SDNode *N, unsigned NumVecs,
2564 unsigned Opcode) {
2565 if (N->getValueType(0) != MVT::nxv4f32)
2566 return;
2567 SelectUnaryMultiIntrinsic(N, NumVecs, true, Opcode);
2568}
2569
2570void AArch64DAGToDAGISel::SelectMultiVectorLutiLane(SDNode *Node,
2571 unsigned NumOutVecs,
2572 unsigned Opc,
2573 uint32_t MaxImm) {
2574 if (ConstantSDNode *Imm = dyn_cast<ConstantSDNode>(Node->getOperand(4)))
2575 if (Imm->getZExtValue() > MaxImm)
2576 return;
2577
2578 SDValue ZtValue;
2579 if (!ImmToReg<AArch64::ZT0, 0>(Node->getOperand(2), ZtValue))
2580 return;
2581
2582 SDValue Chain = Node->getOperand(0);
2583 SDValue Ops[] = {ZtValue, Node->getOperand(3), Node->getOperand(4), Chain};
2584 SDLoc DL(Node);
2585 EVT VT = Node->getValueType(0);
2586
2587 SDNode *Instruction =
2588 CurDAG->getMachineNode(Opc, DL, {MVT::Untyped, MVT::Other}, Ops);
2589 SDValue SuperReg = SDValue(Instruction, 0);
2590
2591 for (unsigned I = 0; I < NumOutVecs; ++I)
2592 ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(
2593 AArch64::zsub0 + I, DL, VT, SuperReg));
2594
2595 // Copy chain
2596 unsigned ChainIdx = NumOutVecs;
2597 ReplaceUses(SDValue(Node, ChainIdx), SDValue(Instruction, 1));
2598 CurDAG->RemoveDeadNode(Node);
2599}
2600
2601void AArch64DAGToDAGISel::SelectMultiVectorLuti6LaneX4(SDNode *Node,
2602 unsigned NumIndexVecs) {
2603 assert((NumIndexVecs == 2 || NumIndexVecs == 3) &&
2604 "unexpected number of index vectors");
2605
2606 constexpr unsigned FirstIndexOp = 3;
2607 unsigned ImmOp = FirstIndexOp + NumIndexVecs;
2608 auto *Imm = dyn_cast<ConstantSDNode>(Node->getOperand(ImmOp));
2609 if (!Imm || Imm->getZExtValue() > 1)
2610 return;
2611
2612 // The luti6 instruction always takes a 2-register Zm index tuple. The x3
2613 // ACLE form provides three index vectors, so the lane selects which adjacent
2614 // pair to use before forming Zm (op 3/4 or op 4/5, with op6 as imm)
2615 unsigned Lane = Imm->getZExtValue();
2616 unsigned IndexOp = FirstIndexOp;
2617 if (NumIndexVecs == 3)
2618 IndexOp += Lane;
2619
2620 SDValue TableTuple = createZTuple({Node->getOperand(1), Node->getOperand(2)});
2621 SDValue IndexTuple =
2622 createZTuple({Node->getOperand(IndexOp), Node->getOperand(IndexOp + 1)});
2623 SDValue Ops[] = {TableTuple, IndexTuple, Node->getOperand(ImmOp)};
2624
2625 SDLoc DL(Node);
2626 EVT VT = Node->getValueType(0);
2627 SDNode *Instruction =
2628 CurDAG->getMachineNode(AArch64::LUTI6_4Z2Z2ZI, DL, MVT::Untyped, Ops);
2629 SDValue SuperReg = SDValue(Instruction, 0);
2630
2631 for (unsigned I = 0; I < 4; ++I)
2632 ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(
2633 AArch64::zsub0 + I, DL, VT, SuperReg));
2634
2635 CurDAG->RemoveDeadNode(Node);
2636}
2637
2638void AArch64DAGToDAGISel::SelectMultiVectorLuti(SDNode *Node,
2639 unsigned NumOutVecs,
2640 unsigned Opc,
2641 unsigned NumInVecs) {
2642 assert((NumInVecs == 2 || NumInVecs == 3) &&
2643 "unexpected number of input vectors");
2644
2645 SDValue ZtValue;
2646 if (!ImmToReg<AArch64::ZT0, 0>(Node->getOperand(2), ZtValue))
2647 return;
2648
2649 SmallVector<SDValue, 4> Regs(Node->ops().slice(3, NumInVecs));
2650 SDValue ZTuple = NumInVecs == 3 ? createZTuple(Regs) : createZMulTuple(Regs);
2651 SDValue Ops[] = {ZtValue, ZTuple, Node->getOperand(0)};
2652
2653 SDLoc DL(Node);
2654 EVT VT = Node->getValueType(0);
2655
2656 SDNode *Instruction =
2657 CurDAG->getMachineNode(Opc, DL, {MVT::Untyped, MVT::Other}, Ops);
2658 SDValue SuperReg = SDValue(Instruction, 0);
2659
2660 for (unsigned I = 0; I < NumOutVecs; ++I)
2661 ReplaceUses(SDValue(Node, I), CurDAG->getTargetExtractSubreg(
2662 AArch64::zsub0 + I, DL, VT, SuperReg));
2663
2664 ReplaceUses(SDValue(Node, NumOutVecs), SDValue(Instruction, 1));
2665 CurDAG->RemoveDeadNode(Node);
2666}
2667
2668void AArch64DAGToDAGISel::SelectClamp(SDNode *N, unsigned NumVecs,
2669 unsigned Op) {
2670 SDLoc DL(N);
2671 EVT VT = N->getValueType(0);
2672
2673 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumVecs));
2674 SDValue Zd = createZMulTuple(Regs);
2675 SDValue Zn = N->getOperand(1 + NumVecs);
2676 SDValue Zm = N->getOperand(2 + NumVecs);
2677
2678 SDValue Ops[] = {Zd, Zn, Zm};
2679
2680 SDNode *Intrinsic = CurDAG->getMachineNode(Op, DL, MVT::Untyped, Ops);
2681 SDValue SuperReg = SDValue(Intrinsic, 0);
2682 for (unsigned i = 0; i < NumVecs; ++i)
2683 ReplaceUses(SDValue(N, i), CurDAG->getTargetExtractSubreg(
2684 AArch64::zsub0 + i, DL, VT, SuperReg));
2685
2686 CurDAG->RemoveDeadNode(N);
2687}
2688
2689bool SelectSMETile(unsigned &BaseReg, unsigned TileNum) {
2690 switch (BaseReg) {
2691 default:
2692 return false;
2693 case AArch64::ZA:
2694 case AArch64::ZAB0:
2695 if (TileNum == 0)
2696 break;
2697 return false;
2698 case AArch64::ZAH0:
2699 if (TileNum <= 1)
2700 break;
2701 return false;
2702 case AArch64::ZAS0:
2703 if (TileNum <= 3)
2704 break;
2705 return false;
2706 case AArch64::ZAD0:
2707 if (TileNum <= 7)
2708 break;
2709 return false;
2710 }
2711
2712 BaseReg += TileNum;
2713 return true;
2714}
2715
2716template <unsigned MaxIdx, unsigned Scale>
2717void AArch64DAGToDAGISel::SelectMultiVectorMove(SDNode *N, unsigned NumVecs,
2718 unsigned BaseReg, unsigned Op) {
2719 unsigned TileNum = 0;
2720 if (BaseReg != AArch64::ZA)
2721 TileNum = N->getConstantOperandVal(2);
2722
2723 if (!SelectSMETile(BaseReg, TileNum))
2724 return;
2725
2726 SDValue SliceBase, Base, Offset;
2727 if (BaseReg == AArch64::ZA)
2728 SliceBase = N->getOperand(2);
2729 else
2730 SliceBase = N->getOperand(3);
2731
2732 if (!SelectSMETileSlice(SliceBase, MaxIdx, Base, Offset, Scale))
2733 return;
2734
2735 SDLoc DL(N);
2736 SDValue SubReg = CurDAG->getRegister(BaseReg, MVT::Other);
2737 SDValue Ops[] = {SubReg, Base, Offset, /*Chain*/ N->getOperand(0)};
2738 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops);
2739
2740 EVT VT = N->getValueType(0);
2741 for (unsigned I = 0; I < NumVecs; ++I)
2742 ReplaceUses(SDValue(N, I),
2743 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT,
2744 SDValue(Mov, 0)));
2745 // Copy chain
2746 unsigned ChainIdx = NumVecs;
2747 ReplaceUses(SDValue(N, ChainIdx), SDValue(Mov, 1));
2748 CurDAG->RemoveDeadNode(N);
2749}
2750
2751void AArch64DAGToDAGISel::SelectMultiVectorMoveZ(SDNode *N, unsigned NumVecs,
2752 unsigned Op, unsigned MaxIdx,
2753 unsigned Scale, unsigned BaseReg) {
2754 // Slice can be in different positions
2755 // The array to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(slice)
2756 // The tile to vector: llvm.aarch64.sme.readz.<h/v>.<sz>(tile, slice)
2757 SDValue SliceBase = N->getOperand(2);
2758 if (BaseReg != AArch64::ZA)
2759 SliceBase = N->getOperand(3);
2760
2762 if (!SelectSMETileSlice(SliceBase, MaxIdx, Base, Offset, Scale))
2763 return;
2764 // The correct Za tile number is computed in Machine Instruction
2765 // See EmitZAInstr
2766 // DAG cannot select Za tile as an output register with ZReg
2767 SDLoc DL(N);
2769 if (BaseReg != AArch64::ZA )
2770 Ops.push_back(N->getOperand(2));
2771 Ops.push_back(Base);
2772 Ops.push_back(Offset);
2773 Ops.push_back(N->getOperand(0)); //Chain
2774 SDNode *Mov = CurDAG->getMachineNode(Op, DL, {MVT::Untyped, MVT::Other}, Ops);
2775
2776 EVT VT = N->getValueType(0);
2777 for (unsigned I = 0; I < NumVecs; ++I)
2778 ReplaceUses(SDValue(N, I),
2779 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT,
2780 SDValue(Mov, 0)));
2781
2782 // Copy chain
2783 unsigned ChainIdx = NumVecs;
2784 ReplaceUses(SDValue(N, ChainIdx), SDValue(Mov, 1));
2785 CurDAG->RemoveDeadNode(N);
2786}
2787
2788void AArch64DAGToDAGISel::SelectUnaryMultiIntrinsic(SDNode *N,
2789 unsigned NumOutVecs,
2790 bool IsTupleInput,
2791 unsigned Opc) {
2792 SDLoc DL(N);
2793 EVT VT = N->getValueType(0);
2794 unsigned NumInVecs = N->getNumOperands() - 1;
2795
2797 if (IsTupleInput) {
2798 assert((NumInVecs == 2 || NumInVecs == 4) &&
2799 "Don't know how to handle multi-register input!");
2800 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumInVecs));
2801 Ops.push_back(createZMulTuple(Regs));
2802 } else {
2803 // All intrinsic nodes have the ID as the first operand, hence the "1 + I".
2804 for (unsigned I = 0; I < NumInVecs; I++)
2805 Ops.push_back(N->getOperand(1 + I));
2806 }
2807
2808 SDNode *Res = CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Ops);
2809 SDValue SuperReg = SDValue(Res, 0);
2810
2811 for (unsigned I = 0; I < NumOutVecs; I++)
2812 ReplaceUses(SDValue(N, I), CurDAG->getTargetExtractSubreg(
2813 AArch64::zsub0 + I, DL, VT, SuperReg));
2814 CurDAG->RemoveDeadNode(N);
2815}
2816
2817void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs,
2818 unsigned Opc) {
2819 SDLoc dl(N);
2820 EVT VT = N->getOperand(2)->getValueType(0);
2821
2822 // Form a REG_SEQUENCE to force register allocation.
2823 bool Is128Bit = VT.getSizeInBits() == 128;
2824 SmallVector<SDValue, 4> Regs(N->ops().slice(2, NumVecs));
2825 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2826
2827 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)};
2828 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
2829
2830 // Transfer memoperands.
2831 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2832 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2833
2834 ReplaceNode(N, St);
2835}
2836
2837void AArch64DAGToDAGISel::SelectPredicatedStore(SDNode *N, unsigned NumVecs,
2838 unsigned Scale, unsigned Opc_rr,
2839 unsigned Opc_ri) {
2840 SDLoc dl(N);
2841
2842 // Form a REG_SEQUENCE to force register allocation.
2843 SmallVector<SDValue, 4> Regs(N->ops().slice(2, NumVecs));
2844 SDValue RegSeq = createZTuple(Regs);
2845
2846 // Optimize addressing mode.
2847 unsigned Opc;
2849 std::tie(Opc, Base, Offset) = findAddrModeSVELoadStore(
2850 N, Opc_rr, Opc_ri, N->getOperand(NumVecs + 3),
2851 CurDAG->getTargetConstant(0, dl, MVT::i64), Scale);
2852
2853 SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), // predicate
2854 Base, // address
2855 Offset, // offset
2856 N->getOperand(0)}; // chain
2857 SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops);
2858
2859 // Transfer memoperands.
2860 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2861 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2862
2863 ReplaceNode(N, St);
2864}
2865
2866void AArch64DAGToDAGISel::SelectPostStore(SDNode *N, unsigned NumVecs,
2867 unsigned Opc) {
2868 SDLoc dl(N);
2869 EVT VT = N->getOperand(2)->getValueType(0);
2870 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2871 MVT::Other}; // Type for the Chain
2872
2873 // Form a REG_SEQUENCE to force register allocation.
2874 bool Is128Bit = VT.getSizeInBits() == 128;
2875 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumVecs));
2876 SDValue RegSeq = Is128Bit ? createQTuple(Regs) : createDTuple(Regs);
2877
2878 SDValue Ops[] = {RegSeq,
2879 N->getOperand(NumVecs + 1), // base register
2880 N->getOperand(NumVecs + 2), // Incremental
2881 N->getOperand(0)}; // Chain
2882 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2883
2884 // Transfer memoperands.
2885 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2886 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
2887
2888 ReplaceNode(N, St);
2889}
2890
2891namespace {
2892/// WidenVector - Given a value in the V64 register class, produce the
2893/// equivalent value in the V128 register class.
2894class WidenVector {
2895 SelectionDAG &DAG;
2896
2897public:
2898 WidenVector(SelectionDAG &DAG) : DAG(DAG) {}
2899
2900 SDValue operator()(SDValue V64Reg) {
2901 EVT VT = V64Reg.getValueType();
2902 unsigned NarrowSize = VT.getVectorNumElements();
2903 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2904 MVT WideTy = MVT::getVectorVT(EltTy, 2 * NarrowSize);
2905 SDLoc DL(V64Reg);
2906
2907 SDValue Undef =
2908 SDValue(DAG.getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, WideTy), 0);
2909 return DAG.getTargetInsertSubreg(AArch64::dsub, DL, WideTy, Undef, V64Reg);
2910 }
2911};
2912} // namespace
2913
2914/// NarrowVector - Given a value in the V128 register class, produce the
2915/// equivalent value in the V64 register class.
2917 EVT VT = V128Reg.getValueType();
2918 unsigned WideSize = VT.getVectorNumElements();
2919 MVT EltTy = VT.getVectorElementType().getSimpleVT();
2920 MVT NarrowTy = MVT::getVectorVT(EltTy, WideSize / 2);
2921
2922 return DAG.getTargetExtractSubreg(AArch64::dsub, SDLoc(V128Reg), NarrowTy,
2923 V128Reg);
2924}
2925
2926void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs,
2927 unsigned Opc) {
2928 SDLoc dl(N);
2929 EVT VT = N->getValueType(0);
2930 bool Narrow = VT.getSizeInBits() == 64;
2931
2932 // Form a REG_SEQUENCE to force register allocation.
2933 SmallVector<SDValue, 4> Regs(N->ops().slice(2, NumVecs));
2934
2935 if (Narrow)
2936 transform(Regs, Regs.begin(),
2937 WidenVector(*CurDAG));
2938
2939 SDValue RegSeq = createQTuple(Regs);
2940
2941 const EVT ResTys[] = {MVT::Untyped, MVT::Other};
2942
2943 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2);
2944
2945 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
2946 N->getOperand(NumVecs + 3), N->getOperand(0)};
2947 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2948 SDValue SuperReg = SDValue(Ld, 0);
2949
2950 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
2951 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
2952 AArch64::qsub2, AArch64::qsub3 };
2953 for (unsigned i = 0; i < NumVecs; ++i) {
2954 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT, SuperReg);
2955 if (Narrow)
2956 NV = NarrowVector(NV, *CurDAG);
2957 ReplaceUses(SDValue(N, i), NV);
2958 }
2959
2960 // Transfer memoperands.
2961 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2962 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
2963
2964 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1));
2965 CurDAG->RemoveDeadNode(N);
2966}
2967
2968void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs,
2969 unsigned Opc) {
2970 SDLoc dl(N);
2971 EVT VT = N->getValueType(0);
2972 bool Narrow = VT.getSizeInBits() == 64;
2973
2974 // Form a REG_SEQUENCE to force register allocation.
2975 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumVecs));
2976
2977 if (Narrow)
2978 transform(Regs, Regs.begin(),
2979 WidenVector(*CurDAG));
2980
2981 SDValue RegSeq = createQTuple(Regs);
2982
2983 const EVT ResTys[] = {MVT::i64, // Type of the write back register
2984 RegSeq->getValueType(0), MVT::Other};
2985
2986 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1);
2987
2988 SDValue Ops[] = {RegSeq,
2989 CurDAG->getTargetConstant(LaneNo, dl,
2990 MVT::i64), // Lane Number
2991 N->getOperand(NumVecs + 2), // Base register
2992 N->getOperand(NumVecs + 3), // Incremental
2993 N->getOperand(0)};
2994 SDNode *Ld = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2995
2996 // Update uses of the write back register
2997 ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 0));
2998
2999 // Update uses of the vector list
3000 SDValue SuperReg = SDValue(Ld, 1);
3001 if (NumVecs == 1) {
3002 ReplaceUses(SDValue(N, 0),
3003 Narrow ? NarrowVector(SuperReg, *CurDAG) : SuperReg);
3004 } else {
3005 EVT WideVT = RegSeq.getOperand(1)->getValueType(0);
3006 static const unsigned QSubs[] = { AArch64::qsub0, AArch64::qsub1,
3007 AArch64::qsub2, AArch64::qsub3 };
3008 for (unsigned i = 0; i < NumVecs; ++i) {
3009 SDValue NV = CurDAG->getTargetExtractSubreg(QSubs[i], dl, WideVT,
3010 SuperReg);
3011 if (Narrow)
3012 NV = NarrowVector(NV, *CurDAG);
3013 ReplaceUses(SDValue(N, i), NV);
3014 }
3015 }
3016
3017 // Transfer memoperands.
3018 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3019 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3020
3021 // Update the Chain
3022 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(Ld, 2));
3023 CurDAG->RemoveDeadNode(N);
3024}
3025
3026void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs,
3027 unsigned Opc) {
3028 SDLoc dl(N);
3029 EVT VT = N->getOperand(2)->getValueType(0);
3030 bool Narrow = VT.getSizeInBits() == 64;
3031
3032 // Form a REG_SEQUENCE to force register allocation.
3033 SmallVector<SDValue, 4> Regs(N->ops().slice(2, NumVecs));
3034
3035 if (Narrow)
3036 transform(Regs, Regs.begin(),
3037 WidenVector(*CurDAG));
3038
3039 SDValue RegSeq = createQTuple(Regs);
3040
3041 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 2);
3042
3043 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
3044 N->getOperand(NumVecs + 3), N->getOperand(0)};
3045 SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
3046
3047 // Transfer memoperands.
3048 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3049 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3050
3051 ReplaceNode(N, St);
3052}
3053
3054void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs,
3055 unsigned Opc) {
3056 SDLoc dl(N);
3057 EVT VT = N->getOperand(2)->getValueType(0);
3058 bool Narrow = VT.getSizeInBits() == 64;
3059
3060 // Form a REG_SEQUENCE to force register allocation.
3061 SmallVector<SDValue, 4> Regs(N->ops().slice(1, NumVecs));
3062
3063 if (Narrow)
3064 transform(Regs, Regs.begin(),
3065 WidenVector(*CurDAG));
3066
3067 SDValue RegSeq = createQTuple(Regs);
3068
3069 const EVT ResTys[] = {MVT::i64, // Type of the write back register
3070 MVT::Other};
3071
3072 unsigned LaneNo = N->getConstantOperandVal(NumVecs + 1);
3073
3074 SDValue Ops[] = {RegSeq, CurDAG->getTargetConstant(LaneNo, dl, MVT::i64),
3075 N->getOperand(NumVecs + 2), // Base Register
3076 N->getOperand(NumVecs + 3), // Incremental
3077 N->getOperand(0)};
3078 SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3079
3080 // Transfer memoperands.
3081 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3082 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3083
3084 ReplaceNode(N, St);
3085}
3086
3088 unsigned &Opc, SDValue &Opd0,
3089 unsigned &LSB, unsigned &MSB,
3090 unsigned NumberOfIgnoredLowBits,
3091 bool BiggerPattern) {
3092 assert(N->getOpcode() == ISD::AND &&
3093 "N must be a AND operation to call this function");
3094
3095 EVT VT = N->getValueType(0);
3096
3097 // Here we can test the type of VT and return false when the type does not
3098 // match, but since it is done prior to that call in the current context
3099 // we turned that into an assert to avoid redundant code.
3100 assert((VT == MVT::i32 || VT == MVT::i64) &&
3101 "Type checking must have been done before calling this function");
3102
3103 // FIXME: simplify-demanded-bits in DAGCombine will probably have
3104 // changed the AND node to a 32-bit mask operation. We'll have to
3105 // undo that as part of the transform here if we want to catch all
3106 // the opportunities.
3107 // Currently the NumberOfIgnoredLowBits argument helps to recover
3108 // from these situations when matching bigger pattern (bitfield insert).
3109
3110 // For unsigned extracts, check for a shift right and mask
3111 uint64_t AndImm = 0;
3112 if (!isOpcWithIntImmediate(N, ISD::AND, AndImm))
3113 return false;
3114
3115 const SDNode *Op0 = N->getOperand(0).getNode();
3116
3117 // Because of simplify-demanded-bits in DAGCombine, the mask may have been
3118 // simplified. Try to undo that
3119 AndImm |= maskTrailingOnes<uint64_t>(NumberOfIgnoredLowBits);
3120
3121 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
3122 if (AndImm & (AndImm + 1))
3123 return false;
3124
3125 bool ClampMSB = false;
3126 uint64_t SrlImm = 0;
3127 // Handle the SRL + ANY_EXTEND case.
3128 if (VT == MVT::i64 && Op0->getOpcode() == ISD::ANY_EXTEND &&
3129 isOpcWithIntImmediate(Op0->getOperand(0).getNode(), ISD::SRL, SrlImm)) {
3130 // Extend the incoming operand of the SRL to 64-bit.
3131 Opd0 = Widen(CurDAG, Op0->getOperand(0).getOperand(0));
3132 // Make sure to clamp the MSB so that we preserve the semantics of the
3133 // original operations.
3134 ClampMSB = true;
3135 } else if (VT == MVT::i32 && Op0->getOpcode() == ISD::TRUNCATE &&
3137 SrlImm)) {
3138 // If the shift result was truncated, we can still combine them.
3139 Opd0 = Op0->getOperand(0).getOperand(0);
3140
3141 // Use the type of SRL node.
3142 VT = Opd0->getValueType(0);
3143 } else if (isOpcWithIntImmediate(Op0, ISD::SRL, SrlImm)) {
3144 Opd0 = Op0->getOperand(0);
3145 ClampMSB = (VT == MVT::i32);
3146 } else if (BiggerPattern) {
3147 // Let's pretend a 0 shift right has been performed.
3148 // The resulting code will be at least as good as the original one
3149 // plus it may expose more opportunities for bitfield insert pattern.
3150 // FIXME: Currently we limit this to the bigger pattern, because
3151 // some optimizations expect AND and not UBFM.
3152 Opd0 = N->getOperand(0);
3153 } else
3154 return false;
3155
3156 // Bail out on large immediates. This happens when no proper
3157 // combining/constant folding was performed.
3158 if (!BiggerPattern && (SrlImm <= 0 || SrlImm >= VT.getSizeInBits())) {
3159 LLVM_DEBUG(
3160 (dbgs() << N
3161 << ": Found large shift immediate, this should not happen\n"));
3162 return false;
3163 }
3164
3165 LSB = SrlImm;
3166 MSB = SrlImm +
3167 (VT == MVT::i32 ? llvm::countr_one<uint32_t>(AndImm)
3168 : llvm::countr_one<uint64_t>(AndImm)) -
3169 1;
3170 if (ClampMSB)
3171 // Since we're moving the extend before the right shift operation, we need
3172 // to clamp the MSB to make sure we don't shift in undefined bits instead of
3173 // the zeros which would get shifted in with the original right shift
3174 // operation.
3175 MSB = MSB > 31 ? 31 : MSB;
3176
3177 Opc = VT == MVT::i32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3178 return true;
3179}
3180
3182 SDValue &Opd0, unsigned &Immr,
3183 unsigned &Imms) {
3184 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG);
3185
3186 EVT VT = N->getValueType(0);
3187 unsigned BitWidth = VT.getSizeInBits();
3188 assert((VT == MVT::i32 || VT == MVT::i64) &&
3189 "Type checking must have been done before calling this function");
3190
3191 SDValue Op = N->getOperand(0);
3192 if (Op->getOpcode() == ISD::TRUNCATE) {
3193 Op = Op->getOperand(0);
3194 VT = Op->getValueType(0);
3195 BitWidth = VT.getSizeInBits();
3196 }
3197
3198 uint64_t ShiftImm;
3199 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SRL, ShiftImm) &&
3200 !isOpcWithIntImmediate(Op.getNode(), ISD::SRA, ShiftImm))
3201 return false;
3202
3203 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
3204 if (ShiftImm + Width > BitWidth)
3205 return false;
3206
3207 Opc = (VT == MVT::i32) ? AArch64::SBFMWri : AArch64::SBFMXri;
3208 Opd0 = Op.getOperand(0);
3209 Immr = ShiftImm;
3210 Imms = ShiftImm + Width - 1;
3211 return true;
3212}
3213
3214static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
3215 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
3216 unsigned NumberOfIgnoredLowBits = 0,
3217 bool BiggerPattern = false) {
3218 if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
3219 return false;
3220
3221 switch (N->getOpcode()) {
3222 default:
3223 if (!N->isMachineOpcode())
3224 return false;
3225 break;
3226 case ISD::AND:
3227 return isBitfieldExtractOpFromAnd(CurDAG, N, Opc, Opd0, Immr, Imms,
3228 NumberOfIgnoredLowBits, BiggerPattern);
3229 case ISD::SRL:
3230 case ISD::SRA:
3231 return isBitfieldExtractOpFromShr(N, Opc, Opd0, Immr, Imms, BiggerPattern);
3232
3234 return isBitfieldExtractOpFromSExtInReg(N, Opc, Opd0, Immr, Imms);
3235 }
3236
3237 unsigned NOpc = N->getMachineOpcode();
3238 switch (NOpc) {
3239 default:
3240 return false;
3241 case AArch64::SBFMWri:
3242 case AArch64::UBFMWri:
3243 case AArch64::SBFMXri:
3244 case AArch64::UBFMXri:
3245 Opc = NOpc;
3246 Opd0 = N->getOperand(0);
3247 Immr = N->getConstantOperandVal(1);
3248 Imms = N->getConstantOperandVal(2);
3249 return true;
3250 }
3251 // Unreachable
3252 return false;
3253}
3254
3255bool AArch64DAGToDAGISel::tryBitfieldExtractOp(SDNode *N) {
3256 unsigned Opc, Immr, Imms;
3257 SDValue Opd0;
3258 if (!isBitfieldExtractOp(CurDAG, N, Opc, Opd0, Immr, Imms))
3259 return false;
3260
3261 EVT VT = N->getValueType(0);
3262 SDLoc dl(N);
3263
3264 // If the bit extract operation is 64bit but the original type is 32bit, we
3265 // need to add one EXTRACT_SUBREG.
3266 if ((Opc == AArch64::SBFMXri || Opc == AArch64::UBFMXri) && VT == MVT::i32) {
3267 SDValue Ops64[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, MVT::i64),
3268 CurDAG->getTargetConstant(Imms, dl, MVT::i64)};
3269
3270 SDNode *BFM = CurDAG->getMachineNode(Opc, dl, MVT::i64, Ops64);
3271 SDValue Inner = CurDAG->getTargetExtractSubreg(AArch64::sub_32, dl,
3272 MVT::i32, SDValue(BFM, 0));
3273 ReplaceNode(N, Inner.getNode());
3274 return true;
3275 }
3276
3277 SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
3278 CurDAG->getTargetConstant(Imms, dl, VT)};
3279 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3280 return true;
3281}
3282
3283/// Does DstMask form a complementary pair with the mask provided by
3284/// BitsToBeInserted, suitable for use in a BFI instruction. Roughly speaking,
3285/// this asks whether DstMask zeroes precisely those bits that will be set by
3286/// the other half.
3287static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted,
3288 unsigned NumberOfIgnoredHighBits, EVT VT) {
3289 assert((VT == MVT::i32 || VT == MVT::i64) &&
3290 "i32 or i64 mask type expected!");
3291 unsigned BitWidth = VT.getSizeInBits() - NumberOfIgnoredHighBits;
3292
3293 // Enable implicitTrunc as we're intentionally ignoring high bits.
3294 APInt SignificantDstMask =
3295 APInt(BitWidth, DstMask, /*isSigned=*/false, /*implicitTrunc=*/true);
3296 APInt SignificantBitsToBeInserted = BitsToBeInserted.zextOrTrunc(BitWidth);
3297
3298 return (SignificantDstMask & SignificantBitsToBeInserted) == 0 &&
3299 (SignificantDstMask | SignificantBitsToBeInserted).isAllOnes();
3300}
3301
3302// Look for bits that will be useful for later uses.
3303// A bit is consider useless as soon as it is dropped and never used
3304// before it as been dropped.
3305// E.g., looking for useful bit of x
3306// 1. y = x & 0x7
3307// 2. z = y >> 2
3308// After #1, x useful bits are 0x7, then the useful bits of x, live through
3309// y.
3310// After #2, the useful bits of x are 0x4.
3311// However, if x is used on an unpredictable instruction, then all its bits
3312// are useful.
3313// E.g.
3314// 1. y = x & 0x7
3315// 2. z = y >> 2
3316// 3. str x, [@x]
3317static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth = 0);
3318
3320 unsigned Depth) {
3321 uint64_t Imm =
3322 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
3324 UsefulBits &= APInt(UsefulBits.getBitWidth(), Imm);
3325 getUsefulBits(Op, UsefulBits, Depth + 1);
3326}
3327
3329 uint64_t Imm, uint64_t MSB,
3330 unsigned Depth) {
3331 // inherit the bitwidth value
3332 APInt OpUsefulBits(UsefulBits);
3333 OpUsefulBits = 1;
3334
3335 if (MSB >= Imm) {
3336 OpUsefulBits <<= MSB - Imm + 1;
3337 --OpUsefulBits;
3338 // The interesting part will be in the lower part of the result
3339 getUsefulBits(Op, OpUsefulBits, Depth + 1);
3340 // The interesting part was starting at Imm in the argument
3341 OpUsefulBits <<= Imm;
3342 } else {
3343 OpUsefulBits <<= MSB + 1;
3344 --OpUsefulBits;
3345 // The interesting part will be shifted in the result
3346 OpUsefulBits <<= OpUsefulBits.getBitWidth() - Imm;
3347 getUsefulBits(Op, OpUsefulBits, Depth + 1);
3348 // The interesting part was at zero in the argument
3349 OpUsefulBits.lshrInPlace(OpUsefulBits.getBitWidth() - Imm);
3350 }
3351
3352 UsefulBits &= OpUsefulBits;
3353}
3354
3355static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits,
3356 unsigned Depth) {
3357 uint64_t Imm =
3358 cast<const ConstantSDNode>(Op.getOperand(1).getNode())->getZExtValue();
3359 uint64_t MSB =
3360 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
3361
3362 getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth);
3363}
3364
3366 unsigned Depth) {
3367 uint64_t ShiftTypeAndValue =
3368 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
3369 APInt Mask(UsefulBits);
3370 Mask.clearAllBits();
3371 Mask.flipAllBits();
3372
3373 if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSL) {
3374 // Shift Left
3375 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
3376 Mask <<= ShiftAmt;
3377 getUsefulBits(Op, Mask, Depth + 1);
3378 Mask.lshrInPlace(ShiftAmt);
3379 } else if (AArch64_AM::getShiftType(ShiftTypeAndValue) == AArch64_AM::LSR) {
3380 // Shift Right
3381 // We do not handle AArch64_AM::ASR, because the sign will change the
3382 // number of useful bits
3383 uint64_t ShiftAmt = AArch64_AM::getShiftValue(ShiftTypeAndValue);
3384 Mask.lshrInPlace(ShiftAmt);
3385 getUsefulBits(Op, Mask, Depth + 1);
3386 Mask <<= ShiftAmt;
3387 } else
3388 return;
3389
3390 UsefulBits &= Mask;
3391}
3392
3393static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits,
3394 unsigned Depth) {
3395 uint64_t Imm =
3396 cast<const ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
3397 uint64_t MSB =
3398 cast<const ConstantSDNode>(Op.getOperand(3).getNode())->getZExtValue();
3399
3400 APInt OpUsefulBits(UsefulBits);
3401 OpUsefulBits = 1;
3402
3403 APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0);
3404 ResultUsefulBits.flipAllBits();
3405 APInt Mask(UsefulBits.getBitWidth(), 0);
3406
3407 getUsefulBits(Op, ResultUsefulBits, Depth + 1);
3408
3409 if (MSB >= Imm) {
3410 // The instruction is a BFXIL.
3411 uint64_t Width = MSB - Imm + 1;
3412 uint64_t LSB = Imm;
3413
3414 OpUsefulBits <<= Width;
3415 --OpUsefulBits;
3416
3417 if (Op.getOperand(1) == Orig) {
3418 // Copy the low bits from the result to bits starting from LSB.
3419 Mask = ResultUsefulBits & OpUsefulBits;
3420 Mask <<= LSB;
3421 }
3422
3423 if (Op.getOperand(0) == Orig)
3424 // Bits starting from LSB in the input contribute to the result.
3425 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3426 } else {
3427 // The instruction is a BFI.
3428 uint64_t Width = MSB + 1;
3429 uint64_t LSB = UsefulBits.getBitWidth() - Imm;
3430
3431 OpUsefulBits <<= Width;
3432 --OpUsefulBits;
3433 OpUsefulBits <<= LSB;
3434
3435 if (Op.getOperand(1) == Orig) {
3436 // Copy the bits from the result to the zero bits.
3437 Mask = ResultUsefulBits & OpUsefulBits;
3438 Mask.lshrInPlace(LSB);
3439 }
3440
3441 if (Op.getOperand(0) == Orig)
3442 Mask |= (ResultUsefulBits & ~OpUsefulBits);
3443 }
3444
3445 UsefulBits &= Mask;
3446}
3447
3448static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits,
3449 SDValue Orig, unsigned Depth) {
3450
3451 // Users of this node should have already been instruction selected
3452 // FIXME: Can we turn that into an assert?
3453 if (!UserNode->isMachineOpcode())
3454 return;
3455
3456 switch (UserNode->getMachineOpcode()) {
3457 default:
3458 return;
3459 case AArch64::ANDSWri:
3460 case AArch64::ANDSXri:
3461 case AArch64::ANDWri:
3462 case AArch64::ANDXri:
3463 // We increment Depth only when we call the getUsefulBits
3464 return getUsefulBitsFromAndWithImmediate(SDValue(UserNode, 0), UsefulBits,
3465 Depth);
3466 case AArch64::UBFMWri:
3467 case AArch64::UBFMXri:
3468 return getUsefulBitsFromUBFM(SDValue(UserNode, 0), UsefulBits, Depth);
3469
3470 case AArch64::ORRWrs:
3471 case AArch64::ORRXrs:
3472 if (UserNode->getOperand(0) != Orig && UserNode->getOperand(1) == Orig)
3473 getUsefulBitsFromOrWithShiftedReg(SDValue(UserNode, 0), UsefulBits,
3474 Depth);
3475 return;
3476 case AArch64::BFMWri:
3477 case AArch64::BFMXri:
3478 return getUsefulBitsFromBFM(SDValue(UserNode, 0), Orig, UsefulBits, Depth);
3479
3480 case AArch64::STRBBui:
3481 case AArch64::STURBBi:
3482 if (UserNode->getOperand(0) != Orig)
3483 return;
3484 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xff);
3485 return;
3486
3487 case AArch64::STRHHui:
3488 case AArch64::STURHHi:
3489 if (UserNode->getOperand(0) != Orig)
3490 return;
3491 UsefulBits &= APInt(UsefulBits.getBitWidth(), 0xffff);
3492 return;
3493 }
3494}
3495
3496static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) {
3498 return;
3499 // Initialize UsefulBits
3500 if (!Depth) {
3501 unsigned Bitwidth = Op.getScalarValueSizeInBits();
3502 // At the beginning, assume every produced bits is useful
3503 UsefulBits = APInt(Bitwidth, 0);
3504 UsefulBits.flipAllBits();
3505 }
3506 APInt UsersUsefulBits(UsefulBits.getBitWidth(), 0);
3507
3508 for (SDNode *Node : Op.getNode()->users()) {
3509 // A use cannot produce useful bits
3510 APInt UsefulBitsForUse = APInt(UsefulBits);
3511 getUsefulBitsForUse(Node, UsefulBitsForUse, Op, Depth);
3512 UsersUsefulBits |= UsefulBitsForUse;
3513 }
3514 // UsefulBits contains the produced bits that are meaningful for the
3515 // current definition, thus a user cannot make a bit meaningful at
3516 // this point
3517 UsefulBits &= UsersUsefulBits;
3518}
3519
3520/// Create a machine node performing a notional SHL of Op by ShlAmount. If
3521/// ShlAmount is negative, do a (logical) right-shift instead. If ShlAmount is
3522/// 0, return Op unchanged.
3523static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount) {
3524 if (ShlAmount == 0)
3525 return Op;
3526
3527 EVT VT = Op.getValueType();
3528 SDLoc dl(Op);
3529 unsigned BitWidth = VT.getSizeInBits();
3530 unsigned UBFMOpc = BitWidth == 32 ? AArch64::UBFMWri : AArch64::UBFMXri;
3531
3532 SDNode *ShiftNode;
3533 if (ShlAmount > 0) {
3534 // LSL wD, wN, #Amt == UBFM wD, wN, #32-Amt, #31-Amt
3535 ShiftNode = CurDAG->getMachineNode(
3536 UBFMOpc, dl, VT, Op,
3537 CurDAG->getTargetConstant(BitWidth - ShlAmount, dl, VT),
3538 CurDAG->getTargetConstant(BitWidth - 1 - ShlAmount, dl, VT));
3539 } else {
3540 // LSR wD, wN, #Amt == UBFM wD, wN, #Amt, #32-1
3541 assert(ShlAmount < 0 && "expected right shift");
3542 int ShrAmount = -ShlAmount;
3543 ShiftNode = CurDAG->getMachineNode(
3544 UBFMOpc, dl, VT, Op, CurDAG->getTargetConstant(ShrAmount, dl, VT),
3545 CurDAG->getTargetConstant(BitWidth - 1, dl, VT));
3546 }
3547
3548 return SDValue(ShiftNode, 0);
3549}
3550
3551// For bit-field-positioning pattern "(and (shl VAL, N), ShiftedMask)".
3552static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op,
3553 bool BiggerPattern,
3554 const uint64_t NonZeroBits,
3555 SDValue &Src, int &DstLSB,
3556 int &Width);
3557
3558// For bit-field-positioning pattern "shl VAL, N)".
3559static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op,
3560 bool BiggerPattern,
3561 const uint64_t NonZeroBits,
3562 SDValue &Src, int &DstLSB,
3563 int &Width);
3564
3565/// Does this tree qualify as an attempt to move a bitfield into position,
3566/// essentially "(and (shl VAL, N), Mask)" or (shl VAL, N).
3568 bool BiggerPattern, SDValue &Src,
3569 int &DstLSB, int &Width) {
3570 EVT VT = Op.getValueType();
3571 unsigned BitWidth = VT.getSizeInBits();
3572 (void)BitWidth;
3573 assert(BitWidth == 32 || BitWidth == 64);
3574
3576
3577 // Non-zero in the sense that they're not provably zero, which is the key
3578 // point if we want to use this value
3579 const uint64_t NonZeroBits = (~Known.Zero).getZExtValue();
3580 if (!isShiftedMask_64(NonZeroBits))
3581 return false;
3582
3583 switch (Op.getOpcode()) {
3584 default:
3585 break;
3586 case ISD::AND:
3587 return isBitfieldPositioningOpFromAnd(CurDAG, Op, BiggerPattern,
3588 NonZeroBits, Src, DstLSB, Width);
3589 case ISD::SHL:
3590 return isBitfieldPositioningOpFromShl(CurDAG, Op, BiggerPattern,
3591 NonZeroBits, Src, DstLSB, Width);
3592 }
3593
3594 return false;
3595}
3596
3598 bool BiggerPattern,
3599 const uint64_t NonZeroBits,
3600 SDValue &Src, int &DstLSB,
3601 int &Width) {
3602 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3603
3604 EVT VT = Op.getValueType();
3605 assert((VT == MVT::i32 || VT == MVT::i64) &&
3606 "Caller guarantees VT is one of i32 or i64");
3607 (void)VT;
3608
3609 uint64_t AndImm;
3610 if (!isOpcWithIntImmediate(Op.getNode(), ISD::AND, AndImm))
3611 return false;
3612
3613 // If (~AndImm & NonZeroBits) is not zero at POS, we know that
3614 // 1) (AndImm & (1 << POS) == 0)
3615 // 2) the result of AND is not zero at POS bit (according to NonZeroBits)
3616 //
3617 // 1) and 2) don't agree so something must be wrong (e.g., in
3618 // 'SelectionDAG::computeKnownBits')
3619 assert((~AndImm & NonZeroBits) == 0 &&
3620 "Something must be wrong (e.g., in SelectionDAG::computeKnownBits)");
3621
3622 SDValue AndOp0 = Op.getOperand(0);
3623
3624 uint64_t ShlImm;
3625 SDValue ShlOp0;
3626 if (isOpcWithIntImmediate(AndOp0.getNode(), ISD::SHL, ShlImm)) {
3627 // For pattern "and(shl(val, N), shifted-mask)", 'ShlOp0' is set to 'val'.
3628 ShlOp0 = AndOp0.getOperand(0);
3629 } else if (VT == MVT::i64 && AndOp0.getOpcode() == ISD::ANY_EXTEND &&
3631 ShlImm)) {
3632 // For pattern "and(any_extend(shl(val, N)), shifted-mask)"
3633
3634 // ShlVal == shl(val, N), which is a left shift on a smaller type.
3635 SDValue ShlVal = AndOp0.getOperand(0);
3636
3637 // Since this is after type legalization and ShlVal is extended to MVT::i64,
3638 // expect VT to be MVT::i32.
3639 assert((ShlVal.getValueType() == MVT::i32) && "Expect VT to be MVT::i32.");
3640
3641 // Widens 'val' to MVT::i64 as the source of bit field positioning.
3642 ShlOp0 = Widen(CurDAG, ShlVal.getOperand(0));
3643 } else
3644 return false;
3645
3646 // For !BiggerPattern, bail out if the AndOp0 has more than one use, since
3647 // then we'll end up generating AndOp0+UBFIZ instead of just keeping
3648 // AndOp0+AND.
3649 if (!BiggerPattern && !AndOp0.hasOneUse())
3650 return false;
3651
3652 DstLSB = llvm::countr_zero(NonZeroBits);
3653 Width = llvm::countr_one(NonZeroBits >> DstLSB);
3654
3655 // Bail out on large Width. This happens when no proper combining / constant
3656 // folding was performed.
3657 if (Width >= (int)VT.getSizeInBits()) {
3658 // If VT is i64, Width > 64 is insensible since NonZeroBits is uint64_t, and
3659 // Width == 64 indicates a missed dag-combine from "(and val, AllOnes)" to
3660 // "val".
3661 // If VT is i32, what Width >= 32 means:
3662 // - For "(and (any_extend(shl val, N)), shifted-mask)", the`and` Op
3663 // demands at least 'Width' bits (after dag-combiner). This together with
3664 // `any_extend` Op (undefined higher bits) indicates missed combination
3665 // when lowering the 'and' IR instruction to an machine IR instruction.
3666 LLVM_DEBUG(
3667 dbgs()
3668 << "Found large Width in bit-field-positioning -- this indicates no "
3669 "proper combining / constant folding was performed\n");
3670 return false;
3671 }
3672
3673 // BFI encompasses sufficiently many nodes that it's worth inserting an extra
3674 // LSL/LSR if the mask in NonZeroBits doesn't quite match up with the ISD::SHL
3675 // amount. BiggerPattern is true when this pattern is being matched for BFI,
3676 // BiggerPattern is false when this pattern is being matched for UBFIZ, in
3677 // which case it is not profitable to insert an extra shift.
3678 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3679 return false;
3680
3681 Src = getLeftShift(CurDAG, ShlOp0, ShlImm - DstLSB);
3682 return true;
3683}
3684
3685// For node (shl (and val, mask), N)), returns true if the node is equivalent to
3686// UBFIZ.
3688 SDValue &Src, int &DstLSB,
3689 int &Width) {
3690 // Caller should have verified that N is a left shift with constant shift
3691 // amount; asserts that.
3692 assert(Op.getOpcode() == ISD::SHL &&
3693 "Op.getNode() should be a SHL node to call this function");
3694 assert(isIntImmediateEq(Op.getOperand(1), ShlImm) &&
3695 "Op.getNode() should shift ShlImm to call this function");
3696
3697 uint64_t AndImm = 0;
3698 SDValue Op0 = Op.getOperand(0);
3699 if (!isOpcWithIntImmediate(Op0.getNode(), ISD::AND, AndImm))
3700 return false;
3701
3702 const uint64_t ShiftedAndImm = ((AndImm << ShlImm) >> ShlImm);
3703 if (isMask_64(ShiftedAndImm)) {
3704 // AndImm is a superset of (AllOnes >> ShlImm); in other words, AndImm
3705 // should end with Mask, and could be prefixed with random bits if those
3706 // bits are shifted out.
3707 //
3708 // For example, xyz11111 (with {x,y,z} being 0 or 1) is fine if ShlImm >= 3;
3709 // the AND result corresponding to those bits are shifted out, so it's fine
3710 // to not extract them.
3711 Width = llvm::countr_one(ShiftedAndImm);
3712 DstLSB = ShlImm;
3713 Src = Op0.getOperand(0);
3714 return true;
3715 }
3716 return false;
3717}
3718
3720 bool BiggerPattern,
3721 const uint64_t NonZeroBits,
3722 SDValue &Src, int &DstLSB,
3723 int &Width) {
3724 assert(isShiftedMask_64(NonZeroBits) && "Caller guaranteed");
3725
3726 EVT VT = Op.getValueType();
3727 assert((VT == MVT::i32 || VT == MVT::i64) &&
3728 "Caller guarantees that type is i32 or i64");
3729 (void)VT;
3730
3731 uint64_t ShlImm;
3732 if (!isOpcWithIntImmediate(Op.getNode(), ISD::SHL, ShlImm))
3733 return false;
3734
3735 if (!BiggerPattern && !Op.hasOneUse())
3736 return false;
3737
3738 if (isSeveralBitsPositioningOpFromShl(ShlImm, Op, Src, DstLSB, Width))
3739 return true;
3740
3741 DstLSB = llvm::countr_zero(NonZeroBits);
3742 Width = llvm::countr_one(NonZeroBits >> DstLSB);
3743
3744 if (ShlImm != uint64_t(DstLSB) && !BiggerPattern)
3745 return false;
3746
3747 Src = getLeftShift(CurDAG, Op.getOperand(0), ShlImm - DstLSB);
3748 return true;
3749}
3750
3751static bool isShiftedMask(uint64_t Mask, EVT VT) {
3752 assert(VT == MVT::i32 || VT == MVT::i64);
3753 if (VT == MVT::i32)
3754 return isShiftedMask_32(Mask);
3755 return isShiftedMask_64(Mask);
3756}
3757
3758// Generate a BFI/BFXIL from 'or (and X, MaskImm), OrImm' iff the value being
3759// inserted only sets known zero bits.
3761 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
3762
3763 EVT VT = N->getValueType(0);
3764 if (VT != MVT::i32 && VT != MVT::i64)
3765 return false;
3766
3767 unsigned BitWidth = VT.getSizeInBits();
3768
3769 uint64_t OrImm;
3770 if (!isOpcWithIntImmediate(N, ISD::OR, OrImm))
3771 return false;
3772
3773 // Skip this transformation if the ORR immediate can be encoded in the ORR.
3774 // Otherwise, we'll trade an AND+ORR for ORR+BFI/BFXIL, which is most likely
3775 // performance neutral.
3777 return false;
3778
3779 uint64_t MaskImm;
3780 SDValue And = N->getOperand(0);
3781 // Must be a single use AND with an immediate operand.
3782 if (!And.hasOneUse() ||
3783 !isOpcWithIntImmediate(And.getNode(), ISD::AND, MaskImm))
3784 return false;
3785
3786 // Compute the Known Zero for the AND as this allows us to catch more general
3787 // cases than just looking for AND with imm.
3789
3790 // Non-zero in the sense that they're not provably zero, which is the key
3791 // point if we want to use this value.
3792 uint64_t NotKnownZero = (~Known.Zero).getZExtValue();
3793
3794 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
3795 if (!isShiftedMask(Known.Zero.getZExtValue(), VT))
3796 return false;
3797
3798 // The bits being inserted must only set those bits that are known to be zero.
3799 if ((OrImm & NotKnownZero) != 0) {
3800 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
3801 // currently handle this case.
3802 return false;
3803 }
3804
3805 // BFI/BFXIL dst, src, #lsb, #width.
3806 int LSB = llvm::countr_one(NotKnownZero);
3807 int Width = BitWidth - APInt(BitWidth, NotKnownZero).popcount();
3808
3809 // BFI/BFXIL is an alias of BFM, so translate to BFM operands.
3810 unsigned ImmR = (BitWidth - LSB) % BitWidth;
3811 unsigned ImmS = Width - 1;
3812
3813 // If we're creating a BFI instruction avoid cases where we need more
3814 // instructions to materialize the BFI constant as compared to the original
3815 // ORR. A BFXIL will use the same constant as the original ORR, so the code
3816 // should be no worse in this case.
3817 bool IsBFI = LSB != 0;
3818 uint64_t BFIImm = OrImm >> LSB;
3819 if (IsBFI && !AArch64_AM::isLogicalImmediate(BFIImm, BitWidth)) {
3820 // We have a BFI instruction and we know the constant can't be materialized
3821 // with a ORR-immediate with the zero register.
3822 unsigned OrChunks = 0, BFIChunks = 0;
3823 for (unsigned Shift = 0; Shift < BitWidth; Shift += 16) {
3824 if (((OrImm >> Shift) & 0xFFFF) != 0)
3825 ++OrChunks;
3826 if (((BFIImm >> Shift) & 0xFFFF) != 0)
3827 ++BFIChunks;
3828 }
3829 if (BFIChunks > OrChunks)
3830 return false;
3831 }
3832
3833 // Materialize the constant to be inserted.
3834 SDLoc DL(N);
3835 unsigned MOVIOpc = VT == MVT::i32 ? AArch64::MOVi32imm : AArch64::MOVi64imm;
3836 SDNode *MOVI = CurDAG->getMachineNode(
3837 MOVIOpc, DL, VT, CurDAG->getTargetConstant(BFIImm, DL, VT));
3838
3839 // Create the BFI/BFXIL instruction.
3840 SDValue Ops[] = {And.getOperand(0), SDValue(MOVI, 0),
3841 CurDAG->getTargetConstant(ImmR, DL, VT),
3842 CurDAG->getTargetConstant(ImmS, DL, VT)};
3843 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
3844 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
3845 return true;
3846}
3847
3849 SDValue &ShiftedOperand,
3850 uint64_t &EncodedShiftImm) {
3851 // Avoid folding Dst into ORR-with-shift if Dst has other uses than ORR.
3852 if (!Dst.hasOneUse())
3853 return false;
3854
3855 EVT VT = Dst.getValueType();
3856 assert((VT == MVT::i32 || VT == MVT::i64) &&
3857 "Caller should guarantee that VT is one of i32 or i64");
3858 const unsigned SizeInBits = VT.getSizeInBits();
3859
3860 SDLoc DL(Dst.getNode());
3861 uint64_t AndImm, ShlImm;
3862 if (isOpcWithIntImmediate(Dst.getNode(), ISD::AND, AndImm) &&
3863 isShiftedMask_64(AndImm)) {
3864 // Avoid transforming 'DstOp0' if it has other uses than the AND node.
3865 SDValue DstOp0 = Dst.getOperand(0);
3866 if (!DstOp0.hasOneUse())
3867 return false;
3868
3869 // An example to illustrate the transformation
3870 // From:
3871 // lsr x8, x1, #1
3872 // and x8, x8, #0x3f80
3873 // bfxil x8, x1, #0, #7
3874 // To:
3875 // and x8, x23, #0x7f
3876 // ubfx x9, x23, #8, #7
3877 // orr x23, x8, x9, lsl #7
3878 //
3879 // The number of instructions remains the same, but ORR is faster than BFXIL
3880 // on many AArch64 processors (or as good as BFXIL if not faster). Besides,
3881 // the dependency chain is improved after the transformation.
3882 uint64_t SrlImm;
3883 if (isOpcWithIntImmediate(DstOp0.getNode(), ISD::SRL, SrlImm)) {
3884 uint64_t NumTrailingZeroInShiftedMask = llvm::countr_zero(AndImm);
3885 if ((SrlImm + NumTrailingZeroInShiftedMask) < SizeInBits) {
3886 unsigned MaskWidth =
3887 llvm::countr_one(AndImm >> NumTrailingZeroInShiftedMask);
3888 unsigned UBFMOpc =
3889 (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
3890 SDNode *UBFMNode = CurDAG->getMachineNode(
3891 UBFMOpc, DL, VT, DstOp0.getOperand(0),
3892 CurDAG->getTargetConstant(SrlImm + NumTrailingZeroInShiftedMask, DL,
3893 VT),
3894 CurDAG->getTargetConstant(
3895 SrlImm + NumTrailingZeroInShiftedMask + MaskWidth - 1, DL, VT));
3896 ShiftedOperand = SDValue(UBFMNode, 0);
3897 EncodedShiftImm = AArch64_AM::getShifterImm(
3898 AArch64_AM::LSL, NumTrailingZeroInShiftedMask);
3899 return true;
3900 }
3901 }
3902 return false;
3903 }
3904
3905 if (isOpcWithIntImmediate(Dst.getNode(), ISD::SHL, ShlImm)) {
3906 ShiftedOperand = Dst.getOperand(0);
3907 EncodedShiftImm = AArch64_AM::getShifterImm(AArch64_AM::LSL, ShlImm);
3908 return true;
3909 }
3910
3911 uint64_t SrlImm;
3912 if (isOpcWithIntImmediate(Dst.getNode(), ISD::SRL, SrlImm)) {
3913 ShiftedOperand = Dst.getOperand(0);
3914 EncodedShiftImm = AArch64_AM::getShifterImm(AArch64_AM::LSR, SrlImm);
3915 return true;
3916 }
3917 return false;
3918}
3919
3920// Given an 'ISD::OR' node that is going to be selected as BFM, analyze
3921// the operands and select it to AArch64::ORR with shifted registers if
3922// that's more efficient. Returns true iff selection to AArch64::ORR happens.
3923static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1,
3924 SDValue Src, SDValue Dst, SelectionDAG *CurDAG,
3925 const bool BiggerPattern) {
3926 EVT VT = N->getValueType(0);
3927 assert(N->getOpcode() == ISD::OR && "Expect N to be an OR node");
3928 assert(((N->getOperand(0) == OrOpd0 && N->getOperand(1) == OrOpd1) ||
3929 (N->getOperand(1) == OrOpd0 && N->getOperand(0) == OrOpd1)) &&
3930 "Expect OrOpd0 and OrOpd1 to be operands of ISD::OR");
3931 assert((VT == MVT::i32 || VT == MVT::i64) &&
3932 "Expect result type to be i32 or i64 since N is combinable to BFM");
3933 SDLoc DL(N);
3934
3935 // Bail out if BFM simplifies away one node in BFM Dst.
3936 if (OrOpd1 != Dst)
3937 return false;
3938
3939 const unsigned OrrOpc = (VT == MVT::i32) ? AArch64::ORRWrs : AArch64::ORRXrs;
3940 // For "BFM Rd, Rn, #immr, #imms", it's known that BFM simplifies away fewer
3941 // nodes from Rn (or inserts additional shift node) if BiggerPattern is true.
3942 if (BiggerPattern) {
3943 uint64_t SrcAndImm;
3944 if (isOpcWithIntImmediate(OrOpd0.getNode(), ISD::AND, SrcAndImm) &&
3945 isMask_64(SrcAndImm) && OrOpd0.getOperand(0) == Src) {
3946 // OrOpd0 = AND Src, #Mask
3947 // So BFM simplifies away one AND node from Src and doesn't simplify away
3948 // nodes from Dst. If ORR with left-shifted operand also simplifies away
3949 // one node (from Rd), ORR is better since it has higher throughput and
3950 // smaller latency than BFM on many AArch64 processors (and for the rest
3951 // ORR is at least as good as BFM).
3952 SDValue ShiftedOperand;
3953 uint64_t EncodedShiftImm;
3954 if (isWorthFoldingIntoOrrWithShift(Dst, CurDAG, ShiftedOperand,
3955 EncodedShiftImm)) {
3956 SDValue Ops[] = {OrOpd0, ShiftedOperand,
3957 CurDAG->getTargetConstant(EncodedShiftImm, DL, VT)};
3958 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3959 return true;
3960 }
3961 }
3962 return false;
3963 }
3964
3965 assert((!BiggerPattern) && "BiggerPattern should be handled above");
3966
3967 uint64_t ShlImm;
3968 if (isOpcWithIntImmediate(OrOpd0.getNode(), ISD::SHL, ShlImm)) {
3969 if (OrOpd0.getOperand(0) == Src && OrOpd0.hasOneUse()) {
3970 SDValue Ops[] = {
3971 Dst, Src,
3972 CurDAG->getTargetConstant(
3974 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3975 return true;
3976 }
3977
3978 // Select the following pattern to left-shifted operand rather than BFI.
3979 // %val1 = op ..
3980 // %val2 = shl %val1, #imm
3981 // %res = or %val1, %val2
3982 //
3983 // If N is selected to be BFI, we know that
3984 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
3985 // BFI) 2) OrOpd1 would be the destination operand (i.e., preserved)
3986 //
3987 // Instead of selecting N to BFI, fold OrOpd0 as a left shift directly.
3988 if (OrOpd0.getOperand(0) == OrOpd1) {
3989 SDValue Ops[] = {
3990 OrOpd1, OrOpd1,
3991 CurDAG->getTargetConstant(
3993 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
3994 return true;
3995 }
3996 }
3997
3998 uint64_t SrlImm;
3999 if (isOpcWithIntImmediate(OrOpd0.getNode(), ISD::SRL, SrlImm)) {
4000 // Select the following pattern to right-shifted operand rather than BFXIL.
4001 // %val1 = op ..
4002 // %val2 = lshr %val1, #imm
4003 // %res = or %val1, %val2
4004 //
4005 // If N is selected to be BFXIL, we know that
4006 // 1) OrOpd0 would be the operand from which extract bits (i.e., folded into
4007 // BFXIL) 2) OrOpd1 would be the destination operand (i.e., preserved)
4008 //
4009 // Instead of selecting N to BFXIL, fold OrOpd0 as a right shift directly.
4010 if (OrOpd0.getOperand(0) == OrOpd1) {
4011 SDValue Ops[] = {
4012 OrOpd1, OrOpd1,
4013 CurDAG->getTargetConstant(
4015 CurDAG->SelectNodeTo(N, OrrOpc, VT, Ops);
4016 return true;
4017 }
4018 }
4019
4020 return false;
4021}
4022
4023static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits,
4024 SelectionDAG *CurDAG) {
4025 assert(N->getOpcode() == ISD::OR && "Expect a OR operation");
4026
4027 EVT VT = N->getValueType(0);
4028 if (VT != MVT::i32 && VT != MVT::i64)
4029 return false;
4030
4031 unsigned BitWidth = VT.getSizeInBits();
4032
4033 // Because of simplify-demanded-bits in DAGCombine, involved masks may not
4034 // have the expected shape. Try to undo that.
4035
4036 unsigned NumberOfIgnoredLowBits = UsefulBits.countr_zero();
4037 unsigned NumberOfIgnoredHighBits = UsefulBits.countl_zero();
4038
4039 // Given a OR operation, check if we have the following pattern
4040 // ubfm c, b, imm, imm2 (or something that does the same jobs, see
4041 // isBitfieldExtractOp)
4042 // d = e & mask2 ; where mask is a binary sequence of 1..10..0 and
4043 // countTrailingZeros(mask2) == imm2 - imm + 1
4044 // f = d | c
4045 // if yes, replace the OR instruction with:
4046 // f = BFM Opd0, Opd1, LSB, MSB ; where LSB = imm, and MSB = imm2
4047
4048 // OR is commutative, check all combinations of operand order and values of
4049 // BiggerPattern, i.e.
4050 // Opd0, Opd1, BiggerPattern=false
4051 // Opd1, Opd0, BiggerPattern=false
4052 // Opd0, Opd1, BiggerPattern=true
4053 // Opd1, Opd0, BiggerPattern=true
4054 // Several of these combinations may match, so check with BiggerPattern=false
4055 // first since that will produce better results by matching more instructions
4056 // and/or inserting fewer extra instructions.
4057 for (int I = 0; I < 4; ++I) {
4058
4059 SDValue Dst, Src;
4060 unsigned ImmR, ImmS;
4061 bool BiggerPattern = I / 2;
4062 SDValue OrOpd0Val = N->getOperand(I % 2);
4063 SDNode *OrOpd0 = OrOpd0Val.getNode();
4064 SDValue OrOpd1Val = N->getOperand((I + 1) % 2);
4065 SDNode *OrOpd1 = OrOpd1Val.getNode();
4066
4067 unsigned BFXOpc;
4068 int DstLSB, Width;
4069 if (isBitfieldExtractOp(CurDAG, OrOpd0, BFXOpc, Src, ImmR, ImmS,
4070 NumberOfIgnoredLowBits, BiggerPattern)) {
4071 // Check that the returned opcode is compatible with the pattern,
4072 // i.e., same type and zero extended (U and not S)
4073 if ((BFXOpc != AArch64::UBFMXri && VT == MVT::i64) ||
4074 (BFXOpc != AArch64::UBFMWri && VT == MVT::i32))
4075 continue;
4076
4077 // Compute the width of the bitfield insertion
4078 DstLSB = 0;
4079 Width = ImmS - ImmR + 1;
4080 // FIXME: This constraint is to catch bitfield insertion we may
4081 // want to widen the pattern if we want to grab general bitfield
4082 // move case
4083 if (Width <= 0)
4084 continue;
4085
4086 // If the mask on the insertee is correct, we have a BFXIL operation. We
4087 // can share the ImmR and ImmS values from the already-computed UBFM.
4088 } else if (isBitfieldPositioningOp(CurDAG, OrOpd0Val,
4089 BiggerPattern,
4090 Src, DstLSB, Width)) {
4091 ImmR = (BitWidth - DstLSB) % BitWidth;
4092 ImmS = Width - 1;
4093 } else
4094 continue;
4095
4096 // Check the second part of the pattern
4097 EVT VT = OrOpd1Val.getValueType();
4098 assert((VT == MVT::i32 || VT == MVT::i64) && "unexpected OR operand");
4099
4100 // Compute the Known Zero for the candidate of the first operand.
4101 // This allows to catch more general case than just looking for
4102 // AND with imm. Indeed, simplify-demanded-bits may have removed
4103 // the AND instruction because it proves it was useless.
4104 KnownBits Known = CurDAG->computeKnownBits(OrOpd1Val);
4105
4106 // Check if there is enough room for the second operand to appear
4107 // in the first one
4108 APInt BitsToBeInserted =
4109 APInt::getBitsSet(Known.getBitWidth(), DstLSB, DstLSB + Width);
4110
4111 if ((BitsToBeInserted & ~Known.Zero) != 0)
4112 continue;
4113
4114 // Set the first operand
4115 uint64_t Imm;
4116 if (isOpcWithIntImmediate(OrOpd1, ISD::AND, Imm) &&
4117 isBitfieldDstMask(Imm, BitsToBeInserted, NumberOfIgnoredHighBits, VT))
4118 // In that case, we can eliminate the AND
4119 Dst = OrOpd1->getOperand(0);
4120 else
4121 // Maybe the AND has been removed by simplify-demanded-bits
4122 // or is useful because it discards more bits
4123 Dst = OrOpd1Val;
4124
4125 // Before selecting ISD::OR node to AArch64::BFM, see if an AArch64::ORR
4126 // with shifted operand is more efficient.
4127 if (tryOrrWithShift(N, OrOpd0Val, OrOpd1Val, Src, Dst, CurDAG,
4128 BiggerPattern))
4129 return true;
4130
4131 // both parts match
4132 SDLoc DL(N);
4133 SDValue Ops[] = {Dst, Src, CurDAG->getTargetConstant(ImmR, DL, VT),
4134 CurDAG->getTargetConstant(ImmS, DL, VT)};
4135 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4136 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
4137 return true;
4138 }
4139
4140 // Generate a BFXIL from 'or (and X, Mask0Imm), (and Y, Mask1Imm)' iff
4141 // Mask0Imm and ~Mask1Imm are equivalent and one of the MaskImms is a shifted
4142 // mask (e.g., 0x000ffff0).
4143 uint64_t Mask0Imm, Mask1Imm;
4144 SDValue And0 = N->getOperand(0);
4145 SDValue And1 = N->getOperand(1);
4146 if (And0.hasOneUse() && And1.hasOneUse() &&
4147 isOpcWithIntImmediate(And0.getNode(), ISD::AND, Mask0Imm) &&
4148 isOpcWithIntImmediate(And1.getNode(), ISD::AND, Mask1Imm) &&
4149 APInt(BitWidth, Mask0Imm) == ~APInt(BitWidth, Mask1Imm) &&
4150 (isShiftedMask(Mask0Imm, VT) || isShiftedMask(Mask1Imm, VT))) {
4151
4152 // ORR is commutative, so canonicalize to the form 'or (and X, Mask0Imm),
4153 // (and Y, Mask1Imm)' where Mask1Imm is the shifted mask masking off the
4154 // bits to be inserted.
4155 if (isShiftedMask(Mask0Imm, VT)) {
4156 std::swap(And0, And1);
4157 std::swap(Mask0Imm, Mask1Imm);
4158 }
4159
4160 SDValue Src = And1->getOperand(0);
4161 SDValue Dst = And0->getOperand(0);
4162 unsigned LSB = llvm::countr_zero(Mask1Imm);
4163 int Width = BitWidth - APInt(BitWidth, Mask0Imm).popcount();
4164
4165 // The BFXIL inserts the low-order bits from a source register, so right
4166 // shift the needed bits into place.
4167 SDLoc DL(N);
4168 unsigned ShiftOpc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4169 uint64_t LsrImm = LSB;
4170 if (Src->hasOneUse() &&
4171 isOpcWithIntImmediate(Src.getNode(), ISD::SRL, LsrImm) &&
4172 (LsrImm + LSB) < BitWidth) {
4173 Src = Src->getOperand(0);
4174 LsrImm += LSB;
4175 }
4176
4177 SDNode *LSR = CurDAG->getMachineNode(
4178 ShiftOpc, DL, VT, Src, CurDAG->getTargetConstant(LsrImm, DL, VT),
4179 CurDAG->getTargetConstant(BitWidth - 1, DL, VT));
4180
4181 // BFXIL is an alias of BFM, so translate to BFM operands.
4182 unsigned ImmR = (BitWidth - LSB) % BitWidth;
4183 unsigned ImmS = Width - 1;
4184
4185 // Create the BFXIL instruction.
4186 SDValue Ops[] = {Dst, SDValue(LSR, 0),
4187 CurDAG->getTargetConstant(ImmR, DL, VT),
4188 CurDAG->getTargetConstant(ImmS, DL, VT)};
4189 unsigned Opc = (VT == MVT::i32) ? AArch64::BFMWri : AArch64::BFMXri;
4190 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
4191 return true;
4192 }
4193
4194 return false;
4195}
4196
4197bool AArch64DAGToDAGISel::tryBitfieldInsertOp(SDNode *N) {
4198 if (N->getOpcode() != ISD::OR)
4199 return false;
4200
4201 APInt NUsefulBits;
4202 getUsefulBits(SDValue(N, 0), NUsefulBits);
4203
4204 // If all bits are not useful, just return UNDEF.
4205 if (!NUsefulBits) {
4206 CurDAG->SelectNodeTo(N, TargetOpcode::IMPLICIT_DEF, N->getValueType(0));
4207 return true;
4208 }
4209
4210 if (tryBitfieldInsertOpFromOr(N, NUsefulBits, CurDAG))
4211 return true;
4212
4213 return tryBitfieldInsertOpFromOrAndImm(N, CurDAG);
4214}
4215
4216/// SelectBitfieldInsertInZeroOp - Match a UBFIZ instruction that is the
4217/// equivalent of a left shift by a constant amount followed by an and masking
4218/// out a contiguous set of bits.
4219bool AArch64DAGToDAGISel::tryBitfieldInsertInZeroOp(SDNode *N) {
4220 if (N->getOpcode() != ISD::AND)
4221 return false;
4222
4223 EVT VT = N->getValueType(0);
4224 if (VT != MVT::i32 && VT != MVT::i64)
4225 return false;
4226
4227 SDValue Op0;
4228 int DstLSB, Width;
4229 if (!isBitfieldPositioningOp(CurDAG, SDValue(N, 0), /*BiggerPattern=*/false,
4230 Op0, DstLSB, Width))
4231 return false;
4232
4233 // ImmR is the rotate right amount.
4234 unsigned ImmR = (VT.getSizeInBits() - DstLSB) % VT.getSizeInBits();
4235 // ImmS is the most significant bit of the source to be moved.
4236 unsigned ImmS = Width - 1;
4237
4238 SDLoc DL(N);
4239 SDValue Ops[] = {Op0, CurDAG->getTargetConstant(ImmR, DL, VT),
4240 CurDAG->getTargetConstant(ImmS, DL, VT)};
4241 unsigned Opc = (VT == MVT::i32) ? AArch64::UBFMWri : AArch64::UBFMXri;
4242 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
4243 return true;
4244}
4245
4247 SDValue &FixedPos,
4248 unsigned RegWidth,
4249 bool isReciprocal) {
4250 APFloat FVal(0.0);
4252 FVal = CN->getValueAPF();
4253 else if (LoadSDNode *LN = dyn_cast<LoadSDNode>(N)) {
4254 // Some otherwise illegal constants are allowed in this case.
4255 if (LN->getOperand(1).getOpcode() != AArch64ISD::ADDlow ||
4256 !isa<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1)))
4257 return false;
4258
4259 ConstantPoolSDNode *CN =
4260 dyn_cast<ConstantPoolSDNode>(LN->getOperand(1)->getOperand(1));
4261 FVal = cast<ConstantFP>(CN->getConstVal())->getValueAPF();
4262 } else
4263 return false;
4264
4265 if (unsigned FBits =
4266 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4267 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
4268 return true;
4269 }
4270
4271 return false;
4272}
4273
4275 SDValue N,
4276 SDValue &FixedPos,
4277 unsigned RegWidth,
4278 bool isReciprocal) {
4279 if ((N.getOpcode() == AArch64ISD::NVCAST || N.getOpcode() == ISD::BITCAST) &&
4280 N.getValueType().getScalarSizeInBits() ==
4281 N.getOperand(0).getValueType().getScalarSizeInBits())
4282 N = N.getOperand(0);
4283
4284 auto ImmToFloat = [RegWidth](APInt Imm) {
4285 switch (RegWidth) {
4286 case 16:
4287 return APFloat(APFloat::IEEEhalf(), Imm);
4288 case 32:
4289 return APFloat(APFloat::IEEEsingle(), Imm);
4290 case 64:
4291 return APFloat(APFloat::IEEEdouble(), Imm);
4292 default:
4293 llvm_unreachable("Unexpected RegWidth!");
4294 };
4295 };
4296
4297 APFloat FVal(0.0);
4298 switch (N->getOpcode()) {
4299 case AArch64ISD::MOVIshift:
4300 FVal = ImmToFloat(APInt(RegWidth, N.getConstantOperandVal(0)
4301 << N.getConstantOperandVal(1)));
4302 break;
4303 case AArch64ISD::FMOV:
4304 FVal = ImmToFloat(DecodeFMOVImm(N.getConstantOperandVal(0), RegWidth));
4305 break;
4306 case AArch64ISD::DUP:
4307 if (isa<ConstantSDNode>(N.getOperand(0)))
4308 FVal = ImmToFloat(N.getConstantOperandAPInt(0).trunc(RegWidth));
4309 else
4310 return false;
4311 break;
4312 default:
4313 return false;
4314 }
4315
4316 if (unsigned FBits =
4317 CheckFixedPointOperandConstant(FVal, RegWidth, isReciprocal)) {
4318 FixedPos = CurDAG->getTargetConstant(FBits, SDLoc(N), MVT::i32);
4319 return true;
4320 }
4321
4322 return false;
4323}
4324
4325bool AArch64DAGToDAGISel::SelectCVTFixedPosOperand(SDValue N, SDValue &FixedPos,
4326 unsigned RegWidth) {
4327 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4328 /*isReciprocal*/ false);
4329}
4330
4331bool AArch64DAGToDAGISel::SelectCVTFixedPointVec(SDValue N, SDValue &FixedPos,
4332 unsigned RegWidth) {
4334 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ false);
4335}
4336
4337bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperandVec(SDValue N,
4338 SDValue &FixedPos,
4339 unsigned RegWidth) {
4341 CurDAG, N, FixedPos, RegWidth, /*isReciprocal*/ true);
4342}
4343
4344bool AArch64DAGToDAGISel::SelectCVTFixedPosRecipOperand(SDValue N,
4345 SDValue &FixedPos,
4346 unsigned RegWidth) {
4347 return checkCVTFixedPointOperandWithFBits(CurDAG, N, FixedPos, RegWidth,
4348 /*isReciprocal*/ true);
4349}
4350
4351// Inspects a register string of the form o0:op1:CRn:CRm:op2 gets the fields
4352// of the string and obtains the integer values from them and combines these
4353// into a single value to be used in the MRS/MSR instruction.
4356 RegString.split(Fields, ':');
4357
4358 if (Fields.size() == 1)
4359 return -1;
4360
4361 assert(Fields.size() == 5
4362 && "Invalid number of fields in read register string");
4363
4365 bool AllIntFields = true;
4366
4367 for (StringRef Field : Fields) {
4368 unsigned IntField;
4369 AllIntFields &= !Field.getAsInteger(10, IntField);
4370 Ops.push_back(IntField);
4371 }
4372
4373 assert(AllIntFields &&
4374 "Unexpected non-integer value in special register string.");
4375 (void)AllIntFields;
4376
4377 // Need to combine the integer fields of the string into a single value
4378 // based on the bit encoding of MRS/MSR instruction.
4379 return (Ops[0] << 14) | (Ops[1] << 11) | (Ops[2] << 7) | (Ops[3] << 3) |
4380 (Ops[4]);
4381}
4382
4383// Lower the read_register intrinsic to an MRS instruction node if the special
4384// register string argument is either of the form detailed in the ALCE (the
4385// form described in getIntOperandsFromRegisterString) or is a named register
4386// known by the MRS SysReg mapper.
4387bool AArch64DAGToDAGISel::tryReadRegister(SDNode *N) {
4388 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
4389 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4390 SDLoc DL(N);
4391
4392 bool ReadIs128Bit = N->getOpcode() == AArch64ISD::MRRS;
4393
4394 unsigned Opcode64Bit = AArch64::MRS;
4395 int Imm = getIntOperandFromRegisterString(RegString->getString());
4396 if (Imm == -1) {
4397 // No match, Use the sysreg mapper to map the remaining possible strings to
4398 // the value for the register to be used for the instruction operand.
4399 const auto *TheReg =
4400 AArch64SysReg::lookupSysRegByName(RegString->getString());
4401 if (TheReg && TheReg->Readable &&
4402 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4403 Imm = TheReg->Encoding;
4404 else
4405 Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
4406
4407 if (Imm == -1) {
4408 // Still no match, see if this is "pc" or give up.
4409 if (!ReadIs128Bit && RegString->getString() == "pc") {
4410 Opcode64Bit = AArch64::ADR;
4411 Imm = 0;
4412 } else {
4413 // Not a system register. It may name an allocatable 64-bit GPR/FPR read
4414 // by the MSVC __getReg/__getRegFp intrinsics. Emit a pseudo that
4415 // carries the source register as an immediate so the read does not
4416 // reference an undefined physical register (which the machine verifier
4417 // rejects); the AsmPrinter materializes the real mov/fmov.
4419 RegString->getString());
4420 unsigned PseudoOp = 0;
4421 if (AArch64::GPR64RegClass.contains(PReg))
4422 PseudoOp = AArch64::READ_REGISTER_GPR64;
4423 else if (AArch64::FPR64RegClass.contains(PReg))
4424 PseudoOp = AArch64::READ_REGISTER_FPR64;
4425 if (!ReadIs128Bit && PseudoOp && N->getValueType(0) == MVT::i64) {
4426 CurDAG->SelectNodeTo(N, PseudoOp, MVT::i64, MVT::Other,
4427 {CurDAG->getTargetConstant(PReg, DL, MVT::i32),
4428 N->getOperand(0)});
4429 return true;
4430 }
4431 return false;
4432 }
4433 }
4434 }
4435
4436 SDValue InChain = N->getOperand(0);
4437 SDValue SysRegImm = CurDAG->getTargetConstant(Imm, DL, MVT::i32);
4438 if (!ReadIs128Bit) {
4439 CurDAG->SelectNodeTo(N, Opcode64Bit, MVT::i64, MVT::Other /* Chain */,
4440 {SysRegImm, InChain});
4441 } else {
4442 SDNode *MRRS = CurDAG->getMachineNode(
4443 AArch64::MRRS, DL,
4444 {MVT::Untyped /* XSeqPair */, MVT::Other /* Chain */},
4445 {SysRegImm, InChain});
4446
4447 // Sysregs are not endian. The even register always contains the low half
4448 // of the register.
4449 SDValue Lo = CurDAG->getTargetExtractSubreg(AArch64::sube64, DL, MVT::i64,
4450 SDValue(MRRS, 0));
4451 SDValue Hi = CurDAG->getTargetExtractSubreg(AArch64::subo64, DL, MVT::i64,
4452 SDValue(MRRS, 0));
4453 SDValue OutChain = SDValue(MRRS, 1);
4454
4455 ReplaceUses(SDValue(N, 0), Lo);
4456 ReplaceUses(SDValue(N, 1), Hi);
4457 ReplaceUses(SDValue(N, 2), OutChain);
4458 };
4459 return true;
4460}
4461
4462// Lower the write_register intrinsic to an MSR instruction node if the special
4463// register string argument is either of the form detailed in the ALCE (the
4464// form described in getIntOperandsFromRegisterString) or is a named register
4465// known by the MSR SysReg mapper.
4466bool AArch64DAGToDAGISel::tryWriteRegister(SDNode *N) {
4467 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
4468 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
4469 SDLoc DL(N);
4470
4471 bool WriteIs128Bit = N->getOpcode() == AArch64ISD::MSRR;
4472
4473 if (!WriteIs128Bit) {
4474 // Check if the register was one of those allowed as the pstatefield value
4475 // in the MSR (immediate) instruction. To accept the values allowed in the
4476 // pstatefield for the MSR (immediate) instruction, we also require that an
4477 // immediate value has been provided as an argument, we know that this is
4478 // the case as it has been ensured by semantic checking.
4479 auto trySelectPState = [&](auto PMapper, unsigned State) {
4480 if (PMapper) {
4481 assert(isa<ConstantSDNode>(N->getOperand(2)) &&
4482 "Expected a constant integer expression.");
4483 unsigned Reg = PMapper->Encoding;
4484 uint64_t Immed = N->getConstantOperandVal(2);
4485 CurDAG->SelectNodeTo(
4486 N, State, MVT::Other, CurDAG->getTargetConstant(Reg, DL, MVT::i32),
4487 CurDAG->getTargetConstant(Immed, DL, MVT::i16), N->getOperand(0));
4488 return true;
4489 }
4490 return false;
4491 };
4492
4493 if (trySelectPState(
4494 AArch64PState::lookupPStateImm0_15ByName(RegString->getString()),
4495 AArch64::MSRpstateImm4))
4496 return true;
4497 if (trySelectPState(
4498 AArch64PState::lookupPStateImm0_1ByName(RegString->getString()),
4499 AArch64::MSRpstateImm1))
4500 return true;
4501 }
4502
4503 int Imm = getIntOperandFromRegisterString(RegString->getString());
4504 if (Imm == -1) {
4505 // Use the sysreg mapper to attempt to map the remaining possible strings
4506 // to the value for the register to be used for the MSR (register)
4507 // instruction operand.
4508 auto TheReg = AArch64SysReg::lookupSysRegByName(RegString->getString());
4509 if (TheReg && TheReg->Writeable &&
4510 TheReg->haveFeatures(Subtarget->getFeatureBits()))
4511 Imm = TheReg->Encoding;
4512 else
4513 Imm = AArch64SysReg::parseGenericRegister(RegString->getString());
4514
4515 if (Imm == -1) {
4516 // Used by the MSVC __setReg/__setRegFp intrinsics. Copy the value into
4517 // the physical register and keep it live with a FAKE_USE so the write is
4518 // not dead-eliminated. (getRegisterByName rejects allocatable registers,
4519 // so the generic write path cannot handle these.)
4521 RegString->getString());
4522 bool IsGPR = AArch64::GPR64RegClass.contains(PReg);
4523 bool IsFPR = AArch64::FPR64RegClass.contains(PReg);
4524 if (!WriteIs128Bit && (IsGPR || IsFPR) &&
4525 N->getOperand(2).getValueType() == MVT::i64) {
4526 SDValue Copy =
4527 CurDAG->getCopyToReg(N->getOperand(0), DL, PReg, N->getOperand(2));
4528 SDValue RegOp = CurDAG->getRegister(PReg, MVT::i64);
4529 SDNode *FakeUse = CurDAG->getMachineNode(TargetOpcode::FAKE_USE, DL,
4530 MVT::Other, {RegOp, Copy});
4531 ReplaceUses(SDValue(N, 0), SDValue(FakeUse, 0));
4532 CurDAG->RemoveDeadNode(N);
4533 return true;
4534 }
4535 return false;
4536 }
4537 }
4538
4539 SDValue InChain = N->getOperand(0);
4540 if (!WriteIs128Bit) {
4541 CurDAG->SelectNodeTo(N, AArch64::MSR, MVT::Other,
4542 CurDAG->getTargetConstant(Imm, DL, MVT::i32),
4543 N->getOperand(2), InChain);
4544 } else {
4545 // No endian swap. The lower half always goes into the even subreg, and the
4546 // higher half always into the odd supreg.
4547 SDNode *Pair = CurDAG->getMachineNode(
4548 TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped /* XSeqPair */,
4549 {CurDAG->getTargetConstant(AArch64::XSeqPairsClassRegClass.getID(), DL,
4550 MVT::i32),
4551 N->getOperand(2),
4552 CurDAG->getTargetConstant(AArch64::sube64, DL, MVT::i32),
4553 N->getOperand(3),
4554 CurDAG->getTargetConstant(AArch64::subo64, DL, MVT::i32)});
4555
4556 CurDAG->SelectNodeTo(N, AArch64::MSRR, MVT::Other,
4557 CurDAG->getTargetConstant(Imm, DL, MVT::i32),
4558 SDValue(Pair, 0), InChain);
4559 }
4560
4561 return true;
4562}
4563
4564/// We've got special pseudo-instructions for these
4565bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
4566 unsigned Opcode;
4567 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
4568
4569 // Leave IR for LSE if subtarget supports it.
4570 if (Subtarget->hasLSE()) return false;
4571
4572 if (MemTy == MVT::i8)
4573 Opcode = AArch64::CMP_SWAP_8;
4574 else if (MemTy == MVT::i16)
4575 Opcode = AArch64::CMP_SWAP_16;
4576 else if (MemTy == MVT::i32)
4577 Opcode = AArch64::CMP_SWAP_32;
4578 else if (MemTy == MVT::i64)
4579 Opcode = AArch64::CMP_SWAP_64;
4580 else
4581 llvm_unreachable("Unknown AtomicCmpSwap type");
4582
4583 MVT RegTy = MemTy == MVT::i64 ? MVT::i64 : MVT::i32;
4584 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
4585 N->getOperand(0)};
4586 SDNode *CmpSwap = CurDAG->getMachineNode(
4587 Opcode, SDLoc(N),
4588 CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops);
4589
4590 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
4591 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
4592
4593 ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
4594 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
4595 CurDAG->RemoveDeadNode(N);
4596
4597 return true;
4598}
4599
4601AArch64DAGToDAGISel::decodeMemoryHintFlags(MachineMemOperand *MMO) const {
4602 int MemoryHint = -1;
4603 const MDNode *MemCacheHint = MMO->getMemCacheHint();
4604 if (!MemCacheHint)
4605 return AArch64MemoryHint::NONE;
4606
4607 for (unsigned I = 0; I + 1 < MemCacheHint->getNumOperands(); I += 2) {
4608 if (MemCacheHint->getOperand(I).equalsStr("aarch64.mem_hint")) {
4609 const Metadata *Val = MemCacheHint->getOperand(I + 1).get();
4611 ->getZExtValue();
4612 }
4613 }
4614
4615 return toAArch64MemoryHint(MemoryHint);
4616}
4617
4618bool AArch64DAGToDAGISel::isAtomicMemoryHint(SDNode *N,
4619 AArch64MemoryHint Hint) const {
4620 return decodeMemoryHintFlags(cast<MemSDNode>(N)->getMemOperand()) == Hint;
4621}
4622
4623bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDValue N, MVT VT, SDValue &Imm,
4624 SDValue &Shift, bool Negate) {
4625 if (!isa<ConstantSDNode>(N))
4626 return false;
4627
4628 APInt Val =
4629 cast<ConstantSDNode>(N)->getAPIntValue().trunc(VT.getFixedSizeInBits());
4630
4631 return SelectSVEAddSubImm(SDLoc(N), Val, VT, Imm, Shift, Negate);
4632}
4633
4634bool AArch64DAGToDAGISel::SelectSVEAddSubImm(SDLoc DL, APInt Val, MVT VT,
4635 SDValue &Imm, SDValue &Shift,
4636 bool Negate) {
4637 if (Negate)
4638 Val = -Val;
4639
4640 switch (VT.SimpleTy) {
4641 case MVT::i8:
4642 // All immediates are supported.
4643 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4644 Imm = CurDAG->getTargetConstant(Val.getZExtValue(), DL, MVT::i32);
4645 return true;
4646 case MVT::i16:
4647 case MVT::i32:
4648 case MVT::i64:
4649 // Support 8bit unsigned immediates.
4650 if ((Val & ~0xff) == 0) {
4651 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4652 Imm = CurDAG->getTargetConstant(Val.getZExtValue(), DL, MVT::i32);
4653 return true;
4654 }
4655 // Support 16bit unsigned immediates that are a multiple of 256.
4656 if ((Val & ~0xff00) == 0) {
4657 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
4658 Imm = CurDAG->getTargetConstant(Val.lshr(8).getZExtValue(), DL, MVT::i32);
4659 return true;
4660 }
4661 break;
4662 default:
4663 break;
4664 }
4665
4666 return false;
4667}
4668
4669bool AArch64DAGToDAGISel::SelectSVEAddSubSSatImm(SDValue N, MVT VT,
4670 SDValue &Imm, SDValue &Shift,
4671 bool Negate) {
4672 if (!isa<ConstantSDNode>(N))
4673 return false;
4674
4675 SDLoc DL(N);
4676 int64_t Val = cast<ConstantSDNode>(N)
4677 ->getAPIntValue()
4679 .getSExtValue();
4680
4681 if (Negate)
4682 Val = -Val;
4683
4684 // Signed saturating instructions treat their immediate operand as unsigned,
4685 // whereas the related intrinsics define their operands to be signed. This
4686 // means we can only use the immediate form when the operand is non-negative.
4687 if (Val < 0)
4688 return false;
4689
4690 switch (VT.SimpleTy) {
4691 case MVT::i8:
4692 // All positive immediates are supported.
4693 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4694 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4695 return true;
4696 case MVT::i16:
4697 case MVT::i32:
4698 case MVT::i64:
4699 // Support 8bit positive immediates.
4700 if (Val <= 255) {
4701 Shift = CurDAG->getTargetConstant(0, DL, MVT::i32);
4702 Imm = CurDAG->getTargetConstant(Val, DL, MVT::i32);
4703 return true;
4704 }
4705 // Support 16bit positive immediates that are a multiple of 256.
4706 if (Val <= 65280 && Val % 256 == 0) {
4707 Shift = CurDAG->getTargetConstant(8, DL, MVT::i32);
4708 Imm = CurDAG->getTargetConstant(Val >> 8, DL, MVT::i32);
4709 return true;
4710 }
4711 break;
4712 default:
4713 break;
4714 }
4715
4716 return false;
4717}
4718
4719bool AArch64DAGToDAGISel::SelectSVECpyDupImm(SDValue N, MVT VT, SDValue &Imm,
4720 SDValue &Shift) {
4721 if (!isa<ConstantSDNode>(N))
4722 return false;
4723
4724 SDLoc DL(N);
4725 int64_t Val = cast<ConstantSDNode>(N)
4726 ->getAPIntValue()
4727 .trunc(VT.getFixedSizeInBits())
4728 .getSExtValue();
4729 int32_t ImmVal, ShiftVal;
4730 if (!AArch64_AM::isSVECpyDupImm(VT.getScalarSizeInBits(), Val, ImmVal,
4731 ShiftVal))
4732 return false;
4733
4734 Shift = CurDAG->getTargetConstant(ShiftVal, DL, MVT::i32);
4735 Imm = CurDAG->getTargetConstant(ImmVal, DL, MVT::i32);
4736 return true;
4737}
4738
4739bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDValue N, SDValue &Imm) {
4740 if (auto CNode = dyn_cast<ConstantSDNode>(N))
4741 return SelectSVESignedArithImm(SDLoc(N), CNode->getAPIntValue(), Imm);
4742 return false;
4743}
4744
4745bool AArch64DAGToDAGISel::SelectSVESignedArithImm(SDLoc DL, APInt Val,
4746 SDValue &Imm) {
4747 int64_t ImmVal = Val.getSExtValue();
4748 if (ImmVal >= -128 && ImmVal < 128) {
4749 Imm = CurDAG->getSignedTargetConstant(ImmVal, DL, MVT::i32);
4750 return true;
4751 }
4752 return false;
4753}
4754
4755bool AArch64DAGToDAGISel::SelectSVEArithImm(SDValue N, MVT VT, SDValue &Imm) {
4756 if (auto CNode = dyn_cast<ConstantSDNode>(N)) {
4757 uint64_t ImmVal = CNode->getZExtValue();
4758
4759 switch (VT.SimpleTy) {
4760 case MVT::i8:
4761 ImmVal &= 0xFF;
4762 break;
4763 case MVT::i16:
4764 ImmVal &= 0xFFFF;
4765 break;
4766 case MVT::i32:
4767 ImmVal &= 0xFFFFFFFF;
4768 break;
4769 case MVT::i64:
4770 break;
4771 default:
4772 llvm_unreachable("Unexpected type");
4773 }
4774
4775 if (ImmVal < 256) {
4776 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32);
4777 return true;
4778 }
4779 }
4780 return false;
4781}
4782
4783bool AArch64DAGToDAGISel::SelectSVELogicalImm(SDValue N, MVT VT, SDValue &Imm,
4784 bool Invert) {
4785 uint64_t ImmVal;
4786 if (auto CI = dyn_cast<ConstantSDNode>(N))
4787 ImmVal = CI->getZExtValue();
4788 else if (auto CFP = dyn_cast<ConstantFPSDNode>(N))
4789 ImmVal = CFP->getValueAPF().bitcastToAPInt().getZExtValue();
4790 else
4791 return false;
4792
4793 if (Invert)
4794 ImmVal = ~ImmVal;
4795
4796 uint64_t encoding;
4797 if (!AArch64_AM::isSVELogicalImm(VT.getScalarSizeInBits(), ImmVal, encoding))
4798 return false;
4799
4800 Imm = CurDAG->getTargetConstant(encoding, SDLoc(N), MVT::i64);
4801 return true;
4802}
4803
4804// SVE shift intrinsics allow shift amounts larger than the element's bitwidth.
4805// Rather than attempt to normalise everything we can sometimes saturate the
4806// shift amount during selection. This function also allows for consistent
4807// isel patterns by ensuring the resulting "Imm" node is of the i32 type
4808// required by the instructions.
4809bool AArch64DAGToDAGISel::SelectSVEShiftImm(SDValue N, uint64_t Low,
4810 uint64_t High, bool AllowSaturation,
4811 SDValue &Imm) {
4812 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
4813 uint64_t ImmVal = CN->getZExtValue();
4814
4815 // Reject shift amounts that are too small.
4816 if (ImmVal < Low)
4817 return false;
4818
4819 // Reject or saturate shift amounts that are too big.
4820 if (ImmVal > High) {
4821 if (!AllowSaturation)
4822 return false;
4823 ImmVal = High;
4824 }
4825
4826 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), MVT::i32);
4827 return true;
4828 }
4829
4830 return false;
4831}
4832
4833bool AArch64DAGToDAGISel::trySelectStackSlotTagP(SDNode *N) {
4834 // tagp(FrameIndex, IRGstack, tag_offset):
4835 // since the offset between FrameIndex and IRGstack is a compile-time
4836 // constant, this can be lowered to a single ADDG instruction.
4837 if (!(isa<FrameIndexSDNode>(N->getOperand(1)))) {
4838 return false;
4839 }
4840
4841 SDValue IRG_SP = N->getOperand(2);
4842 if (IRG_SP->getOpcode() != ISD::INTRINSIC_W_CHAIN ||
4843 IRG_SP->getConstantOperandVal(1) != Intrinsic::aarch64_irg_sp) {
4844 return false;
4845 }
4846
4847 const TargetLowering *TLI = getTargetLowering();
4848 SDLoc DL(N);
4849 int FI = cast<FrameIndexSDNode>(N->getOperand(1))->getIndex();
4850 SDValue FiOp = CurDAG->getTargetFrameIndex(
4851 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
4852 int TagOffset = N->getConstantOperandVal(3);
4853
4854 SDNode *Out = CurDAG->getMachineNode(
4855 AArch64::TAGPstack, DL, MVT::i64,
4856 {FiOp, CurDAG->getTargetConstant(0, DL, MVT::i64), N->getOperand(2),
4857 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
4858 ReplaceNode(N, Out);
4859 return true;
4860}
4861
4862void AArch64DAGToDAGISel::SelectTagP(SDNode *N) {
4863 assert(isa<ConstantSDNode>(N->getOperand(3)) &&
4864 "llvm.aarch64.tagp third argument must be an immediate");
4865 if (trySelectStackSlotTagP(N))
4866 return;
4867 // FIXME: above applies in any case when offset between Op1 and Op2 is a
4868 // compile-time constant, not just for stack allocations.
4869
4870 // General case for unrelated pointers in Op1 and Op2.
4871 SDLoc DL(N);
4872 int TagOffset = N->getConstantOperandVal(3);
4873 SDNode *N1 = CurDAG->getMachineNode(AArch64::SUBP, DL, MVT::i64,
4874 {N->getOperand(1), N->getOperand(2)});
4875 SDNode *N2 = CurDAG->getMachineNode(AArch64::ADDXrr, DL, MVT::i64,
4876 {SDValue(N1, 0), N->getOperand(2)});
4877 SDNode *N3 = CurDAG->getMachineNode(
4878 AArch64::ADDG, DL, MVT::i64,
4879 {SDValue(N2, 0), CurDAG->getTargetConstant(0, DL, MVT::i64),
4880 CurDAG->getTargetConstant(TagOffset, DL, MVT::i64)});
4881 ReplaceNode(N, N3);
4882}
4883
4884bool AArch64DAGToDAGISel::trySelectCastFixedLengthToScalableVector(SDNode *N) {
4885 assert(N->getOpcode() == ISD::INSERT_SUBVECTOR && "Invalid Node!");
4886
4887 // Bail when not a "cast" like insert_subvector.
4888 if (N->getConstantOperandVal(2) != 0)
4889 return false;
4890 if (!N->getOperand(0).isUndef())
4891 return false;
4892
4893 // Bail when normal isel should do the job.
4894 EVT VT = N->getValueType(0);
4895 EVT InVT = N->getOperand(1).getValueType();
4896 if (VT.isFixedLengthVector() || InVT.isScalableVector())
4897 return false;
4898 if (InVT.getSizeInBits() <= 128)
4899 return false;
4900
4901 // NOTE: We can only get here when doing fixed length SVE code generation.
4902 // We do manual selection because the types involved are not linked to real
4903 // registers (despite being legal) and must be coerced into SVE registers.
4904
4906 "Expected to insert into a packed scalable vector!");
4907
4908 SDLoc DL(N);
4909 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
4910 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT,
4911 N->getOperand(1), RC));
4912 return true;
4913}
4914
4915bool AArch64DAGToDAGISel::trySelectCastScalableToFixedLengthVector(SDNode *N) {
4916 assert(N->getOpcode() == ISD::EXTRACT_SUBVECTOR && "Invalid Node!");
4917
4918 // Bail when not a "cast" like extract_subvector.
4919 if (N->getConstantOperandVal(1) != 0)
4920 return false;
4921
4922 // Bail when normal isel can do the job.
4923 EVT VT = N->getValueType(0);
4924 EVT InVT = N->getOperand(0).getValueType();
4925 if (VT.isScalableVector() || InVT.isFixedLengthVector())
4926 return false;
4927 if (VT.getSizeInBits() <= 128)
4928 return false;
4929
4930 // NOTE: We can only get here when doing fixed length SVE code generation.
4931 // We do manual selection because the types involved are not linked to real
4932 // registers (despite being legal) and must be coerced into SVE registers.
4933
4935 "Expected to extract from a packed scalable vector!");
4936
4937 SDLoc DL(N);
4938 auto RC = CurDAG->getTargetConstant(AArch64::ZPRRegClassID, DL, MVT::i64);
4939 ReplaceNode(N, CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT,
4940 N->getOperand(0), RC));
4941 return true;
4942}
4943
4944bool AArch64DAGToDAGISel::trySelectXAR(SDNode *N) {
4945 assert(N->getOpcode() == ISD::OR && "Expected OR instruction");
4946
4947 SDValue N0 = N->getOperand(0);
4948 SDValue N1 = N->getOperand(1);
4949
4950 EVT VT = N->getValueType(0);
4951 SDLoc DL(N);
4952
4953 // Essentially: rotr (xor(x, y), imm) -> xar (x, y, imm)
4954 // Rotate by a constant is a funnel shift in IR which is expanded to
4955 // an OR with shifted operands.
4956 // We do the following transform:
4957 // OR N0, N1 -> xar (x, y, imm)
4958 // Where:
4959 // N1 = SRL_PRED true, V, splat(imm) --> rotr amount
4960 // N0 = SHL_PRED true, V, splat(bits-imm)
4961 // V = (xor x, y)
4962 if (VT.isScalableVector() &&
4963 (Subtarget->hasSVE2() ||
4964 (Subtarget->hasSME() && Subtarget->isStreaming()))) {
4965 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4966 N1.getOpcode() != AArch64ISD::SRL_PRED)
4967 std::swap(N0, N1);
4968 if (N0.getOpcode() != AArch64ISD::SHL_PRED ||
4969 N1.getOpcode() != AArch64ISD::SRL_PRED)
4970 return false;
4971
4972 auto *TLI = static_cast<const AArch64TargetLowering *>(getTargetLowering());
4973 if (!TLI->isAllActivePredicate(*CurDAG, N0.getOperand(0)) ||
4974 !TLI->isAllActivePredicate(*CurDAG, N1.getOperand(0)))
4975 return false;
4976
4977 if (N0.getOperand(1) != N1.getOperand(1))
4978 return false;
4979
4980 SDValue R1, R2;
4981 bool IsXOROperand = true;
4982 if (N0.getOperand(1).getOpcode() != ISD::XOR) {
4983 IsXOROperand = false;
4984 } else {
4985 R1 = N0.getOperand(1).getOperand(0);
4986 R2 = N1.getOperand(1).getOperand(1);
4987 }
4988
4989 APInt ShlAmt, ShrAmt;
4990 if (!ISD::isConstantSplatVector(N0.getOperand(2), ShlAmt) ||
4991 !ISD::isConstantSplatVector(N1.getOperand(2), ShrAmt))
4992 return false;
4993
4994 if (ShlAmt + ShrAmt != VT.getScalarSizeInBits())
4995 return false;
4996
4997 if (!IsXOROperand) {
4998 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i64);
4999 SDNode *MOV = CurDAG->getMachineNode(AArch64::MOVIv2d_ns, DL, VT, Zero);
5000 SDValue MOVIV = SDValue(MOV, 0);
5001
5002 SDValue ZSub = CurDAG->getTargetConstant(AArch64::zsub, DL, MVT::i32);
5003 SDNode *SubRegToReg =
5004 CurDAG->getMachineNode(AArch64::SUBREG_TO_REG, DL, VT, MOVIV, ZSub);
5005
5006 R1 = N1->getOperand(1);
5007 R2 = SDValue(SubRegToReg, 0);
5008 }
5009
5010 SDValue Imm =
5011 CurDAG->getTargetConstant(ShrAmt.getZExtValue(), DL, MVT::i32);
5012
5013 SDValue Ops[] = {R1, R2, Imm};
5015 VT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5016 AArch64::XAR_ZZZI_D})) {
5017 CurDAG->SelectNodeTo(N, Opc, VT, Ops);
5018 return true;
5019 }
5020 return false;
5021 }
5022
5023 // We have Neon SHA3 XAR operation for v2i64 but for types
5024 // v4i32, v8i16, v16i8 we can use SVE operations when SVE2-SHA3
5025 // is available.
5026 EVT SVT;
5027 switch (VT.getSimpleVT().SimpleTy) {
5028 case MVT::v4i32:
5029 case MVT::v2i32:
5030 SVT = MVT::nxv4i32;
5031 break;
5032 case MVT::v8i16:
5033 case MVT::v4i16:
5034 SVT = MVT::nxv8i16;
5035 break;
5036 case MVT::v16i8:
5037 case MVT::v8i8:
5038 SVT = MVT::nxv16i8;
5039 break;
5040 case MVT::v2i64:
5041 case MVT::v1i64:
5042 SVT = Subtarget->hasSHA3() ? MVT::v2i64 : MVT::nxv2i64;
5043 break;
5044 default:
5045 return false;
5046 }
5047
5048 if ((!SVT.isScalableVector() && !Subtarget->hasSHA3()) ||
5049 (SVT.isScalableVector() && !Subtarget->hasSVE2()))
5050 return false;
5051
5052 if (N0->getOpcode() != AArch64ISD::VSHL ||
5053 N1->getOpcode() != AArch64ISD::VLSHR)
5054 return false;
5055
5056 if (N0->getOperand(0) != N1->getOperand(0))
5057 return false;
5058
5059 SDValue R1, R2;
5060 bool IsXOROperand = true;
5061 if (N1->getOperand(0)->getOpcode() != ISD::XOR) {
5062 IsXOROperand = false;
5063 } else {
5064 SDValue XOR = N0.getOperand(0);
5065 R1 = XOR.getOperand(0);
5066 R2 = XOR.getOperand(1);
5067 }
5068
5069 unsigned HsAmt = N0.getConstantOperandVal(1);
5070 unsigned ShAmt = N1.getConstantOperandVal(1);
5071
5072 SDValue Imm = CurDAG->getTargetConstant(
5073 ShAmt, DL, N0.getOperand(1).getValueType(), false);
5074
5075 unsigned VTSizeInBits = VT.getScalarSizeInBits();
5076 if (ShAmt + HsAmt != VTSizeInBits)
5077 return false;
5078
5079 if (!IsXOROperand) {
5080 SDValue Zero = CurDAG->getTargetConstant(0, DL, MVT::i64);
5081 SDNode *MOV =
5082 CurDAG->getMachineNode(AArch64::MOVIv2d_ns, DL, MVT::v2i64, Zero);
5083 SDValue MOVIV = SDValue(MOV, 0);
5084
5085 R1 = N1->getOperand(0);
5086 R2 = MOVIV;
5087 }
5088
5089 if (SVT != VT) {
5090 SDValue Undef =
5091 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, SVT), 0);
5092
5093 if (SVT.isScalableVector() && VT.is64BitVector()) {
5094 EVT QVT = VT.getDoubleNumVectorElementsVT(*CurDAG->getContext());
5095
5096 SDValue UndefQ = SDValue(
5097 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, QVT), 0);
5098 SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
5099
5100 R1 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, QVT,
5101 UndefQ, R1, DSub),
5102 0);
5103 if (R2.getValueType() == VT)
5104 R2 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, QVT,
5105 UndefQ, R2, DSub),
5106 0);
5107 }
5108
5109 SDValue SubReg = CurDAG->getTargetConstant(
5110 (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL, MVT::i32);
5111
5112 R1 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, SVT, Undef,
5113 R1, SubReg),
5114 0);
5115
5116 if (SVT.isScalableVector() || R2.getValueType() != SVT)
5117 R2 = SDValue(CurDAG->getMachineNode(AArch64::INSERT_SUBREG, DL, SVT,
5118 Undef, R2, SubReg),
5119 0);
5120 }
5121
5122 SDValue Ops[] = {R1, R2, Imm};
5123 SDNode *XAR = nullptr;
5124
5125 if (SVT.isScalableVector()) {
5127 SVT, {AArch64::XAR_ZZZI_B, AArch64::XAR_ZZZI_H, AArch64::XAR_ZZZI_S,
5128 AArch64::XAR_ZZZI_D}))
5129 XAR = CurDAG->getMachineNode(Opc, DL, SVT, Ops);
5130 } else {
5131 XAR = CurDAG->getMachineNode(AArch64::XAR, DL, SVT, Ops);
5132 }
5133
5134 assert(XAR && "Unexpected NULL value for XAR instruction in DAG");
5135
5136 if (SVT != VT) {
5137 if (VT.is64BitVector() && SVT.isScalableVector()) {
5138 EVT QVT = VT.getDoubleNumVectorElementsVT(*CurDAG->getContext());
5139
5140 SDValue ZSub = CurDAG->getTargetConstant(AArch64::zsub, DL, MVT::i32);
5141 SDNode *Q = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, QVT,
5142 SDValue(XAR, 0), ZSub);
5143
5144 SDValue DSub = CurDAG->getTargetConstant(AArch64::dsub, DL, MVT::i32);
5145 XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, VT,
5146 SDValue(Q, 0), DSub);
5147 } else {
5148 SDValue SubReg = CurDAG->getTargetConstant(
5149 (SVT.isScalableVector() ? AArch64::zsub : AArch64::dsub), DL,
5150 MVT::i32);
5151 XAR = CurDAG->getMachineNode(AArch64::EXTRACT_SUBREG, DL, VT,
5152 SDValue(XAR, 0), SubReg);
5153 }
5154 }
5155 ReplaceNode(N, XAR);
5156 return true;
5157}
5158
5159/// Returns a copy from WZR or XZR. This can be used during instruction
5160/// selection (it does not require any further selection/legalization).
5162 assert(VT == MVT::i32 || VT == MVT::i64);
5163 return DAG.getCopyFromReg(DAG.getEntryNode(), DL,
5164 VT == MVT::i32 ? AArch64::WZR : AArch64::XZR, VT);
5165}
5166
5167void AArch64DAGToDAGISel::Select(SDNode *Node) {
5168 // If we have a custom node, we already have selected!
5169 if (Node->isMachineOpcode()) {
5170 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n");
5171 Node->setNodeId(-1);
5172 return;
5173 }
5174
5175 // Few custom selection stuff.
5176 EVT VT = Node->getValueType(0);
5177
5178 switch (Node->getOpcode()) {
5179 default:
5180 break;
5181
5183 if (SelectCMP_SWAP(Node))
5184 return;
5185 break;
5186
5187 case ISD::READ_REGISTER:
5188 case AArch64ISD::MRRS:
5189 if (tryReadRegister(Node))
5190 return;
5191 break;
5192
5194 case AArch64ISD::MSRR:
5195 if (tryWriteRegister(Node))
5196 return;
5197 break;
5198
5199 case ISD::LOAD: {
5200 // Try to select as an indexed load. Fall through to normal processing
5201 // if we can't.
5202 if (tryIndexedLoad(Node))
5203 return;
5204 break;
5205 }
5206
5207 case ISD::SRL:
5208 case ISD::AND:
5209 case ISD::SRA:
5211 if (tryBitfieldExtractOp(Node))
5212 return;
5213 if (tryBitfieldInsertInZeroOp(Node))
5214 return;
5215 [[fallthrough]];
5216 case ISD::ROTR:
5217 case ISD::SHL:
5218 break;
5219
5220 case ISD::OR:
5221 if (tryBitfieldInsertOp(Node))
5222 return;
5223 if (trySelectXAR(Node))
5224 return;
5225 break;
5226
5228 if (trySelectCastScalableToFixedLengthVector(Node))
5229 return;
5230 break;
5231 }
5232
5233 case ISD::INSERT_SUBVECTOR: {
5234 if (trySelectCastFixedLengthToScalableVector(Node))
5235 return;
5236 break;
5237 }
5238
5239 case AArch64ISD::CSEL:
5240 if (tryFoldCselToFMaxMin(Node))
5241 return;
5242 break;
5243
5244 case ISD::Constant: {
5245 // Materialize zero constants as copies from WZR/XZR. This allows
5246 // the coalescer to propagate these into other instructions.
5247 ConstantSDNode *ConstNode = cast<ConstantSDNode>(Node);
5248 if (ConstNode->isZero() && (VT == MVT::i32 || VT == MVT::i64)) {
5249 ReplaceNode(Node, getZeroRegister(*CurDAG, SDLoc(Node), VT).getNode());
5250 return;
5251 }
5252 break;
5253 }
5254
5255 case ISD::FrameIndex: {
5256 // Selects to ADDXri FI, 0 which in turn will become ADDXri SP, imm.
5257 int FI = cast<FrameIndexSDNode>(Node)->getIndex();
5258 unsigned Shifter = AArch64_AM::getShifterImm(AArch64_AM::LSL, 0);
5259 const TargetLowering *TLI = getTargetLowering();
5260 SDValue TFI = CurDAG->getTargetFrameIndex(
5261 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
5262 SDLoc DL(Node);
5263 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, DL, MVT::i32),
5264 CurDAG->getTargetConstant(Shifter, DL, MVT::i32) };
5265 CurDAG->SelectNodeTo(Node, AArch64::ADDXri, MVT::i64, Ops);
5266 return;
5267 }
5269 unsigned IntNo = Node->getConstantOperandVal(1);
5270 switch (IntNo) {
5271 default:
5272 break;
5273 case Intrinsic::aarch64_gcsss: {
5274 SDLoc DL(Node);
5275 SDValue Chain = Node->getOperand(0);
5276 SDValue Val = Node->getOperand(2);
5277 SDValue Zero = CurDAG->getCopyFromReg(Chain, DL, AArch64::XZR, MVT::i64);
5278 SDNode *SS1 =
5279 CurDAG->getMachineNode(AArch64::GCSSS1, DL, MVT::Other, Val, Chain);
5280 SDNode *SS2 = CurDAG->getMachineNode(AArch64::GCSSS2, DL, MVT::i64,
5281 MVT::Other, Zero, SDValue(SS1, 0));
5282 ReplaceNode(Node, SS2);
5283 return;
5284 }
5285 case Intrinsic::aarch64_ldaxp:
5286 case Intrinsic::aarch64_ldxp: {
5287 unsigned Op =
5288 IntNo == Intrinsic::aarch64_ldaxp ? AArch64::LDAXPX : AArch64::LDXPX;
5289 SDValue MemAddr = Node->getOperand(2);
5290 SDLoc DL(Node);
5291 SDValue Chain = Node->getOperand(0);
5292
5293 SDNode *Ld = CurDAG->getMachineNode(Op, DL, MVT::i64, MVT::i64,
5294 MVT::Other, MemAddr, Chain);
5295
5296 // Transfer memoperands.
5297 MachineMemOperand *MemOp =
5298 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
5299 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
5300 ReplaceNode(Node, Ld);
5301 return;
5302 }
5303 case Intrinsic::aarch64_stlxp:
5304 case Intrinsic::aarch64_stxp: {
5305 unsigned Op =
5306 IntNo == Intrinsic::aarch64_stlxp ? AArch64::STLXPX : AArch64::STXPX;
5307 SDLoc DL(Node);
5308 SDValue Chain = Node->getOperand(0);
5309 SDValue ValLo = Node->getOperand(2);
5310 SDValue ValHi = Node->getOperand(3);
5311 SDValue MemAddr = Node->getOperand(4);
5312
5313 // Place arguments in the right order.
5314 SDValue Ops[] = {ValLo, ValHi, MemAddr, Chain};
5315
5316 SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops);
5317 // Transfer memoperands.
5318 MachineMemOperand *MemOp =
5319 cast<MemIntrinsicSDNode>(Node)->getMemOperand();
5320 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
5321
5322 ReplaceNode(Node, St);
5323 return;
5324 }
5325 case Intrinsic::aarch64_neon_ld1x2:
5326 if (VT == MVT::v8i8) {
5327 SelectLoad(Node, 2, AArch64::LD1Twov8b, AArch64::dsub0);
5328 return;
5329 } else if (VT == MVT::v16i8) {
5330 SelectLoad(Node, 2, AArch64::LD1Twov16b, AArch64::qsub0);
5331 return;
5332 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5333 SelectLoad(Node, 2, AArch64::LD1Twov4h, AArch64::dsub0);
5334 return;
5335 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5336 SelectLoad(Node, 2, AArch64::LD1Twov8h, AArch64::qsub0);
5337 return;
5338 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5339 SelectLoad(Node, 2, AArch64::LD1Twov2s, AArch64::dsub0);
5340 return;
5341 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5342 SelectLoad(Node, 2, AArch64::LD1Twov4s, AArch64::qsub0);
5343 return;
5344 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5345 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
5346 return;
5347 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5348 SelectLoad(Node, 2, AArch64::LD1Twov2d, AArch64::qsub0);
5349 return;
5350 }
5351 break;
5352 case Intrinsic::aarch64_neon_ld1x3:
5353 if (VT == MVT::v8i8) {
5354 SelectLoad(Node, 3, AArch64::LD1Threev8b, AArch64::dsub0);
5355 return;
5356 } else if (VT == MVT::v16i8) {
5357 SelectLoad(Node, 3, AArch64::LD1Threev16b, AArch64::qsub0);
5358 return;
5359 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5360 SelectLoad(Node, 3, AArch64::LD1Threev4h, AArch64::dsub0);
5361 return;
5362 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5363 SelectLoad(Node, 3, AArch64::LD1Threev8h, AArch64::qsub0);
5364 return;
5365 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5366 SelectLoad(Node, 3, AArch64::LD1Threev2s, AArch64::dsub0);
5367 return;
5368 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5369 SelectLoad(Node, 3, AArch64::LD1Threev4s, AArch64::qsub0);
5370 return;
5371 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5372 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
5373 return;
5374 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5375 SelectLoad(Node, 3, AArch64::LD1Threev2d, AArch64::qsub0);
5376 return;
5377 }
5378 break;
5379 case Intrinsic::aarch64_neon_ld1x4:
5380 if (VT == MVT::v8i8) {
5381 SelectLoad(Node, 4, AArch64::LD1Fourv8b, AArch64::dsub0);
5382 return;
5383 } else if (VT == MVT::v16i8) {
5384 SelectLoad(Node, 4, AArch64::LD1Fourv16b, AArch64::qsub0);
5385 return;
5386 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5387 SelectLoad(Node, 4, AArch64::LD1Fourv4h, AArch64::dsub0);
5388 return;
5389 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5390 SelectLoad(Node, 4, AArch64::LD1Fourv8h, AArch64::qsub0);
5391 return;
5392 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5393 SelectLoad(Node, 4, AArch64::LD1Fourv2s, AArch64::dsub0);
5394 return;
5395 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5396 SelectLoad(Node, 4, AArch64::LD1Fourv4s, AArch64::qsub0);
5397 return;
5398 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5399 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
5400 return;
5401 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5402 SelectLoad(Node, 4, AArch64::LD1Fourv2d, AArch64::qsub0);
5403 return;
5404 }
5405 break;
5406 case Intrinsic::aarch64_neon_ld2:
5407 if (VT == MVT::v8i8) {
5408 SelectLoad(Node, 2, AArch64::LD2Twov8b, AArch64::dsub0);
5409 return;
5410 } else if (VT == MVT::v16i8) {
5411 SelectLoad(Node, 2, AArch64::LD2Twov16b, AArch64::qsub0);
5412 return;
5413 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5414 SelectLoad(Node, 2, AArch64::LD2Twov4h, AArch64::dsub0);
5415 return;
5416 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5417 SelectLoad(Node, 2, AArch64::LD2Twov8h, AArch64::qsub0);
5418 return;
5419 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5420 SelectLoad(Node, 2, AArch64::LD2Twov2s, AArch64::dsub0);
5421 return;
5422 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5423 SelectLoad(Node, 2, AArch64::LD2Twov4s, AArch64::qsub0);
5424 return;
5425 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5426 SelectLoad(Node, 2, AArch64::LD1Twov1d, AArch64::dsub0);
5427 return;
5428 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5429 SelectLoad(Node, 2, AArch64::LD2Twov2d, AArch64::qsub0);
5430 return;
5431 }
5432 break;
5433 case Intrinsic::aarch64_neon_ld3:
5434 if (VT == MVT::v8i8) {
5435 SelectLoad(Node, 3, AArch64::LD3Threev8b, AArch64::dsub0);
5436 return;
5437 } else if (VT == MVT::v16i8) {
5438 SelectLoad(Node, 3, AArch64::LD3Threev16b, AArch64::qsub0);
5439 return;
5440 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5441 SelectLoad(Node, 3, AArch64::LD3Threev4h, AArch64::dsub0);
5442 return;
5443 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5444 SelectLoad(Node, 3, AArch64::LD3Threev8h, AArch64::qsub0);
5445 return;
5446 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5447 SelectLoad(Node, 3, AArch64::LD3Threev2s, AArch64::dsub0);
5448 return;
5449 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5450 SelectLoad(Node, 3, AArch64::LD3Threev4s, AArch64::qsub0);
5451 return;
5452 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5453 SelectLoad(Node, 3, AArch64::LD1Threev1d, AArch64::dsub0);
5454 return;
5455 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5456 SelectLoad(Node, 3, AArch64::LD3Threev2d, AArch64::qsub0);
5457 return;
5458 }
5459 break;
5460 case Intrinsic::aarch64_neon_ld4:
5461 if (VT == MVT::v8i8) {
5462 SelectLoad(Node, 4, AArch64::LD4Fourv8b, AArch64::dsub0);
5463 return;
5464 } else if (VT == MVT::v16i8) {
5465 SelectLoad(Node, 4, AArch64::LD4Fourv16b, AArch64::qsub0);
5466 return;
5467 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5468 SelectLoad(Node, 4, AArch64::LD4Fourv4h, AArch64::dsub0);
5469 return;
5470 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5471 SelectLoad(Node, 4, AArch64::LD4Fourv8h, AArch64::qsub0);
5472 return;
5473 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5474 SelectLoad(Node, 4, AArch64::LD4Fourv2s, AArch64::dsub0);
5475 return;
5476 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5477 SelectLoad(Node, 4, AArch64::LD4Fourv4s, AArch64::qsub0);
5478 return;
5479 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5480 SelectLoad(Node, 4, AArch64::LD1Fourv1d, AArch64::dsub0);
5481 return;
5482 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5483 SelectLoad(Node, 4, AArch64::LD4Fourv2d, AArch64::qsub0);
5484 return;
5485 }
5486 break;
5487 case Intrinsic::aarch64_neon_ld2r:
5488 if (VT == MVT::v8i8) {
5489 SelectLoad(Node, 2, AArch64::LD2Rv8b, AArch64::dsub0);
5490 return;
5491 } else if (VT == MVT::v16i8) {
5492 SelectLoad(Node, 2, AArch64::LD2Rv16b, AArch64::qsub0);
5493 return;
5494 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5495 SelectLoad(Node, 2, AArch64::LD2Rv4h, AArch64::dsub0);
5496 return;
5497 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5498 SelectLoad(Node, 2, AArch64::LD2Rv8h, AArch64::qsub0);
5499 return;
5500 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5501 SelectLoad(Node, 2, AArch64::LD2Rv2s, AArch64::dsub0);
5502 return;
5503 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5504 SelectLoad(Node, 2, AArch64::LD2Rv4s, AArch64::qsub0);
5505 return;
5506 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5507 SelectLoad(Node, 2, AArch64::LD2Rv1d, AArch64::dsub0);
5508 return;
5509 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5510 SelectLoad(Node, 2, AArch64::LD2Rv2d, AArch64::qsub0);
5511 return;
5512 }
5513 break;
5514 case Intrinsic::aarch64_neon_ld3r:
5515 if (VT == MVT::v8i8) {
5516 SelectLoad(Node, 3, AArch64::LD3Rv8b, AArch64::dsub0);
5517 return;
5518 } else if (VT == MVT::v16i8) {
5519 SelectLoad(Node, 3, AArch64::LD3Rv16b, AArch64::qsub0);
5520 return;
5521 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5522 SelectLoad(Node, 3, AArch64::LD3Rv4h, AArch64::dsub0);
5523 return;
5524 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5525 SelectLoad(Node, 3, AArch64::LD3Rv8h, AArch64::qsub0);
5526 return;
5527 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5528 SelectLoad(Node, 3, AArch64::LD3Rv2s, AArch64::dsub0);
5529 return;
5530 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5531 SelectLoad(Node, 3, AArch64::LD3Rv4s, AArch64::qsub0);
5532 return;
5533 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5534 SelectLoad(Node, 3, AArch64::LD3Rv1d, AArch64::dsub0);
5535 return;
5536 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5537 SelectLoad(Node, 3, AArch64::LD3Rv2d, AArch64::qsub0);
5538 return;
5539 }
5540 break;
5541 case Intrinsic::aarch64_neon_ld4r:
5542 if (VT == MVT::v8i8) {
5543 SelectLoad(Node, 4, AArch64::LD4Rv8b, AArch64::dsub0);
5544 return;
5545 } else if (VT == MVT::v16i8) {
5546 SelectLoad(Node, 4, AArch64::LD4Rv16b, AArch64::qsub0);
5547 return;
5548 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
5549 SelectLoad(Node, 4, AArch64::LD4Rv4h, AArch64::dsub0);
5550 return;
5551 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
5552 SelectLoad(Node, 4, AArch64::LD4Rv8h, AArch64::qsub0);
5553 return;
5554 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
5555 SelectLoad(Node, 4, AArch64::LD4Rv2s, AArch64::dsub0);
5556 return;
5557 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
5558 SelectLoad(Node, 4, AArch64::LD4Rv4s, AArch64::qsub0);
5559 return;
5560 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
5561 SelectLoad(Node, 4, AArch64::LD4Rv1d, AArch64::dsub0);
5562 return;
5563 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
5564 SelectLoad(Node, 4, AArch64::LD4Rv2d, AArch64::qsub0);
5565 return;
5566 }
5567 break;
5568 case Intrinsic::aarch64_neon_ld2lane:
5569 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5570 SelectLoadLane(Node, 2, AArch64::LD2i8);
5571 return;
5572 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5573 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5574 SelectLoadLane(Node, 2, AArch64::LD2i16);
5575 return;
5576 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5577 VT == MVT::v2f32) {
5578 SelectLoadLane(Node, 2, AArch64::LD2i32);
5579 return;
5580 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5581 VT == MVT::v1f64) {
5582 SelectLoadLane(Node, 2, AArch64::LD2i64);
5583 return;
5584 }
5585 break;
5586 case Intrinsic::aarch64_neon_ld3lane:
5587 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5588 SelectLoadLane(Node, 3, AArch64::LD3i8);
5589 return;
5590 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5591 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5592 SelectLoadLane(Node, 3, AArch64::LD3i16);
5593 return;
5594 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5595 VT == MVT::v2f32) {
5596 SelectLoadLane(Node, 3, AArch64::LD3i32);
5597 return;
5598 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5599 VT == MVT::v1f64) {
5600 SelectLoadLane(Node, 3, AArch64::LD3i64);
5601 return;
5602 }
5603 break;
5604 case Intrinsic::aarch64_neon_ld4lane:
5605 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
5606 SelectLoadLane(Node, 4, AArch64::LD4i8);
5607 return;
5608 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
5609 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
5610 SelectLoadLane(Node, 4, AArch64::LD4i16);
5611 return;
5612 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
5613 VT == MVT::v2f32) {
5614 SelectLoadLane(Node, 4, AArch64::LD4i32);
5615 return;
5616 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
5617 VT == MVT::v1f64) {
5618 SelectLoadLane(Node, 4, AArch64::LD4i64);
5619 return;
5620 }
5621 break;
5622 case Intrinsic::aarch64_ld64b:
5623 SelectLoad(Node, 8, AArch64::LD64B, AArch64::x8sub_0);
5624 return;
5625 case Intrinsic::aarch64_sve_ld2q_sret: {
5626 SelectPredicatedLoad(Node, 2, 4, AArch64::LD2Q_IMM, AArch64::LD2Q, true);
5627 return;
5628 }
5629 case Intrinsic::aarch64_sve_ld3q_sret: {
5630 SelectPredicatedLoad(Node, 3, 4, AArch64::LD3Q_IMM, AArch64::LD3Q, true);
5631 return;
5632 }
5633 case Intrinsic::aarch64_sve_ld4q_sret: {
5634 SelectPredicatedLoad(Node, 4, 4, AArch64::LD4Q_IMM, AArch64::LD4Q, true);
5635 return;
5636 }
5637 case Intrinsic::aarch64_sve_ld2_sret: {
5638 if (VT == MVT::nxv16i8) {
5639 SelectPredicatedLoad(Node, 2, 0, AArch64::LD2B_IMM, AArch64::LD2B,
5640 true);
5641 return;
5642 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5643 VT == MVT::nxv8bf16) {
5644 SelectPredicatedLoad(Node, 2, 1, AArch64::LD2H_IMM, AArch64::LD2H,
5645 true);
5646 return;
5647 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5648 SelectPredicatedLoad(Node, 2, 2, AArch64::LD2W_IMM, AArch64::LD2W,
5649 true);
5650 return;
5651 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5652 SelectPredicatedLoad(Node, 2, 3, AArch64::LD2D_IMM, AArch64::LD2D,
5653 true);
5654 return;
5655 }
5656 break;
5657 }
5658 case Intrinsic::aarch64_sve_ld1_pn_x2: {
5659 if (VT == MVT::nxv16i8) {
5660 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5661 SelectContiguousMultiVectorLoad(
5662 Node, 2, 0, AArch64::LD1B_2Z_IMM_PSEUDO, AArch64::LD1B_2Z_PSEUDO);
5663 else if (Subtarget->hasSVE2p1())
5664 SelectContiguousMultiVectorLoad(Node, 2, 0, AArch64::LD1B_2Z_IMM,
5665 AArch64::LD1B_2Z);
5666 else
5667 break;
5668 return;
5669 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5670 VT == MVT::nxv8bf16) {
5671 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5672 SelectContiguousMultiVectorLoad(
5673 Node, 2, 1, AArch64::LD1H_2Z_IMM_PSEUDO, AArch64::LD1H_2Z_PSEUDO);
5674 else if (Subtarget->hasSVE2p1())
5675 SelectContiguousMultiVectorLoad(Node, 2, 1, AArch64::LD1H_2Z_IMM,
5676 AArch64::LD1H_2Z);
5677 else
5678 break;
5679 return;
5680 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5681 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5682 SelectContiguousMultiVectorLoad(
5683 Node, 2, 2, AArch64::LD1W_2Z_IMM_PSEUDO, AArch64::LD1W_2Z_PSEUDO);
5684 else if (Subtarget->hasSVE2p1())
5685 SelectContiguousMultiVectorLoad(Node, 2, 2, AArch64::LD1W_2Z_IMM,
5686 AArch64::LD1W_2Z);
5687 else
5688 break;
5689 return;
5690 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5691 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5692 SelectContiguousMultiVectorLoad(
5693 Node, 2, 3, AArch64::LD1D_2Z_IMM_PSEUDO, AArch64::LD1D_2Z_PSEUDO);
5694 else if (Subtarget->hasSVE2p1())
5695 SelectContiguousMultiVectorLoad(Node, 2, 3, AArch64::LD1D_2Z_IMM,
5696 AArch64::LD1D_2Z);
5697 else
5698 break;
5699 return;
5700 }
5701 break;
5702 }
5703 case Intrinsic::aarch64_sve_ld1_pn_x4: {
5704 if (VT == MVT::nxv16i8) {
5705 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5706 SelectContiguousMultiVectorLoad(
5707 Node, 4, 0, AArch64::LD1B_4Z_IMM_PSEUDO, AArch64::LD1B_4Z_PSEUDO);
5708 else if (Subtarget->hasSVE2p1())
5709 SelectContiguousMultiVectorLoad(Node, 4, 0, AArch64::LD1B_4Z_IMM,
5710 AArch64::LD1B_4Z);
5711 else
5712 break;
5713 return;
5714 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5715 VT == MVT::nxv8bf16) {
5716 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5717 SelectContiguousMultiVectorLoad(
5718 Node, 4, 1, AArch64::LD1H_4Z_IMM_PSEUDO, AArch64::LD1H_4Z_PSEUDO);
5719 else if (Subtarget->hasSVE2p1())
5720 SelectContiguousMultiVectorLoad(Node, 4, 1, AArch64::LD1H_4Z_IMM,
5721 AArch64::LD1H_4Z);
5722 else
5723 break;
5724 return;
5725 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5726 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5727 SelectContiguousMultiVectorLoad(
5728 Node, 4, 2, AArch64::LD1W_4Z_IMM_PSEUDO, AArch64::LD1W_4Z_PSEUDO);
5729 else if (Subtarget->hasSVE2p1())
5730 SelectContiguousMultiVectorLoad(Node, 4, 2, AArch64::LD1W_4Z_IMM,
5731 AArch64::LD1W_4Z);
5732 else
5733 break;
5734 return;
5735 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5736 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5737 SelectContiguousMultiVectorLoad(
5738 Node, 4, 3, AArch64::LD1D_4Z_IMM_PSEUDO, AArch64::LD1D_4Z_PSEUDO);
5739 else if (Subtarget->hasSVE2p1())
5740 SelectContiguousMultiVectorLoad(Node, 4, 3, AArch64::LD1D_4Z_IMM,
5741 AArch64::LD1D_4Z);
5742 else
5743 break;
5744 return;
5745 }
5746 break;
5747 }
5748 case Intrinsic::aarch64_sve_ldnt1_pn_x2: {
5749 if (VT == MVT::nxv16i8) {
5750 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5751 SelectContiguousMultiVectorLoad(Node, 2, 0,
5752 AArch64::LDNT1B_2Z_IMM_PSEUDO,
5753 AArch64::LDNT1B_2Z_PSEUDO);
5754 else if (Subtarget->hasSVE2p1())
5755 SelectContiguousMultiVectorLoad(Node, 2, 0, AArch64::LDNT1B_2Z_IMM,
5756 AArch64::LDNT1B_2Z);
5757 else
5758 break;
5759 return;
5760 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5761 VT == MVT::nxv8bf16) {
5762 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5763 SelectContiguousMultiVectorLoad(Node, 2, 1,
5764 AArch64::LDNT1H_2Z_IMM_PSEUDO,
5765 AArch64::LDNT1H_2Z_PSEUDO);
5766 else if (Subtarget->hasSVE2p1())
5767 SelectContiguousMultiVectorLoad(Node, 2, 1, AArch64::LDNT1H_2Z_IMM,
5768 AArch64::LDNT1H_2Z);
5769 else
5770 break;
5771 return;
5772 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5773 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5774 SelectContiguousMultiVectorLoad(Node, 2, 2,
5775 AArch64::LDNT1W_2Z_IMM_PSEUDO,
5776 AArch64::LDNT1W_2Z_PSEUDO);
5777 else if (Subtarget->hasSVE2p1())
5778 SelectContiguousMultiVectorLoad(Node, 2, 2, AArch64::LDNT1W_2Z_IMM,
5779 AArch64::LDNT1W_2Z);
5780 else
5781 break;
5782 return;
5783 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5784 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5785 SelectContiguousMultiVectorLoad(Node, 2, 3,
5786 AArch64::LDNT1D_2Z_IMM_PSEUDO,
5787 AArch64::LDNT1D_2Z_PSEUDO);
5788 else if (Subtarget->hasSVE2p1())
5789 SelectContiguousMultiVectorLoad(Node, 2, 3, AArch64::LDNT1D_2Z_IMM,
5790 AArch64::LDNT1D_2Z);
5791 else
5792 break;
5793 return;
5794 }
5795 break;
5796 }
5797 case Intrinsic::aarch64_sve_ldnt1_pn_x4: {
5798 if (VT == MVT::nxv16i8) {
5799 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5800 SelectContiguousMultiVectorLoad(Node, 4, 0,
5801 AArch64::LDNT1B_4Z_IMM_PSEUDO,
5802 AArch64::LDNT1B_4Z_PSEUDO);
5803 else if (Subtarget->hasSVE2p1())
5804 SelectContiguousMultiVectorLoad(Node, 4, 0, AArch64::LDNT1B_4Z_IMM,
5805 AArch64::LDNT1B_4Z);
5806 else
5807 break;
5808 return;
5809 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5810 VT == MVT::nxv8bf16) {
5811 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5812 SelectContiguousMultiVectorLoad(Node, 4, 1,
5813 AArch64::LDNT1H_4Z_IMM_PSEUDO,
5814 AArch64::LDNT1H_4Z_PSEUDO);
5815 else if (Subtarget->hasSVE2p1())
5816 SelectContiguousMultiVectorLoad(Node, 4, 1, AArch64::LDNT1H_4Z_IMM,
5817 AArch64::LDNT1H_4Z);
5818 else
5819 break;
5820 return;
5821 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5822 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5823 SelectContiguousMultiVectorLoad(Node, 4, 2,
5824 AArch64::LDNT1W_4Z_IMM_PSEUDO,
5825 AArch64::LDNT1W_4Z_PSEUDO);
5826 else if (Subtarget->hasSVE2p1())
5827 SelectContiguousMultiVectorLoad(Node, 4, 2, AArch64::LDNT1W_4Z_IMM,
5828 AArch64::LDNT1W_4Z);
5829 else
5830 break;
5831 return;
5832 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5833 if (Subtarget->hasSME2() && Subtarget->isStreaming())
5834 SelectContiguousMultiVectorLoad(Node, 4, 3,
5835 AArch64::LDNT1D_4Z_IMM_PSEUDO,
5836 AArch64::LDNT1D_4Z_PSEUDO);
5837 else if (Subtarget->hasSVE2p1())
5838 SelectContiguousMultiVectorLoad(Node, 4, 3, AArch64::LDNT1D_4Z_IMM,
5839 AArch64::LDNT1D_4Z);
5840 else
5841 break;
5842 return;
5843 }
5844 break;
5845 }
5846 case Intrinsic::aarch64_sve_ld3_sret: {
5847 if (VT == MVT::nxv16i8) {
5848 SelectPredicatedLoad(Node, 3, 0, AArch64::LD3B_IMM, AArch64::LD3B,
5849 true);
5850 return;
5851 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5852 VT == MVT::nxv8bf16) {
5853 SelectPredicatedLoad(Node, 3, 1, AArch64::LD3H_IMM, AArch64::LD3H,
5854 true);
5855 return;
5856 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5857 SelectPredicatedLoad(Node, 3, 2, AArch64::LD3W_IMM, AArch64::LD3W,
5858 true);
5859 return;
5860 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5861 SelectPredicatedLoad(Node, 3, 3, AArch64::LD3D_IMM, AArch64::LD3D,
5862 true);
5863 return;
5864 }
5865 break;
5866 }
5867 case Intrinsic::aarch64_sve_ld4_sret: {
5868 if (VT == MVT::nxv16i8) {
5869 SelectPredicatedLoad(Node, 4, 0, AArch64::LD4B_IMM, AArch64::LD4B,
5870 true);
5871 return;
5872 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5873 VT == MVT::nxv8bf16) {
5874 SelectPredicatedLoad(Node, 4, 1, AArch64::LD4H_IMM, AArch64::LD4H,
5875 true);
5876 return;
5877 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5878 SelectPredicatedLoad(Node, 4, 2, AArch64::LD4W_IMM, AArch64::LD4W,
5879 true);
5880 return;
5881 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5882 SelectPredicatedLoad(Node, 4, 3, AArch64::LD4D_IMM, AArch64::LD4D,
5883 true);
5884 return;
5885 }
5886 break;
5887 }
5888 case Intrinsic::aarch64_sme_read_hor_vg2: {
5889 if (VT == MVT::nxv16i8) {
5890 SelectMultiVectorMove<14, 2>(Node, 2, AArch64::ZAB0,
5891 AArch64::MOVA_2ZMXI_H_B);
5892 return;
5893 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5894 VT == MVT::nxv8bf16) {
5895 SelectMultiVectorMove<6, 2>(Node, 2, AArch64::ZAH0,
5896 AArch64::MOVA_2ZMXI_H_H);
5897 return;
5898 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5899 SelectMultiVectorMove<2, 2>(Node, 2, AArch64::ZAS0,
5900 AArch64::MOVA_2ZMXI_H_S);
5901 return;
5902 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5903 SelectMultiVectorMove<0, 2>(Node, 2, AArch64::ZAD0,
5904 AArch64::MOVA_2ZMXI_H_D);
5905 return;
5906 }
5907 break;
5908 }
5909 case Intrinsic::aarch64_sme_read_ver_vg2: {
5910 if (VT == MVT::nxv16i8) {
5911 SelectMultiVectorMove<14, 2>(Node, 2, AArch64::ZAB0,
5912 AArch64::MOVA_2ZMXI_V_B);
5913 return;
5914 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5915 VT == MVT::nxv8bf16) {
5916 SelectMultiVectorMove<6, 2>(Node, 2, AArch64::ZAH0,
5917 AArch64::MOVA_2ZMXI_V_H);
5918 return;
5919 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5920 SelectMultiVectorMove<2, 2>(Node, 2, AArch64::ZAS0,
5921 AArch64::MOVA_2ZMXI_V_S);
5922 return;
5923 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5924 SelectMultiVectorMove<0, 2>(Node, 2, AArch64::ZAD0,
5925 AArch64::MOVA_2ZMXI_V_D);
5926 return;
5927 }
5928 break;
5929 }
5930 case Intrinsic::aarch64_sme_read_hor_vg4: {
5931 if (VT == MVT::nxv16i8) {
5932 SelectMultiVectorMove<12, 4>(Node, 4, AArch64::ZAB0,
5933 AArch64::MOVA_4ZMXI_H_B);
5934 return;
5935 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5936 VT == MVT::nxv8bf16) {
5937 SelectMultiVectorMove<4, 4>(Node, 4, AArch64::ZAH0,
5938 AArch64::MOVA_4ZMXI_H_H);
5939 return;
5940 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5941 SelectMultiVectorMove<0, 2>(Node, 4, AArch64::ZAS0,
5942 AArch64::MOVA_4ZMXI_H_S);
5943 return;
5944 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5945 SelectMultiVectorMove<0, 2>(Node, 4, AArch64::ZAD0,
5946 AArch64::MOVA_4ZMXI_H_D);
5947 return;
5948 }
5949 break;
5950 }
5951 case Intrinsic::aarch64_sme_read_ver_vg4: {
5952 if (VT == MVT::nxv16i8) {
5953 SelectMultiVectorMove<12, 4>(Node, 4, AArch64::ZAB0,
5954 AArch64::MOVA_4ZMXI_V_B);
5955 return;
5956 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5957 VT == MVT::nxv8bf16) {
5958 SelectMultiVectorMove<4, 4>(Node, 4, AArch64::ZAH0,
5959 AArch64::MOVA_4ZMXI_V_H);
5960 return;
5961 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5962 SelectMultiVectorMove<0, 4>(Node, 4, AArch64::ZAS0,
5963 AArch64::MOVA_4ZMXI_V_S);
5964 return;
5965 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5966 SelectMultiVectorMove<0, 4>(Node, 4, AArch64::ZAD0,
5967 AArch64::MOVA_4ZMXI_V_D);
5968 return;
5969 }
5970 break;
5971 }
5972 case Intrinsic::aarch64_sme_read_vg1x2: {
5973 SelectMultiVectorMove<7, 1>(Node, 2, AArch64::ZA,
5974 AArch64::MOVA_VG2_2ZMXI);
5975 return;
5976 }
5977 case Intrinsic::aarch64_sme_read_vg1x4: {
5978 SelectMultiVectorMove<7, 1>(Node, 4, AArch64::ZA,
5979 AArch64::MOVA_VG4_4ZMXI);
5980 return;
5981 }
5982 case Intrinsic::aarch64_sme_readz_horiz_x2: {
5983 if (VT == MVT::nxv16i8) {
5984 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_B_PSEUDO, 14, 2);
5985 return;
5986 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
5987 VT == MVT::nxv8bf16) {
5988 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_H_PSEUDO, 6, 2);
5989 return;
5990 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
5991 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_S_PSEUDO, 2, 2);
5992 return;
5993 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
5994 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_H_D_PSEUDO, 0, 2);
5995 return;
5996 }
5997 break;
5998 }
5999 case Intrinsic::aarch64_sme_readz_vert_x2: {
6000 if (VT == MVT::nxv16i8) {
6001 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_B_PSEUDO, 14, 2);
6002 return;
6003 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6004 VT == MVT::nxv8bf16) {
6005 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_H_PSEUDO, 6, 2);
6006 return;
6007 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6008 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_S_PSEUDO, 2, 2);
6009 return;
6010 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6011 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_2ZMI_V_D_PSEUDO, 0, 2);
6012 return;
6013 }
6014 break;
6015 }
6016 case Intrinsic::aarch64_sme_readz_horiz_x4: {
6017 if (VT == MVT::nxv16i8) {
6018 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_B_PSEUDO, 12, 4);
6019 return;
6020 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6021 VT == MVT::nxv8bf16) {
6022 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_H_PSEUDO, 4, 4);
6023 return;
6024 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6025 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_S_PSEUDO, 0, 4);
6026 return;
6027 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6028 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_H_D_PSEUDO, 0, 4);
6029 return;
6030 }
6031 break;
6032 }
6033 case Intrinsic::aarch64_sme_readz_vert_x4: {
6034 if (VT == MVT::nxv16i8) {
6035 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_B_PSEUDO, 12, 4);
6036 return;
6037 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
6038 VT == MVT::nxv8bf16) {
6039 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_H_PSEUDO, 4, 4);
6040 return;
6041 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
6042 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_S_PSEUDO, 0, 4);
6043 return;
6044 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
6045 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_4ZMI_V_D_PSEUDO, 0, 4);
6046 return;
6047 }
6048 break;
6049 }
6050 case Intrinsic::aarch64_sme_readz_x2: {
6051 SelectMultiVectorMoveZ(Node, 2, AArch64::MOVAZ_VG2_2ZMXI_PSEUDO, 7, 1,
6052 AArch64::ZA);
6053 return;
6054 }
6055 case Intrinsic::aarch64_sme_readz_x4: {
6056 SelectMultiVectorMoveZ(Node, 4, AArch64::MOVAZ_VG4_4ZMXI_PSEUDO, 7, 1,
6057 AArch64::ZA);
6058 return;
6059 }
6060 case Intrinsic::swift_async_context_addr: {
6061 SDLoc DL(Node);
6062 SDValue Chain = Node->getOperand(0);
6063 SDValue CopyFP = CurDAG->getCopyFromReg(Chain, DL, AArch64::FP, MVT::i64);
6064 SDValue Res = SDValue(
6065 CurDAG->getMachineNode(AArch64::SUBXri, DL, MVT::i64, CopyFP,
6066 CurDAG->getTargetConstant(8, DL, MVT::i32),
6067 CurDAG->getTargetConstant(0, DL, MVT::i32)),
6068 0);
6069 ReplaceUses(SDValue(Node, 0), Res);
6070 ReplaceUses(SDValue(Node, 1), CopyFP.getValue(1));
6071 CurDAG->RemoveDeadNode(Node);
6072
6073 auto &MF = CurDAG->getMachineFunction();
6074 MF.getFrameInfo().setFrameAddressIsTaken(true);
6075 MF.getInfo<AArch64FunctionInfo>()->setHasSwiftAsyncContext(true);
6076 return;
6077 }
6078 case Intrinsic::aarch64_sme_luti2_lane_zt_x4: {
6080 Node->getValueType(0),
6081 {AArch64::LUTI2_4ZTZI_B, AArch64::LUTI2_4ZTZI_H,
6082 AArch64::LUTI2_4ZTZI_S}))
6083 // Second Immediate must be <= 3:
6084 SelectMultiVectorLutiLane(Node, 4, Opc, 3);
6085 return;
6086 }
6087 case Intrinsic::aarch64_sme_luti4_lane_zt_x4: {
6089 Node->getValueType(0),
6090 {0, AArch64::LUTI4_4ZTZI_H, AArch64::LUTI4_4ZTZI_S}))
6091 // Second Immediate must be <= 1:
6092 SelectMultiVectorLutiLane(Node, 4, Opc, 1);
6093 return;
6094 }
6095 case Intrinsic::aarch64_sme_luti2_lane_zt_x2: {
6097 Node->getValueType(0),
6098 {AArch64::LUTI2_2ZTZI_B, AArch64::LUTI2_2ZTZI_H,
6099 AArch64::LUTI2_2ZTZI_S}))
6100 // Second Immediate must be <= 7:
6101 SelectMultiVectorLutiLane(Node, 2, Opc, 7);
6102 return;
6103 }
6104 case Intrinsic::aarch64_sme_luti4_lane_zt_x2: {
6106 Node->getValueType(0),
6107 {AArch64::LUTI4_2ZTZI_B, AArch64::LUTI4_2ZTZI_H,
6108 AArch64::LUTI4_2ZTZI_S}))
6109 // Second Immediate must be <= 3:
6110 SelectMultiVectorLutiLane(Node, 2, Opc, 3);
6111 return;
6112 }
6113 case Intrinsic::aarch64_sme_luti4_zt_x4: {
6114 SelectMultiVectorLuti(Node, 4, AArch64::LUTI4_4ZZT2Z, 2);
6115 return;
6116 }
6117 case Intrinsic::aarch64_sme_luti6_zt_x4: {
6118 SelectMultiVectorLuti(Node, 4, AArch64::LUTI6_4ZT3Z, 3);
6119 return;
6120 }
6121 case Intrinsic::aarch64_sve_fp8_cvtl1_x2:
6123 Node->getValueType(0),
6124 {AArch64::BF1CVTL_2ZZ_BtoH, AArch64::F1CVTL_2ZZ_BtoH}))
6125 SelectCVTIntrinsicFP8(Node, 2, Opc);
6126 return;
6127 case Intrinsic::aarch64_sve_fp8_cvtl2_x2:
6129 Node->getValueType(0),
6130 {AArch64::BF2CVTL_2ZZ_BtoH, AArch64::F2CVTL_2ZZ_BtoH}))
6131 SelectCVTIntrinsicFP8(Node, 2, Opc);
6132 return;
6133 case Intrinsic::aarch64_sve_fp8_cvt1_x2:
6135 Node->getValueType(0),
6136 {AArch64::BF1CVT_2ZZ_BtoH, AArch64::F1CVT_2ZZ_BtoH}))
6137 SelectCVTIntrinsicFP8(Node, 2, Opc);
6138 return;
6139 case Intrinsic::aarch64_sve_fp8_cvt2_x2:
6141 Node->getValueType(0),
6142 {AArch64::BF2CVT_2ZZ_BtoH, AArch64::F2CVT_2ZZ_BtoH}))
6143 SelectCVTIntrinsicFP8(Node, 2, Opc);
6144 return;
6145 case Intrinsic::ptrauth_resign_load_relative:
6146 SelectPtrauthResign(Node);
6147 return;
6148 }
6149 } break;
6151 unsigned IntNo = Node->getConstantOperandVal(0);
6152 switch (IntNo) {
6153 default:
6154 break;
6155 case Intrinsic::aarch64_tagp:
6156 SelectTagP(Node);
6157 return;
6158
6159 case Intrinsic::ptrauth_auth:
6160 SelectPtrauthAuth(Node);
6161 return;
6162
6163 case Intrinsic::ptrauth_resign:
6164 SelectPtrauthResign(Node);
6165 return;
6166
6167 case Intrinsic::ptrauth_auth_with_pc_and_resign:
6168 SelectPtrauthResignWithPC(Node);
6169 return;
6170
6171 case Intrinsic::aarch64_neon_tbl2:
6172 SelectTable(Node, 2,
6173 VT == MVT::v8i8 ? AArch64::TBLv8i8Two : AArch64::TBLv16i8Two,
6174 false);
6175 return;
6176 case Intrinsic::aarch64_neon_tbl3:
6177 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBLv8i8Three
6178 : AArch64::TBLv16i8Three,
6179 false);
6180 return;
6181 case Intrinsic::aarch64_neon_tbl4:
6182 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBLv8i8Four
6183 : AArch64::TBLv16i8Four,
6184 false);
6185 return;
6186 case Intrinsic::aarch64_neon_tbx2:
6187 SelectTable(Node, 2,
6188 VT == MVT::v8i8 ? AArch64::TBXv8i8Two : AArch64::TBXv16i8Two,
6189 true);
6190 return;
6191 case Intrinsic::aarch64_neon_tbx3:
6192 SelectTable(Node, 3, VT == MVT::v8i8 ? AArch64::TBXv8i8Three
6193 : AArch64::TBXv16i8Three,
6194 true);
6195 return;
6196 case Intrinsic::aarch64_neon_tbx4:
6197 SelectTable(Node, 4, VT == MVT::v8i8 ? AArch64::TBXv8i8Four
6198 : AArch64::TBXv16i8Four,
6199 true);
6200 return;
6201 case Intrinsic::aarch64_sve_srshl_single_x2:
6203 Node->getValueType(0),
6204 {AArch64::SRSHL_VG2_2ZZ_B, AArch64::SRSHL_VG2_2ZZ_H,
6205 AArch64::SRSHL_VG2_2ZZ_S, AArch64::SRSHL_VG2_2ZZ_D}))
6206 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6207 return;
6208 case Intrinsic::aarch64_sve_srshl_single_x4:
6210 Node->getValueType(0),
6211 {AArch64::SRSHL_VG4_4ZZ_B, AArch64::SRSHL_VG4_4ZZ_H,
6212 AArch64::SRSHL_VG4_4ZZ_S, AArch64::SRSHL_VG4_4ZZ_D}))
6213 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6214 return;
6215 case Intrinsic::aarch64_sme_luti6_lane_x4_x2:
6216 SelectMultiVectorLuti6LaneX4(Node, 2);
6217 return;
6218 case Intrinsic::aarch64_sme_luti6_lane_x4_x3:
6219 SelectMultiVectorLuti6LaneX4(Node, 3);
6220 return;
6221 case Intrinsic::aarch64_sve_urshl_single_x2:
6223 Node->getValueType(0),
6224 {AArch64::URSHL_VG2_2ZZ_B, AArch64::URSHL_VG2_2ZZ_H,
6225 AArch64::URSHL_VG2_2ZZ_S, AArch64::URSHL_VG2_2ZZ_D}))
6226 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6227 return;
6228 case Intrinsic::aarch64_sve_urshl_single_x4:
6230 Node->getValueType(0),
6231 {AArch64::URSHL_VG4_4ZZ_B, AArch64::URSHL_VG4_4ZZ_H,
6232 AArch64::URSHL_VG4_4ZZ_S, AArch64::URSHL_VG4_4ZZ_D}))
6233 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6234 return;
6235 case Intrinsic::aarch64_sve_srshl_x2:
6237 Node->getValueType(0),
6238 {AArch64::SRSHL_VG2_2Z2Z_B, AArch64::SRSHL_VG2_2Z2Z_H,
6239 AArch64::SRSHL_VG2_2Z2Z_S, AArch64::SRSHL_VG2_2Z2Z_D}))
6240 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6241 return;
6242 case Intrinsic::aarch64_sve_srshl_x4:
6244 Node->getValueType(0),
6245 {AArch64::SRSHL_VG4_4Z4Z_B, AArch64::SRSHL_VG4_4Z4Z_H,
6246 AArch64::SRSHL_VG4_4Z4Z_S, AArch64::SRSHL_VG4_4Z4Z_D}))
6247 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6248 return;
6249 case Intrinsic::aarch64_sve_urshl_x2:
6251 Node->getValueType(0),
6252 {AArch64::URSHL_VG2_2Z2Z_B, AArch64::URSHL_VG2_2Z2Z_H,
6253 AArch64::URSHL_VG2_2Z2Z_S, AArch64::URSHL_VG2_2Z2Z_D}))
6254 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6255 return;
6256 case Intrinsic::aarch64_sve_urshl_x4:
6258 Node->getValueType(0),
6259 {AArch64::URSHL_VG4_4Z4Z_B, AArch64::URSHL_VG4_4Z4Z_H,
6260 AArch64::URSHL_VG4_4Z4Z_S, AArch64::URSHL_VG4_4Z4Z_D}))
6261 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6262 return;
6263 case Intrinsic::aarch64_sve_sqdmulh_single_vgx2:
6265 Node->getValueType(0),
6266 {AArch64::SQDMULH_VG2_2ZZ_B, AArch64::SQDMULH_VG2_2ZZ_H,
6267 AArch64::SQDMULH_VG2_2ZZ_S, AArch64::SQDMULH_VG2_2ZZ_D}))
6268 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6269 return;
6270 case Intrinsic::aarch64_sve_sqdmulh_single_vgx4:
6272 Node->getValueType(0),
6273 {AArch64::SQDMULH_VG4_4ZZ_B, AArch64::SQDMULH_VG4_4ZZ_H,
6274 AArch64::SQDMULH_VG4_4ZZ_S, AArch64::SQDMULH_VG4_4ZZ_D}))
6275 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6276 return;
6277 case Intrinsic::aarch64_sve_sqdmulh_vgx2:
6279 Node->getValueType(0),
6280 {AArch64::SQDMULH_VG2_2Z2Z_B, AArch64::SQDMULH_VG2_2Z2Z_H,
6281 AArch64::SQDMULH_VG2_2Z2Z_S, AArch64::SQDMULH_VG2_2Z2Z_D}))
6282 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6283 return;
6284 case Intrinsic::aarch64_sve_sqdmulh_vgx4:
6286 Node->getValueType(0),
6287 {AArch64::SQDMULH_VG4_4Z4Z_B, AArch64::SQDMULH_VG4_4Z4Z_H,
6288 AArch64::SQDMULH_VG4_4Z4Z_S, AArch64::SQDMULH_VG4_4Z4Z_D}))
6289 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6290 return;
6291 case Intrinsic::aarch64_sme_fp8_scale_single_x2:
6293 Node->getValueType(0),
6294 {0, AArch64::FSCALE_2ZZ_H, AArch64::FSCALE_2ZZ_S,
6295 AArch64::FSCALE_2ZZ_D}))
6296 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6297 return;
6298 case Intrinsic::aarch64_sme_fp8_scale_single_x4:
6300 Node->getValueType(0),
6301 {0, AArch64::FSCALE_4ZZ_H, AArch64::FSCALE_4ZZ_S,
6302 AArch64::FSCALE_4ZZ_D}))
6303 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6304 return;
6305 case Intrinsic::aarch64_sme_fp8_scale_x2:
6307 Node->getValueType(0),
6308 {0, AArch64::FSCALE_2Z2Z_H, AArch64::FSCALE_2Z2Z_S,
6309 AArch64::FSCALE_2Z2Z_D}))
6310 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6311 return;
6312 case Intrinsic::aarch64_sme_fp8_scale_x4:
6314 Node->getValueType(0),
6315 {0, AArch64::FSCALE_4Z4Z_H, AArch64::FSCALE_4Z4Z_S,
6316 AArch64::FSCALE_4Z4Z_D}))
6317 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6318 return;
6319 case Intrinsic::aarch64_sve_whilege_x2:
6321 Node->getValueType(0),
6322 {AArch64::WHILEGE_2PXX_B, AArch64::WHILEGE_2PXX_H,
6323 AArch64::WHILEGE_2PXX_S, AArch64::WHILEGE_2PXX_D}))
6324 SelectWhilePair(Node, Op);
6325 return;
6326 case Intrinsic::aarch64_sve_whilegt_x2:
6328 Node->getValueType(0),
6329 {AArch64::WHILEGT_2PXX_B, AArch64::WHILEGT_2PXX_H,
6330 AArch64::WHILEGT_2PXX_S, AArch64::WHILEGT_2PXX_D}))
6331 SelectWhilePair(Node, Op);
6332 return;
6333 case Intrinsic::aarch64_sve_whilehi_x2:
6335 Node->getValueType(0),
6336 {AArch64::WHILEHI_2PXX_B, AArch64::WHILEHI_2PXX_H,
6337 AArch64::WHILEHI_2PXX_S, AArch64::WHILEHI_2PXX_D}))
6338 SelectWhilePair(Node, Op);
6339 return;
6340 case Intrinsic::aarch64_sve_whilehs_x2:
6342 Node->getValueType(0),
6343 {AArch64::WHILEHS_2PXX_B, AArch64::WHILEHS_2PXX_H,
6344 AArch64::WHILEHS_2PXX_S, AArch64::WHILEHS_2PXX_D}))
6345 SelectWhilePair(Node, Op);
6346 return;
6347 case Intrinsic::aarch64_sve_whilele_x2:
6349 Node->getValueType(0),
6350 {AArch64::WHILELE_2PXX_B, AArch64::WHILELE_2PXX_H,
6351 AArch64::WHILELE_2PXX_S, AArch64::WHILELE_2PXX_D}))
6352 SelectWhilePair(Node, Op);
6353 return;
6354 case Intrinsic::aarch64_sve_whilelo_x2:
6356 Node->getValueType(0),
6357 {AArch64::WHILELO_2PXX_B, AArch64::WHILELO_2PXX_H,
6358 AArch64::WHILELO_2PXX_S, AArch64::WHILELO_2PXX_D}))
6359 SelectWhilePair(Node, Op);
6360 return;
6361 case Intrinsic::aarch64_sve_whilels_x2:
6363 Node->getValueType(0),
6364 {AArch64::WHILELS_2PXX_B, AArch64::WHILELS_2PXX_H,
6365 AArch64::WHILELS_2PXX_S, AArch64::WHILELS_2PXX_D}))
6366 SelectWhilePair(Node, Op);
6367 return;
6368 case Intrinsic::aarch64_sve_whilelt_x2:
6370 Node->getValueType(0),
6371 {AArch64::WHILELT_2PXX_B, AArch64::WHILELT_2PXX_H,
6372 AArch64::WHILELT_2PXX_S, AArch64::WHILELT_2PXX_D}))
6373 SelectWhilePair(Node, Op);
6374 return;
6375 case Intrinsic::aarch64_sve_smax_single_x2:
6377 Node->getValueType(0),
6378 {AArch64::SMAX_VG2_2ZZ_B, AArch64::SMAX_VG2_2ZZ_H,
6379 AArch64::SMAX_VG2_2ZZ_S, AArch64::SMAX_VG2_2ZZ_D}))
6380 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6381 return;
6382 case Intrinsic::aarch64_sve_umax_single_x2:
6384 Node->getValueType(0),
6385 {AArch64::UMAX_VG2_2ZZ_B, AArch64::UMAX_VG2_2ZZ_H,
6386 AArch64::UMAX_VG2_2ZZ_S, AArch64::UMAX_VG2_2ZZ_D}))
6387 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6388 return;
6389 case Intrinsic::aarch64_sve_fmax_single_x2:
6391 Node->getValueType(0),
6392 {AArch64::BFMAX_VG2_2ZZ_H, AArch64::FMAX_VG2_2ZZ_H,
6393 AArch64::FMAX_VG2_2ZZ_S, AArch64::FMAX_VG2_2ZZ_D}))
6394 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6395 return;
6396 case Intrinsic::aarch64_sve_smax_single_x4:
6398 Node->getValueType(0),
6399 {AArch64::SMAX_VG4_4ZZ_B, AArch64::SMAX_VG4_4ZZ_H,
6400 AArch64::SMAX_VG4_4ZZ_S, AArch64::SMAX_VG4_4ZZ_D}))
6401 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6402 return;
6403 case Intrinsic::aarch64_sve_umax_single_x4:
6405 Node->getValueType(0),
6406 {AArch64::UMAX_VG4_4ZZ_B, AArch64::UMAX_VG4_4ZZ_H,
6407 AArch64::UMAX_VG4_4ZZ_S, AArch64::UMAX_VG4_4ZZ_D}))
6408 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6409 return;
6410 case Intrinsic::aarch64_sve_fmax_single_x4:
6412 Node->getValueType(0),
6413 {AArch64::BFMAX_VG4_4ZZ_H, AArch64::FMAX_VG4_4ZZ_H,
6414 AArch64::FMAX_VG4_4ZZ_S, AArch64::FMAX_VG4_4ZZ_D}))
6415 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6416 return;
6417 case Intrinsic::aarch64_sve_smin_single_x2:
6419 Node->getValueType(0),
6420 {AArch64::SMIN_VG2_2ZZ_B, AArch64::SMIN_VG2_2ZZ_H,
6421 AArch64::SMIN_VG2_2ZZ_S, AArch64::SMIN_VG2_2ZZ_D}))
6422 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6423 return;
6424 case Intrinsic::aarch64_sve_umin_single_x2:
6426 Node->getValueType(0),
6427 {AArch64::UMIN_VG2_2ZZ_B, AArch64::UMIN_VG2_2ZZ_H,
6428 AArch64::UMIN_VG2_2ZZ_S, AArch64::UMIN_VG2_2ZZ_D}))
6429 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6430 return;
6431 case Intrinsic::aarch64_sve_fmin_single_x2:
6433 Node->getValueType(0),
6434 {AArch64::BFMIN_VG2_2ZZ_H, AArch64::FMIN_VG2_2ZZ_H,
6435 AArch64::FMIN_VG2_2ZZ_S, AArch64::FMIN_VG2_2ZZ_D}))
6436 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6437 return;
6438 case Intrinsic::aarch64_sve_smin_single_x4:
6440 Node->getValueType(0),
6441 {AArch64::SMIN_VG4_4ZZ_B, AArch64::SMIN_VG4_4ZZ_H,
6442 AArch64::SMIN_VG4_4ZZ_S, AArch64::SMIN_VG4_4ZZ_D}))
6443 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6444 return;
6445 case Intrinsic::aarch64_sve_umin_single_x4:
6447 Node->getValueType(0),
6448 {AArch64::UMIN_VG4_4ZZ_B, AArch64::UMIN_VG4_4ZZ_H,
6449 AArch64::UMIN_VG4_4ZZ_S, AArch64::UMIN_VG4_4ZZ_D}))
6450 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6451 return;
6452 case Intrinsic::aarch64_sve_fmin_single_x4:
6454 Node->getValueType(0),
6455 {AArch64::BFMIN_VG4_4ZZ_H, AArch64::FMIN_VG4_4ZZ_H,
6456 AArch64::FMIN_VG4_4ZZ_S, AArch64::FMIN_VG4_4ZZ_D}))
6457 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6458 return;
6459 case Intrinsic::aarch64_sve_smax_x2:
6461 Node->getValueType(0),
6462 {AArch64::SMAX_VG2_2Z2Z_B, AArch64::SMAX_VG2_2Z2Z_H,
6463 AArch64::SMAX_VG2_2Z2Z_S, AArch64::SMAX_VG2_2Z2Z_D}))
6464 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6465 return;
6466 case Intrinsic::aarch64_sve_umax_x2:
6468 Node->getValueType(0),
6469 {AArch64::UMAX_VG2_2Z2Z_B, AArch64::UMAX_VG2_2Z2Z_H,
6470 AArch64::UMAX_VG2_2Z2Z_S, AArch64::UMAX_VG2_2Z2Z_D}))
6471 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6472 return;
6473 case Intrinsic::aarch64_sve_fmax_x2:
6475 Node->getValueType(0),
6476 {AArch64::BFMAX_VG2_2Z2Z_H, AArch64::FMAX_VG2_2Z2Z_H,
6477 AArch64::FMAX_VG2_2Z2Z_S, AArch64::FMAX_VG2_2Z2Z_D}))
6478 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6479 return;
6480 case Intrinsic::aarch64_sve_smax_x4:
6482 Node->getValueType(0),
6483 {AArch64::SMAX_VG4_4Z4Z_B, AArch64::SMAX_VG4_4Z4Z_H,
6484 AArch64::SMAX_VG4_4Z4Z_S, AArch64::SMAX_VG4_4Z4Z_D}))
6485 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6486 return;
6487 case Intrinsic::aarch64_sve_umax_x4:
6489 Node->getValueType(0),
6490 {AArch64::UMAX_VG4_4Z4Z_B, AArch64::UMAX_VG4_4Z4Z_H,
6491 AArch64::UMAX_VG4_4Z4Z_S, AArch64::UMAX_VG4_4Z4Z_D}))
6492 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6493 return;
6494 case Intrinsic::aarch64_sve_fmax_x4:
6496 Node->getValueType(0),
6497 {AArch64::BFMAX_VG4_4Z2Z_H, AArch64::FMAX_VG4_4Z4Z_H,
6498 AArch64::FMAX_VG4_4Z4Z_S, AArch64::FMAX_VG4_4Z4Z_D}))
6499 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6500 return;
6501 case Intrinsic::aarch64_sme_famax_x2:
6503 Node->getValueType(0),
6504 {0, AArch64::FAMAX_2Z2Z_H, AArch64::FAMAX_2Z2Z_S,
6505 AArch64::FAMAX_2Z2Z_D}))
6506 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6507 return;
6508 case Intrinsic::aarch64_sme_famax_x4:
6510 Node->getValueType(0),
6511 {0, AArch64::FAMAX_4Z4Z_H, AArch64::FAMAX_4Z4Z_S,
6512 AArch64::FAMAX_4Z4Z_D}))
6513 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6514 return;
6515 case Intrinsic::aarch64_sme_famin_x2:
6517 Node->getValueType(0),
6518 {0, AArch64::FAMIN_2Z2Z_H, AArch64::FAMIN_2Z2Z_S,
6519 AArch64::FAMIN_2Z2Z_D}))
6520 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6521 return;
6522 case Intrinsic::aarch64_sme_famin_x4:
6524 Node->getValueType(0),
6525 {0, AArch64::FAMIN_4Z4Z_H, AArch64::FAMIN_4Z4Z_S,
6526 AArch64::FAMIN_4Z4Z_D}))
6527 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6528 return;
6529 case Intrinsic::aarch64_sve_smin_x2:
6531 Node->getValueType(0),
6532 {AArch64::SMIN_VG2_2Z2Z_B, AArch64::SMIN_VG2_2Z2Z_H,
6533 AArch64::SMIN_VG2_2Z2Z_S, AArch64::SMIN_VG2_2Z2Z_D}))
6534 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6535 return;
6536 case Intrinsic::aarch64_sve_umin_x2:
6538 Node->getValueType(0),
6539 {AArch64::UMIN_VG2_2Z2Z_B, AArch64::UMIN_VG2_2Z2Z_H,
6540 AArch64::UMIN_VG2_2Z2Z_S, AArch64::UMIN_VG2_2Z2Z_D}))
6541 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6542 return;
6543 case Intrinsic::aarch64_sve_fmin_x2:
6545 Node->getValueType(0),
6546 {AArch64::BFMIN_VG2_2Z2Z_H, AArch64::FMIN_VG2_2Z2Z_H,
6547 AArch64::FMIN_VG2_2Z2Z_S, AArch64::FMIN_VG2_2Z2Z_D}))
6548 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6549 return;
6550 case Intrinsic::aarch64_sve_smin_x4:
6552 Node->getValueType(0),
6553 {AArch64::SMIN_VG4_4Z4Z_B, AArch64::SMIN_VG4_4Z4Z_H,
6554 AArch64::SMIN_VG4_4Z4Z_S, AArch64::SMIN_VG4_4Z4Z_D}))
6555 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6556 return;
6557 case Intrinsic::aarch64_sve_umin_x4:
6559 Node->getValueType(0),
6560 {AArch64::UMIN_VG4_4Z4Z_B, AArch64::UMIN_VG4_4Z4Z_H,
6561 AArch64::UMIN_VG4_4Z4Z_S, AArch64::UMIN_VG4_4Z4Z_D}))
6562 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6563 return;
6564 case Intrinsic::aarch64_sve_fmin_x4:
6566 Node->getValueType(0),
6567 {AArch64::BFMIN_VG4_4Z2Z_H, AArch64::FMIN_VG4_4Z4Z_H,
6568 AArch64::FMIN_VG4_4Z4Z_S, AArch64::FMIN_VG4_4Z4Z_D}))
6569 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6570 return;
6571 case Intrinsic::aarch64_sve_fmaxnm_single_x2 :
6573 Node->getValueType(0),
6574 {AArch64::BFMAXNM_VG2_2ZZ_H, AArch64::FMAXNM_VG2_2ZZ_H,
6575 AArch64::FMAXNM_VG2_2ZZ_S, AArch64::FMAXNM_VG2_2ZZ_D}))
6576 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6577 return;
6578 case Intrinsic::aarch64_sve_fmaxnm_single_x4 :
6580 Node->getValueType(0),
6581 {AArch64::BFMAXNM_VG4_4ZZ_H, AArch64::FMAXNM_VG4_4ZZ_H,
6582 AArch64::FMAXNM_VG4_4ZZ_S, AArch64::FMAXNM_VG4_4ZZ_D}))
6583 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6584 return;
6585 case Intrinsic::aarch64_sve_fminnm_single_x2:
6587 Node->getValueType(0),
6588 {AArch64::BFMINNM_VG2_2ZZ_H, AArch64::FMINNM_VG2_2ZZ_H,
6589 AArch64::FMINNM_VG2_2ZZ_S, AArch64::FMINNM_VG2_2ZZ_D}))
6590 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6591 return;
6592 case Intrinsic::aarch64_sve_fminnm_single_x4:
6594 Node->getValueType(0),
6595 {AArch64::BFMINNM_VG4_4ZZ_H, AArch64::FMINNM_VG4_4ZZ_H,
6596 AArch64::FMINNM_VG4_4ZZ_S, AArch64::FMINNM_VG4_4ZZ_D}))
6597 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6598 return;
6599 case Intrinsic::aarch64_sve_fscale_single_x4:
6600 SelectDestructiveMultiIntrinsic(Node, 4, false, AArch64::BFSCALE_4ZZ);
6601 return;
6602 case Intrinsic::aarch64_sve_fscale_single_x2:
6603 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::BFSCALE_2ZZ);
6604 return;
6605 case Intrinsic::aarch64_sve_fmul_single_x4:
6607 Node->getValueType(0),
6608 {AArch64::BFMUL_4ZZ, AArch64::FMUL_4ZZ_H, AArch64::FMUL_4ZZ_S,
6609 AArch64::FMUL_4ZZ_D}))
6610 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6611 return;
6612 case Intrinsic::aarch64_sve_fmul_single_x2:
6614 Node->getValueType(0),
6615 {AArch64::BFMUL_2ZZ, AArch64::FMUL_2ZZ_H, AArch64::FMUL_2ZZ_S,
6616 AArch64::FMUL_2ZZ_D}))
6617 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6618 return;
6619 case Intrinsic::aarch64_sve_fmaxnm_x2:
6621 Node->getValueType(0),
6622 {AArch64::BFMAXNM_VG2_2Z2Z_H, AArch64::FMAXNM_VG2_2Z2Z_H,
6623 AArch64::FMAXNM_VG2_2Z2Z_S, AArch64::FMAXNM_VG2_2Z2Z_D}))
6624 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6625 return;
6626 case Intrinsic::aarch64_sve_fmaxnm_x4:
6628 Node->getValueType(0),
6629 {AArch64::BFMAXNM_VG4_4Z2Z_H, AArch64::FMAXNM_VG4_4Z4Z_H,
6630 AArch64::FMAXNM_VG4_4Z4Z_S, AArch64::FMAXNM_VG4_4Z4Z_D}))
6631 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6632 return;
6633 case Intrinsic::aarch64_sve_fminnm_x2:
6635 Node->getValueType(0),
6636 {AArch64::BFMINNM_VG2_2Z2Z_H, AArch64::FMINNM_VG2_2Z2Z_H,
6637 AArch64::FMINNM_VG2_2Z2Z_S, AArch64::FMINNM_VG2_2Z2Z_D}))
6638 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6639 return;
6640 case Intrinsic::aarch64_sve_fminnm_x4:
6642 Node->getValueType(0),
6643 {AArch64::BFMINNM_VG4_4Z2Z_H, AArch64::FMINNM_VG4_4Z4Z_H,
6644 AArch64::FMINNM_VG4_4Z4Z_S, AArch64::FMINNM_VG4_4Z4Z_D}))
6645 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6646 return;
6647 case Intrinsic::aarch64_sve_aese_lane_x2:
6648 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::AESE_2ZZI_B);
6649 return;
6650 case Intrinsic::aarch64_sve_aesd_lane_x2:
6651 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::AESD_2ZZI_B);
6652 return;
6653 case Intrinsic::aarch64_sve_aesemc_lane_x2:
6654 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::AESEMC_2ZZI_B);
6655 return;
6656 case Intrinsic::aarch64_sve_aesdimc_lane_x2:
6657 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::AESDIMC_2ZZI_B);
6658 return;
6659 case Intrinsic::aarch64_sve_aese_lane_x4:
6660 SelectDestructiveMultiIntrinsic(Node, 4, false, AArch64::AESE_4ZZI_B);
6661 return;
6662 case Intrinsic::aarch64_sve_aesd_lane_x4:
6663 SelectDestructiveMultiIntrinsic(Node, 4, false, AArch64::AESD_4ZZI_B);
6664 return;
6665 case Intrinsic::aarch64_sve_aesemc_lane_x4:
6666 SelectDestructiveMultiIntrinsic(Node, 4, false, AArch64::AESEMC_4ZZI_B);
6667 return;
6668 case Intrinsic::aarch64_sve_aesdimc_lane_x4:
6669 SelectDestructiveMultiIntrinsic(Node, 4, false, AArch64::AESDIMC_4ZZI_B);
6670 return;
6671 case Intrinsic::aarch64_sve_pmlal_pair_x2:
6672 SelectDestructiveMultiIntrinsic(Node, 2, false, AArch64::PMLAL_2ZZZ_Q);
6673 return;
6674 case Intrinsic::aarch64_sve_pmull_pair_x2: {
6675 SDLoc DL(Node);
6676 SmallVector<SDValue, 4> Regs(Node->ops().slice(1, 2));
6677 SDNode *Res =
6678 CurDAG->getMachineNode(AArch64::PMULL_2ZZZ_Q, DL, MVT::Untyped, Regs);
6679 SDValue SuperReg = SDValue(Res, 0);
6680 for (unsigned I = 0; I < 2; I++)
6681 ReplaceUses(SDValue(Node, I),
6682 CurDAG->getTargetExtractSubreg(AArch64::zsub0 + I, DL, VT,
6683 SuperReg));
6684 CurDAG->RemoveDeadNode(Node);
6685 return;
6686 }
6687 case Intrinsic::aarch64_sve_fscale_x4:
6688 SelectDestructiveMultiIntrinsic(Node, 4, true, AArch64::BFSCALE_4Z4Z);
6689 return;
6690 case Intrinsic::aarch64_sve_fscale_x2:
6691 SelectDestructiveMultiIntrinsic(Node, 2, true, AArch64::BFSCALE_2Z2Z);
6692 return;
6693 case Intrinsic::aarch64_sve_fmul_x4:
6695 Node->getValueType(0),
6696 {AArch64::BFMUL_4Z4Z, AArch64::FMUL_4Z4Z_H, AArch64::FMUL_4Z4Z_S,
6697 AArch64::FMUL_4Z4Z_D}))
6698 SelectDestructiveMultiIntrinsic(Node, 4, true, Op);
6699 return;
6700 case Intrinsic::aarch64_sve_fmul_x2:
6702 Node->getValueType(0),
6703 {AArch64::BFMUL_2Z2Z, AArch64::FMUL_2Z2Z_H, AArch64::FMUL_2Z2Z_S,
6704 AArch64::FMUL_2Z2Z_D}))
6705 SelectDestructiveMultiIntrinsic(Node, 2, true, Op);
6706 return;
6707 case Intrinsic::aarch64_sve_fcvtzs_x2:
6708 SelectCVTIntrinsic(Node, 2, AArch64::FCVTZS_2Z2Z_StoS);
6709 return;
6710 case Intrinsic::aarch64_sve_scvtf_x2:
6711 SelectCVTIntrinsic(Node, 2, AArch64::SCVTF_2Z2Z_StoS);
6712 return;
6713 case Intrinsic::aarch64_sve_fcvtzu_x2:
6714 SelectCVTIntrinsic(Node, 2, AArch64::FCVTZU_2Z2Z_StoS);
6715 return;
6716 case Intrinsic::aarch64_sve_ucvtf_x2:
6717 SelectCVTIntrinsic(Node, 2, AArch64::UCVTF_2Z2Z_StoS);
6718 return;
6719 case Intrinsic::aarch64_sve_fcvtzs_x4:
6720 SelectCVTIntrinsic(Node, 4, AArch64::FCVTZS_4Z4Z_StoS);
6721 return;
6722 case Intrinsic::aarch64_sve_scvtf_x4:
6723 SelectCVTIntrinsic(Node, 4, AArch64::SCVTF_4Z4Z_StoS);
6724 return;
6725 case Intrinsic::aarch64_sve_fcvtzu_x4:
6726 SelectCVTIntrinsic(Node, 4, AArch64::FCVTZU_4Z4Z_StoS);
6727 return;
6728 case Intrinsic::aarch64_sve_ucvtf_x4:
6729 SelectCVTIntrinsic(Node, 4, AArch64::UCVTF_4Z4Z_StoS);
6730 return;
6731 case Intrinsic::aarch64_sve_fcvt_widen_x2:
6732 SelectUnaryMultiIntrinsic(Node, 2, false, AArch64::FCVT_2ZZ_H_S);
6733 return;
6734 case Intrinsic::aarch64_sve_fcvtl_widen_x2:
6735 SelectUnaryMultiIntrinsic(Node, 2, false, AArch64::FCVTL_2ZZ_H_S);
6736 return;
6737 case Intrinsic::aarch64_sve_sclamp_single_x2:
6739 Node->getValueType(0),
6740 {AArch64::SCLAMP_VG2_2Z2Z_B, AArch64::SCLAMP_VG2_2Z2Z_H,
6741 AArch64::SCLAMP_VG2_2Z2Z_S, AArch64::SCLAMP_VG2_2Z2Z_D}))
6742 SelectClamp(Node, 2, Op);
6743 return;
6744 case Intrinsic::aarch64_sve_uclamp_single_x2:
6746 Node->getValueType(0),
6747 {AArch64::UCLAMP_VG2_2Z2Z_B, AArch64::UCLAMP_VG2_2Z2Z_H,
6748 AArch64::UCLAMP_VG2_2Z2Z_S, AArch64::UCLAMP_VG2_2Z2Z_D}))
6749 SelectClamp(Node, 2, Op);
6750 return;
6751 case Intrinsic::aarch64_sve_fclamp_single_x2:
6753 Node->getValueType(0),
6754 {0, AArch64::FCLAMP_VG2_2Z2Z_H, AArch64::FCLAMP_VG2_2Z2Z_S,
6755 AArch64::FCLAMP_VG2_2Z2Z_D}))
6756 SelectClamp(Node, 2, Op);
6757 return;
6758 case Intrinsic::aarch64_sve_bfclamp_single_x2:
6759 SelectClamp(Node, 2, AArch64::BFCLAMP_VG2_2ZZZ_H);
6760 return;
6761 case Intrinsic::aarch64_sve_sclamp_single_x4:
6763 Node->getValueType(0),
6764 {AArch64::SCLAMP_VG4_4Z4Z_B, AArch64::SCLAMP_VG4_4Z4Z_H,
6765 AArch64::SCLAMP_VG4_4Z4Z_S, AArch64::SCLAMP_VG4_4Z4Z_D}))
6766 SelectClamp(Node, 4, Op);
6767 return;
6768 case Intrinsic::aarch64_sve_uclamp_single_x4:
6770 Node->getValueType(0),
6771 {AArch64::UCLAMP_VG4_4Z4Z_B, AArch64::UCLAMP_VG4_4Z4Z_H,
6772 AArch64::UCLAMP_VG4_4Z4Z_S, AArch64::UCLAMP_VG4_4Z4Z_D}))
6773 SelectClamp(Node, 4, Op);
6774 return;
6775 case Intrinsic::aarch64_sve_fclamp_single_x4:
6777 Node->getValueType(0),
6778 {0, AArch64::FCLAMP_VG4_4Z4Z_H, AArch64::FCLAMP_VG4_4Z4Z_S,
6779 AArch64::FCLAMP_VG4_4Z4Z_D}))
6780 SelectClamp(Node, 4, Op);
6781 return;
6782 case Intrinsic::aarch64_sve_bfclamp_single_x4:
6783 SelectClamp(Node, 4, AArch64::BFCLAMP_VG4_4ZZZ_H);
6784 return;
6785 case Intrinsic::aarch64_sve_add_single_x2:
6787 Node->getValueType(0),
6788 {AArch64::ADD_VG2_2ZZ_B, AArch64::ADD_VG2_2ZZ_H,
6789 AArch64::ADD_VG2_2ZZ_S, AArch64::ADD_VG2_2ZZ_D}))
6790 SelectDestructiveMultiIntrinsic(Node, 2, false, Op);
6791 return;
6792 case Intrinsic::aarch64_sve_add_single_x4:
6794 Node->getValueType(0),
6795 {AArch64::ADD_VG4_4ZZ_B, AArch64::ADD_VG4_4ZZ_H,
6796 AArch64::ADD_VG4_4ZZ_S, AArch64::ADD_VG4_4ZZ_D}))
6797 SelectDestructiveMultiIntrinsic(Node, 4, false, Op);
6798 return;
6799 case Intrinsic::aarch64_sve_zip_x2:
6801 Node->getValueType(0),
6802 {AArch64::ZIP_VG2_2ZZZ_B, AArch64::ZIP_VG2_2ZZZ_H,
6803 AArch64::ZIP_VG2_2ZZZ_S, AArch64::ZIP_VG2_2ZZZ_D}))
6804 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false, Op);
6805 return;
6806 case Intrinsic::aarch64_sve_zipq_x2:
6807 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false,
6808 AArch64::ZIP_VG2_2ZZZ_Q);
6809 return;
6810 case Intrinsic::aarch64_sve_zip_x4:
6812 Node->getValueType(0),
6813 {AArch64::ZIP_VG4_4Z4Z_B, AArch64::ZIP_VG4_4Z4Z_H,
6814 AArch64::ZIP_VG4_4Z4Z_S, AArch64::ZIP_VG4_4Z4Z_D}))
6815 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true, Op);
6816 return;
6817 case Intrinsic::aarch64_sve_zipq_x4:
6818 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true,
6819 AArch64::ZIP_VG4_4Z4Z_Q);
6820 return;
6821 case Intrinsic::aarch64_sve_uzp_x2:
6823 Node->getValueType(0),
6824 {AArch64::UZP_VG2_2ZZZ_B, AArch64::UZP_VG2_2ZZZ_H,
6825 AArch64::UZP_VG2_2ZZZ_S, AArch64::UZP_VG2_2ZZZ_D}))
6826 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false, Op);
6827 return;
6828 case Intrinsic::aarch64_sve_uzpq_x2:
6829 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false,
6830 AArch64::UZP_VG2_2ZZZ_Q);
6831 return;
6832 case Intrinsic::aarch64_sve_uzp_x4:
6834 Node->getValueType(0),
6835 {AArch64::UZP_VG4_4Z4Z_B, AArch64::UZP_VG4_4Z4Z_H,
6836 AArch64::UZP_VG4_4Z4Z_S, AArch64::UZP_VG4_4Z4Z_D}))
6837 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true, Op);
6838 return;
6839 case Intrinsic::aarch64_sve_uzpq_x4:
6840 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true,
6841 AArch64::UZP_VG4_4Z4Z_Q);
6842 return;
6843 case Intrinsic::aarch64_sve_sel_x2:
6845 Node->getValueType(0),
6846 {AArch64::SEL_VG2_2ZC2Z2Z_B, AArch64::SEL_VG2_2ZC2Z2Z_H,
6847 AArch64::SEL_VG2_2ZC2Z2Z_S, AArch64::SEL_VG2_2ZC2Z2Z_D}))
6848 SelectDestructiveMultiIntrinsic(Node, 2, true, Op, /*HasPred=*/true);
6849 return;
6850 case Intrinsic::aarch64_sve_sel_x4:
6852 Node->getValueType(0),
6853 {AArch64::SEL_VG4_4ZC4Z4Z_B, AArch64::SEL_VG4_4ZC4Z4Z_H,
6854 AArch64::SEL_VG4_4ZC4Z4Z_S, AArch64::SEL_VG4_4ZC4Z4Z_D}))
6855 SelectDestructiveMultiIntrinsic(Node, 4, true, Op, /*HasPred=*/true);
6856 return;
6857 case Intrinsic::aarch64_sve_frinta_x2:
6858 SelectFrintFromVT(Node, 2, AArch64::FRINTA_2Z2Z_S);
6859 return;
6860 case Intrinsic::aarch64_sve_frinta_x4:
6861 SelectFrintFromVT(Node, 4, AArch64::FRINTA_4Z4Z_S);
6862 return;
6863 case Intrinsic::aarch64_sve_frintm_x2:
6864 SelectFrintFromVT(Node, 2, AArch64::FRINTM_2Z2Z_S);
6865 return;
6866 case Intrinsic::aarch64_sve_frintm_x4:
6867 SelectFrintFromVT(Node, 4, AArch64::FRINTM_4Z4Z_S);
6868 return;
6869 case Intrinsic::aarch64_sve_frintn_x2:
6870 SelectFrintFromVT(Node, 2, AArch64::FRINTN_2Z2Z_S);
6871 return;
6872 case Intrinsic::aarch64_sve_frintn_x4:
6873 SelectFrintFromVT(Node, 4, AArch64::FRINTN_4Z4Z_S);
6874 return;
6875 case Intrinsic::aarch64_sve_frintp_x2:
6876 SelectFrintFromVT(Node, 2, AArch64::FRINTP_2Z2Z_S);
6877 return;
6878 case Intrinsic::aarch64_sve_frintp_x4:
6879 SelectFrintFromVT(Node, 4, AArch64::FRINTP_4Z4Z_S);
6880 return;
6881 case Intrinsic::aarch64_sve_sunpk_x2:
6883 Node->getValueType(0),
6884 {0, AArch64::SUNPK_VG2_2ZZ_H, AArch64::SUNPK_VG2_2ZZ_S,
6885 AArch64::SUNPK_VG2_2ZZ_D}))
6886 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false, Op);
6887 return;
6888 case Intrinsic::aarch64_sve_uunpk_x2:
6890 Node->getValueType(0),
6891 {0, AArch64::UUNPK_VG2_2ZZ_H, AArch64::UUNPK_VG2_2ZZ_S,
6892 AArch64::UUNPK_VG2_2ZZ_D}))
6893 SelectUnaryMultiIntrinsic(Node, 2, /*IsTupleInput=*/false, Op);
6894 return;
6895 case Intrinsic::aarch64_sve_sunpk_x4:
6897 Node->getValueType(0),
6898 {0, AArch64::SUNPK_VG4_4Z2Z_H, AArch64::SUNPK_VG4_4Z2Z_S,
6899 AArch64::SUNPK_VG4_4Z2Z_D}))
6900 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true, Op);
6901 return;
6902 case Intrinsic::aarch64_sve_uunpk_x4:
6904 Node->getValueType(0),
6905 {0, AArch64::UUNPK_VG4_4Z2Z_H, AArch64::UUNPK_VG4_4Z2Z_S,
6906 AArch64::UUNPK_VG4_4Z2Z_D}))
6907 SelectUnaryMultiIntrinsic(Node, 4, /*IsTupleInput=*/true, Op);
6908 return;
6909 case Intrinsic::aarch64_sve_pext_x2: {
6911 Node->getValueType(0),
6912 {AArch64::PEXT_2PCI_B, AArch64::PEXT_2PCI_H, AArch64::PEXT_2PCI_S,
6913 AArch64::PEXT_2PCI_D}))
6914 SelectPExtPair(Node, Op);
6915 return;
6916 }
6917 }
6918 break;
6919 }
6920 case ISD::INTRINSIC_VOID: {
6921 unsigned IntNo = Node->getConstantOperandVal(1);
6922 if (Node->getNumOperands() >= 3)
6923 VT = Node->getOperand(2)->getValueType(0);
6924 switch (IntNo) {
6925 default:
6926 break;
6927 case Intrinsic::aarch64_neon_st1x2: {
6928 if (VT == MVT::v8i8) {
6929 SelectStore(Node, 2, AArch64::ST1Twov8b);
6930 return;
6931 } else if (VT == MVT::v16i8) {
6932 SelectStore(Node, 2, AArch64::ST1Twov16b);
6933 return;
6934 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6935 VT == MVT::v4bf16) {
6936 SelectStore(Node, 2, AArch64::ST1Twov4h);
6937 return;
6938 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6939 VT == MVT::v8bf16) {
6940 SelectStore(Node, 2, AArch64::ST1Twov8h);
6941 return;
6942 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6943 SelectStore(Node, 2, AArch64::ST1Twov2s);
6944 return;
6945 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6946 SelectStore(Node, 2, AArch64::ST1Twov4s);
6947 return;
6948 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6949 SelectStore(Node, 2, AArch64::ST1Twov2d);
6950 return;
6951 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6952 SelectStore(Node, 2, AArch64::ST1Twov1d);
6953 return;
6954 }
6955 break;
6956 }
6957 case Intrinsic::aarch64_neon_st1x3: {
6958 if (VT == MVT::v8i8) {
6959 SelectStore(Node, 3, AArch64::ST1Threev8b);
6960 return;
6961 } else if (VT == MVT::v16i8) {
6962 SelectStore(Node, 3, AArch64::ST1Threev16b);
6963 return;
6964 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6965 VT == MVT::v4bf16) {
6966 SelectStore(Node, 3, AArch64::ST1Threev4h);
6967 return;
6968 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6969 VT == MVT::v8bf16) {
6970 SelectStore(Node, 3, AArch64::ST1Threev8h);
6971 return;
6972 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
6973 SelectStore(Node, 3, AArch64::ST1Threev2s);
6974 return;
6975 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
6976 SelectStore(Node, 3, AArch64::ST1Threev4s);
6977 return;
6978 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
6979 SelectStore(Node, 3, AArch64::ST1Threev2d);
6980 return;
6981 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
6982 SelectStore(Node, 3, AArch64::ST1Threev1d);
6983 return;
6984 }
6985 break;
6986 }
6987 case Intrinsic::aarch64_neon_st1x4: {
6988 if (VT == MVT::v8i8) {
6989 SelectStore(Node, 4, AArch64::ST1Fourv8b);
6990 return;
6991 } else if (VT == MVT::v16i8) {
6992 SelectStore(Node, 4, AArch64::ST1Fourv16b);
6993 return;
6994 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
6995 VT == MVT::v4bf16) {
6996 SelectStore(Node, 4, AArch64::ST1Fourv4h);
6997 return;
6998 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
6999 VT == MVT::v8bf16) {
7000 SelectStore(Node, 4, AArch64::ST1Fourv8h);
7001 return;
7002 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7003 SelectStore(Node, 4, AArch64::ST1Fourv2s);
7004 return;
7005 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7006 SelectStore(Node, 4, AArch64::ST1Fourv4s);
7007 return;
7008 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7009 SelectStore(Node, 4, AArch64::ST1Fourv2d);
7010 return;
7011 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7012 SelectStore(Node, 4, AArch64::ST1Fourv1d);
7013 return;
7014 }
7015 break;
7016 }
7017 case Intrinsic::aarch64_neon_st2: {
7018 if (VT == MVT::v8i8) {
7019 SelectStore(Node, 2, AArch64::ST2Twov8b);
7020 return;
7021 } else if (VT == MVT::v16i8) {
7022 SelectStore(Node, 2, AArch64::ST2Twov16b);
7023 return;
7024 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7025 VT == MVT::v4bf16) {
7026 SelectStore(Node, 2, AArch64::ST2Twov4h);
7027 return;
7028 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7029 VT == MVT::v8bf16) {
7030 SelectStore(Node, 2, AArch64::ST2Twov8h);
7031 return;
7032 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7033 SelectStore(Node, 2, AArch64::ST2Twov2s);
7034 return;
7035 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7036 SelectStore(Node, 2, AArch64::ST2Twov4s);
7037 return;
7038 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7039 SelectStore(Node, 2, AArch64::ST2Twov2d);
7040 return;
7041 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7042 SelectStore(Node, 2, AArch64::ST1Twov1d);
7043 return;
7044 }
7045 break;
7046 }
7047 case Intrinsic::aarch64_neon_st3: {
7048 if (VT == MVT::v8i8) {
7049 SelectStore(Node, 3, AArch64::ST3Threev8b);
7050 return;
7051 } else if (VT == MVT::v16i8) {
7052 SelectStore(Node, 3, AArch64::ST3Threev16b);
7053 return;
7054 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7055 VT == MVT::v4bf16) {
7056 SelectStore(Node, 3, AArch64::ST3Threev4h);
7057 return;
7058 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7059 VT == MVT::v8bf16) {
7060 SelectStore(Node, 3, AArch64::ST3Threev8h);
7061 return;
7062 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7063 SelectStore(Node, 3, AArch64::ST3Threev2s);
7064 return;
7065 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7066 SelectStore(Node, 3, AArch64::ST3Threev4s);
7067 return;
7068 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7069 SelectStore(Node, 3, AArch64::ST3Threev2d);
7070 return;
7071 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7072 SelectStore(Node, 3, AArch64::ST1Threev1d);
7073 return;
7074 }
7075 break;
7076 }
7077 case Intrinsic::aarch64_neon_st4: {
7078 if (VT == MVT::v8i8) {
7079 SelectStore(Node, 4, AArch64::ST4Fourv8b);
7080 return;
7081 } else if (VT == MVT::v16i8) {
7082 SelectStore(Node, 4, AArch64::ST4Fourv16b);
7083 return;
7084 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 ||
7085 VT == MVT::v4bf16) {
7086 SelectStore(Node, 4, AArch64::ST4Fourv4h);
7087 return;
7088 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 ||
7089 VT == MVT::v8bf16) {
7090 SelectStore(Node, 4, AArch64::ST4Fourv8h);
7091 return;
7092 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7093 SelectStore(Node, 4, AArch64::ST4Fourv2s);
7094 return;
7095 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7096 SelectStore(Node, 4, AArch64::ST4Fourv4s);
7097 return;
7098 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7099 SelectStore(Node, 4, AArch64::ST4Fourv2d);
7100 return;
7101 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7102 SelectStore(Node, 4, AArch64::ST1Fourv1d);
7103 return;
7104 }
7105 break;
7106 }
7107 case Intrinsic::aarch64_neon_st2lane: {
7108 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7109 SelectStoreLane(Node, 2, AArch64::ST2i8);
7110 return;
7111 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7112 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7113 SelectStoreLane(Node, 2, AArch64::ST2i16);
7114 return;
7115 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7116 VT == MVT::v2f32) {
7117 SelectStoreLane(Node, 2, AArch64::ST2i32);
7118 return;
7119 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7120 VT == MVT::v1f64) {
7121 SelectStoreLane(Node, 2, AArch64::ST2i64);
7122 return;
7123 }
7124 break;
7125 }
7126 case Intrinsic::aarch64_neon_st3lane: {
7127 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7128 SelectStoreLane(Node, 3, AArch64::ST3i8);
7129 return;
7130 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7131 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7132 SelectStoreLane(Node, 3, AArch64::ST3i16);
7133 return;
7134 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7135 VT == MVT::v2f32) {
7136 SelectStoreLane(Node, 3, AArch64::ST3i32);
7137 return;
7138 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7139 VT == MVT::v1f64) {
7140 SelectStoreLane(Node, 3, AArch64::ST3i64);
7141 return;
7142 }
7143 break;
7144 }
7145 case Intrinsic::aarch64_neon_st4lane: {
7146 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7147 SelectStoreLane(Node, 4, AArch64::ST4i8);
7148 return;
7149 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7150 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7151 SelectStoreLane(Node, 4, AArch64::ST4i16);
7152 return;
7153 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7154 VT == MVT::v2f32) {
7155 SelectStoreLane(Node, 4, AArch64::ST4i32);
7156 return;
7157 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7158 VT == MVT::v1f64) {
7159 SelectStoreLane(Node, 4, AArch64::ST4i64);
7160 return;
7161 }
7162 break;
7163 }
7164 case Intrinsic::aarch64_sve_st2q: {
7165 SelectPredicatedStore(Node, 2, 4, AArch64::ST2Q, AArch64::ST2Q_IMM);
7166 return;
7167 }
7168 case Intrinsic::aarch64_sve_st3q: {
7169 SelectPredicatedStore(Node, 3, 4, AArch64::ST3Q, AArch64::ST3Q_IMM);
7170 return;
7171 }
7172 case Intrinsic::aarch64_sve_st4q: {
7173 SelectPredicatedStore(Node, 4, 4, AArch64::ST4Q, AArch64::ST4Q_IMM);
7174 return;
7175 }
7176 case Intrinsic::aarch64_sve_st2: {
7177 if (VT == MVT::nxv16i8) {
7178 SelectPredicatedStore(Node, 2, 0, AArch64::ST2B, AArch64::ST2B_IMM);
7179 return;
7180 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7181 VT == MVT::nxv8bf16) {
7182 SelectPredicatedStore(Node, 2, 1, AArch64::ST2H, AArch64::ST2H_IMM);
7183 return;
7184 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7185 SelectPredicatedStore(Node, 2, 2, AArch64::ST2W, AArch64::ST2W_IMM);
7186 return;
7187 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7188 SelectPredicatedStore(Node, 2, 3, AArch64::ST2D, AArch64::ST2D_IMM);
7189 return;
7190 }
7191 break;
7192 }
7193 case Intrinsic::aarch64_sve_st3: {
7194 if (VT == MVT::nxv16i8) {
7195 SelectPredicatedStore(Node, 3, 0, AArch64::ST3B, AArch64::ST3B_IMM);
7196 return;
7197 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7198 VT == MVT::nxv8bf16) {
7199 SelectPredicatedStore(Node, 3, 1, AArch64::ST3H, AArch64::ST3H_IMM);
7200 return;
7201 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7202 SelectPredicatedStore(Node, 3, 2, AArch64::ST3W, AArch64::ST3W_IMM);
7203 return;
7204 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7205 SelectPredicatedStore(Node, 3, 3, AArch64::ST3D, AArch64::ST3D_IMM);
7206 return;
7207 }
7208 break;
7209 }
7210 case Intrinsic::aarch64_sve_st4: {
7211 if (VT == MVT::nxv16i8) {
7212 SelectPredicatedStore(Node, 4, 0, AArch64::ST4B, AArch64::ST4B_IMM);
7213 return;
7214 } else if (VT == MVT::nxv8i16 || VT == MVT::nxv8f16 ||
7215 VT == MVT::nxv8bf16) {
7216 SelectPredicatedStore(Node, 4, 1, AArch64::ST4H, AArch64::ST4H_IMM);
7217 return;
7218 } else if (VT == MVT::nxv4i32 || VT == MVT::nxv4f32) {
7219 SelectPredicatedStore(Node, 4, 2, AArch64::ST4W, AArch64::ST4W_IMM);
7220 return;
7221 } else if (VT == MVT::nxv2i64 || VT == MVT::nxv2f64) {
7222 SelectPredicatedStore(Node, 4, 3, AArch64::ST4D, AArch64::ST4D_IMM);
7223 return;
7224 }
7225 break;
7226 }
7227 }
7228 break;
7229 }
7230 case AArch64ISD::LD2post: {
7231 if (VT == MVT::v8i8) {
7232 SelectPostLoad(Node, 2, AArch64::LD2Twov8b_POST, AArch64::dsub0);
7233 return;
7234 } else if (VT == MVT::v16i8) {
7235 SelectPostLoad(Node, 2, AArch64::LD2Twov16b_POST, AArch64::qsub0);
7236 return;
7237 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7238 SelectPostLoad(Node, 2, AArch64::LD2Twov4h_POST, AArch64::dsub0);
7239 return;
7240 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7241 SelectPostLoad(Node, 2, AArch64::LD2Twov8h_POST, AArch64::qsub0);
7242 return;
7243 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7244 SelectPostLoad(Node, 2, AArch64::LD2Twov2s_POST, AArch64::dsub0);
7245 return;
7246 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7247 SelectPostLoad(Node, 2, AArch64::LD2Twov4s_POST, AArch64::qsub0);
7248 return;
7249 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7250 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
7251 return;
7252 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7253 SelectPostLoad(Node, 2, AArch64::LD2Twov2d_POST, AArch64::qsub0);
7254 return;
7255 }
7256 break;
7257 }
7258 case AArch64ISD::LD3post: {
7259 if (VT == MVT::v8i8) {
7260 SelectPostLoad(Node, 3, AArch64::LD3Threev8b_POST, AArch64::dsub0);
7261 return;
7262 } else if (VT == MVT::v16i8) {
7263 SelectPostLoad(Node, 3, AArch64::LD3Threev16b_POST, AArch64::qsub0);
7264 return;
7265 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7266 SelectPostLoad(Node, 3, AArch64::LD3Threev4h_POST, AArch64::dsub0);
7267 return;
7268 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7269 SelectPostLoad(Node, 3, AArch64::LD3Threev8h_POST, AArch64::qsub0);
7270 return;
7271 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7272 SelectPostLoad(Node, 3, AArch64::LD3Threev2s_POST, AArch64::dsub0);
7273 return;
7274 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7275 SelectPostLoad(Node, 3, AArch64::LD3Threev4s_POST, AArch64::qsub0);
7276 return;
7277 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7278 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
7279 return;
7280 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7281 SelectPostLoad(Node, 3, AArch64::LD3Threev2d_POST, AArch64::qsub0);
7282 return;
7283 }
7284 break;
7285 }
7286 case AArch64ISD::LD4post: {
7287 if (VT == MVT::v8i8) {
7288 SelectPostLoad(Node, 4, AArch64::LD4Fourv8b_POST, AArch64::dsub0);
7289 return;
7290 } else if (VT == MVT::v16i8) {
7291 SelectPostLoad(Node, 4, AArch64::LD4Fourv16b_POST, AArch64::qsub0);
7292 return;
7293 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7294 SelectPostLoad(Node, 4, AArch64::LD4Fourv4h_POST, AArch64::dsub0);
7295 return;
7296 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7297 SelectPostLoad(Node, 4, AArch64::LD4Fourv8h_POST, AArch64::qsub0);
7298 return;
7299 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7300 SelectPostLoad(Node, 4, AArch64::LD4Fourv2s_POST, AArch64::dsub0);
7301 return;
7302 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7303 SelectPostLoad(Node, 4, AArch64::LD4Fourv4s_POST, AArch64::qsub0);
7304 return;
7305 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7306 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
7307 return;
7308 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7309 SelectPostLoad(Node, 4, AArch64::LD4Fourv2d_POST, AArch64::qsub0);
7310 return;
7311 }
7312 break;
7313 }
7314 case AArch64ISD::LD1x2post: {
7315 if (VT == MVT::v8i8) {
7316 SelectPostLoad(Node, 2, AArch64::LD1Twov8b_POST, AArch64::dsub0);
7317 return;
7318 } else if (VT == MVT::v16i8) {
7319 SelectPostLoad(Node, 2, AArch64::LD1Twov16b_POST, AArch64::qsub0);
7320 return;
7321 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7322 SelectPostLoad(Node, 2, AArch64::LD1Twov4h_POST, AArch64::dsub0);
7323 return;
7324 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7325 SelectPostLoad(Node, 2, AArch64::LD1Twov8h_POST, AArch64::qsub0);
7326 return;
7327 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7328 SelectPostLoad(Node, 2, AArch64::LD1Twov2s_POST, AArch64::dsub0);
7329 return;
7330 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7331 SelectPostLoad(Node, 2, AArch64::LD1Twov4s_POST, AArch64::qsub0);
7332 return;
7333 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7334 SelectPostLoad(Node, 2, AArch64::LD1Twov1d_POST, AArch64::dsub0);
7335 return;
7336 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7337 SelectPostLoad(Node, 2, AArch64::LD1Twov2d_POST, AArch64::qsub0);
7338 return;
7339 }
7340 break;
7341 }
7342 case AArch64ISD::LD1x3post: {
7343 if (VT == MVT::v8i8) {
7344 SelectPostLoad(Node, 3, AArch64::LD1Threev8b_POST, AArch64::dsub0);
7345 return;
7346 } else if (VT == MVT::v16i8) {
7347 SelectPostLoad(Node, 3, AArch64::LD1Threev16b_POST, AArch64::qsub0);
7348 return;
7349 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7350 SelectPostLoad(Node, 3, AArch64::LD1Threev4h_POST, AArch64::dsub0);
7351 return;
7352 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7353 SelectPostLoad(Node, 3, AArch64::LD1Threev8h_POST, AArch64::qsub0);
7354 return;
7355 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7356 SelectPostLoad(Node, 3, AArch64::LD1Threev2s_POST, AArch64::dsub0);
7357 return;
7358 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7359 SelectPostLoad(Node, 3, AArch64::LD1Threev4s_POST, AArch64::qsub0);
7360 return;
7361 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7362 SelectPostLoad(Node, 3, AArch64::LD1Threev1d_POST, AArch64::dsub0);
7363 return;
7364 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7365 SelectPostLoad(Node, 3, AArch64::LD1Threev2d_POST, AArch64::qsub0);
7366 return;
7367 }
7368 break;
7369 }
7370 case AArch64ISD::LD1x4post: {
7371 if (VT == MVT::v8i8) {
7372 SelectPostLoad(Node, 4, AArch64::LD1Fourv8b_POST, AArch64::dsub0);
7373 return;
7374 } else if (VT == MVT::v16i8) {
7375 SelectPostLoad(Node, 4, AArch64::LD1Fourv16b_POST, AArch64::qsub0);
7376 return;
7377 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7378 SelectPostLoad(Node, 4, AArch64::LD1Fourv4h_POST, AArch64::dsub0);
7379 return;
7380 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7381 SelectPostLoad(Node, 4, AArch64::LD1Fourv8h_POST, AArch64::qsub0);
7382 return;
7383 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7384 SelectPostLoad(Node, 4, AArch64::LD1Fourv2s_POST, AArch64::dsub0);
7385 return;
7386 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7387 SelectPostLoad(Node, 4, AArch64::LD1Fourv4s_POST, AArch64::qsub0);
7388 return;
7389 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7390 SelectPostLoad(Node, 4, AArch64::LD1Fourv1d_POST, AArch64::dsub0);
7391 return;
7392 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7393 SelectPostLoad(Node, 4, AArch64::LD1Fourv2d_POST, AArch64::qsub0);
7394 return;
7395 }
7396 break;
7397 }
7398 case AArch64ISD::LD1DUPpost: {
7399 if (VT == MVT::v8i8) {
7400 SelectPostLoad(Node, 1, AArch64::LD1Rv8b_POST, AArch64::dsub0);
7401 return;
7402 } else if (VT == MVT::v16i8) {
7403 SelectPostLoad(Node, 1, AArch64::LD1Rv16b_POST, AArch64::qsub0);
7404 return;
7405 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7406 SelectPostLoad(Node, 1, AArch64::LD1Rv4h_POST, AArch64::dsub0);
7407 return;
7408 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7409 SelectPostLoad(Node, 1, AArch64::LD1Rv8h_POST, AArch64::qsub0);
7410 return;
7411 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7412 SelectPostLoad(Node, 1, AArch64::LD1Rv2s_POST, AArch64::dsub0);
7413 return;
7414 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7415 SelectPostLoad(Node, 1, AArch64::LD1Rv4s_POST, AArch64::qsub0);
7416 return;
7417 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7418 SelectPostLoad(Node, 1, AArch64::LD1Rv1d_POST, AArch64::dsub0);
7419 return;
7420 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7421 SelectPostLoad(Node, 1, AArch64::LD1Rv2d_POST, AArch64::qsub0);
7422 return;
7423 }
7424 break;
7425 }
7426 case AArch64ISD::LD2DUPpost: {
7427 if (VT == MVT::v8i8) {
7428 SelectPostLoad(Node, 2, AArch64::LD2Rv8b_POST, AArch64::dsub0);
7429 return;
7430 } else if (VT == MVT::v16i8) {
7431 SelectPostLoad(Node, 2, AArch64::LD2Rv16b_POST, AArch64::qsub0);
7432 return;
7433 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7434 SelectPostLoad(Node, 2, AArch64::LD2Rv4h_POST, AArch64::dsub0);
7435 return;
7436 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7437 SelectPostLoad(Node, 2, AArch64::LD2Rv8h_POST, AArch64::qsub0);
7438 return;
7439 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7440 SelectPostLoad(Node, 2, AArch64::LD2Rv2s_POST, AArch64::dsub0);
7441 return;
7442 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7443 SelectPostLoad(Node, 2, AArch64::LD2Rv4s_POST, AArch64::qsub0);
7444 return;
7445 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7446 SelectPostLoad(Node, 2, AArch64::LD2Rv1d_POST, AArch64::dsub0);
7447 return;
7448 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7449 SelectPostLoad(Node, 2, AArch64::LD2Rv2d_POST, AArch64::qsub0);
7450 return;
7451 }
7452 break;
7453 }
7454 case AArch64ISD::LD3DUPpost: {
7455 if (VT == MVT::v8i8) {
7456 SelectPostLoad(Node, 3, AArch64::LD3Rv8b_POST, AArch64::dsub0);
7457 return;
7458 } else if (VT == MVT::v16i8) {
7459 SelectPostLoad(Node, 3, AArch64::LD3Rv16b_POST, AArch64::qsub0);
7460 return;
7461 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7462 SelectPostLoad(Node, 3, AArch64::LD3Rv4h_POST, AArch64::dsub0);
7463 return;
7464 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7465 SelectPostLoad(Node, 3, AArch64::LD3Rv8h_POST, AArch64::qsub0);
7466 return;
7467 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7468 SelectPostLoad(Node, 3, AArch64::LD3Rv2s_POST, AArch64::dsub0);
7469 return;
7470 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7471 SelectPostLoad(Node, 3, AArch64::LD3Rv4s_POST, AArch64::qsub0);
7472 return;
7473 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7474 SelectPostLoad(Node, 3, AArch64::LD3Rv1d_POST, AArch64::dsub0);
7475 return;
7476 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7477 SelectPostLoad(Node, 3, AArch64::LD3Rv2d_POST, AArch64::qsub0);
7478 return;
7479 }
7480 break;
7481 }
7482 case AArch64ISD::LD4DUPpost: {
7483 if (VT == MVT::v8i8) {
7484 SelectPostLoad(Node, 4, AArch64::LD4Rv8b_POST, AArch64::dsub0);
7485 return;
7486 } else if (VT == MVT::v16i8) {
7487 SelectPostLoad(Node, 4, AArch64::LD4Rv16b_POST, AArch64::qsub0);
7488 return;
7489 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7490 SelectPostLoad(Node, 4, AArch64::LD4Rv4h_POST, AArch64::dsub0);
7491 return;
7492 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7493 SelectPostLoad(Node, 4, AArch64::LD4Rv8h_POST, AArch64::qsub0);
7494 return;
7495 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7496 SelectPostLoad(Node, 4, AArch64::LD4Rv2s_POST, AArch64::dsub0);
7497 return;
7498 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7499 SelectPostLoad(Node, 4, AArch64::LD4Rv4s_POST, AArch64::qsub0);
7500 return;
7501 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7502 SelectPostLoad(Node, 4, AArch64::LD4Rv1d_POST, AArch64::dsub0);
7503 return;
7504 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7505 SelectPostLoad(Node, 4, AArch64::LD4Rv2d_POST, AArch64::qsub0);
7506 return;
7507 }
7508 break;
7509 }
7510 case AArch64ISD::LD1LANEpost: {
7511 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7512 SelectPostLoadLane(Node, 1, AArch64::LD1i8_POST);
7513 return;
7514 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7515 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7516 SelectPostLoadLane(Node, 1, AArch64::LD1i16_POST);
7517 return;
7518 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7519 VT == MVT::v2f32) {
7520 SelectPostLoadLane(Node, 1, AArch64::LD1i32_POST);
7521 return;
7522 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7523 VT == MVT::v1f64) {
7524 SelectPostLoadLane(Node, 1, AArch64::LD1i64_POST);
7525 return;
7526 }
7527 break;
7528 }
7529 case AArch64ISD::LD2LANEpost: {
7530 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7531 SelectPostLoadLane(Node, 2, AArch64::LD2i8_POST);
7532 return;
7533 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7534 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7535 SelectPostLoadLane(Node, 2, AArch64::LD2i16_POST);
7536 return;
7537 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7538 VT == MVT::v2f32) {
7539 SelectPostLoadLane(Node, 2, AArch64::LD2i32_POST);
7540 return;
7541 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7542 VT == MVT::v1f64) {
7543 SelectPostLoadLane(Node, 2, AArch64::LD2i64_POST);
7544 return;
7545 }
7546 break;
7547 }
7548 case AArch64ISD::LD3LANEpost: {
7549 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7550 SelectPostLoadLane(Node, 3, AArch64::LD3i8_POST);
7551 return;
7552 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7553 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7554 SelectPostLoadLane(Node, 3, AArch64::LD3i16_POST);
7555 return;
7556 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7557 VT == MVT::v2f32) {
7558 SelectPostLoadLane(Node, 3, AArch64::LD3i32_POST);
7559 return;
7560 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7561 VT == MVT::v1f64) {
7562 SelectPostLoadLane(Node, 3, AArch64::LD3i64_POST);
7563 return;
7564 }
7565 break;
7566 }
7567 case AArch64ISD::LD4LANEpost: {
7568 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7569 SelectPostLoadLane(Node, 4, AArch64::LD4i8_POST);
7570 return;
7571 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7572 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7573 SelectPostLoadLane(Node, 4, AArch64::LD4i16_POST);
7574 return;
7575 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7576 VT == MVT::v2f32) {
7577 SelectPostLoadLane(Node, 4, AArch64::LD4i32_POST);
7578 return;
7579 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7580 VT == MVT::v1f64) {
7581 SelectPostLoadLane(Node, 4, AArch64::LD4i64_POST);
7582 return;
7583 }
7584 break;
7585 }
7586 case AArch64ISD::ST2post: {
7587 VT = Node->getOperand(1).getValueType();
7588 if (VT == MVT::v8i8) {
7589 SelectPostStore(Node, 2, AArch64::ST2Twov8b_POST);
7590 return;
7591 } else if (VT == MVT::v16i8) {
7592 SelectPostStore(Node, 2, AArch64::ST2Twov16b_POST);
7593 return;
7594 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7595 SelectPostStore(Node, 2, AArch64::ST2Twov4h_POST);
7596 return;
7597 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7598 SelectPostStore(Node, 2, AArch64::ST2Twov8h_POST);
7599 return;
7600 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7601 SelectPostStore(Node, 2, AArch64::ST2Twov2s_POST);
7602 return;
7603 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7604 SelectPostStore(Node, 2, AArch64::ST2Twov4s_POST);
7605 return;
7606 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7607 SelectPostStore(Node, 2, AArch64::ST2Twov2d_POST);
7608 return;
7609 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7610 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
7611 return;
7612 }
7613 break;
7614 }
7615 case AArch64ISD::ST3post: {
7616 VT = Node->getOperand(1).getValueType();
7617 if (VT == MVT::v8i8) {
7618 SelectPostStore(Node, 3, AArch64::ST3Threev8b_POST);
7619 return;
7620 } else if (VT == MVT::v16i8) {
7621 SelectPostStore(Node, 3, AArch64::ST3Threev16b_POST);
7622 return;
7623 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7624 SelectPostStore(Node, 3, AArch64::ST3Threev4h_POST);
7625 return;
7626 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7627 SelectPostStore(Node, 3, AArch64::ST3Threev8h_POST);
7628 return;
7629 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7630 SelectPostStore(Node, 3, AArch64::ST3Threev2s_POST);
7631 return;
7632 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7633 SelectPostStore(Node, 3, AArch64::ST3Threev4s_POST);
7634 return;
7635 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7636 SelectPostStore(Node, 3, AArch64::ST3Threev2d_POST);
7637 return;
7638 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7639 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
7640 return;
7641 }
7642 break;
7643 }
7644 case AArch64ISD::ST4post: {
7645 VT = Node->getOperand(1).getValueType();
7646 if (VT == MVT::v8i8) {
7647 SelectPostStore(Node, 4, AArch64::ST4Fourv8b_POST);
7648 return;
7649 } else if (VT == MVT::v16i8) {
7650 SelectPostStore(Node, 4, AArch64::ST4Fourv16b_POST);
7651 return;
7652 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7653 SelectPostStore(Node, 4, AArch64::ST4Fourv4h_POST);
7654 return;
7655 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7656 SelectPostStore(Node, 4, AArch64::ST4Fourv8h_POST);
7657 return;
7658 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7659 SelectPostStore(Node, 4, AArch64::ST4Fourv2s_POST);
7660 return;
7661 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7662 SelectPostStore(Node, 4, AArch64::ST4Fourv4s_POST);
7663 return;
7664 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7665 SelectPostStore(Node, 4, AArch64::ST4Fourv2d_POST);
7666 return;
7667 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7668 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
7669 return;
7670 }
7671 break;
7672 }
7673 case AArch64ISD::ST1x2post: {
7674 VT = Node->getOperand(1).getValueType();
7675 if (VT == MVT::v8i8) {
7676 SelectPostStore(Node, 2, AArch64::ST1Twov8b_POST);
7677 return;
7678 } else if (VT == MVT::v16i8) {
7679 SelectPostStore(Node, 2, AArch64::ST1Twov16b_POST);
7680 return;
7681 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7682 SelectPostStore(Node, 2, AArch64::ST1Twov4h_POST);
7683 return;
7684 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7685 SelectPostStore(Node, 2, AArch64::ST1Twov8h_POST);
7686 return;
7687 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7688 SelectPostStore(Node, 2, AArch64::ST1Twov2s_POST);
7689 return;
7690 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7691 SelectPostStore(Node, 2, AArch64::ST1Twov4s_POST);
7692 return;
7693 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7694 SelectPostStore(Node, 2, AArch64::ST1Twov1d_POST);
7695 return;
7696 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7697 SelectPostStore(Node, 2, AArch64::ST1Twov2d_POST);
7698 return;
7699 }
7700 break;
7701 }
7702 case AArch64ISD::ST1x3post: {
7703 VT = Node->getOperand(1).getValueType();
7704 if (VT == MVT::v8i8) {
7705 SelectPostStore(Node, 3, AArch64::ST1Threev8b_POST);
7706 return;
7707 } else if (VT == MVT::v16i8) {
7708 SelectPostStore(Node, 3, AArch64::ST1Threev16b_POST);
7709 return;
7710 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7711 SelectPostStore(Node, 3, AArch64::ST1Threev4h_POST);
7712 return;
7713 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16 ) {
7714 SelectPostStore(Node, 3, AArch64::ST1Threev8h_POST);
7715 return;
7716 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7717 SelectPostStore(Node, 3, AArch64::ST1Threev2s_POST);
7718 return;
7719 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7720 SelectPostStore(Node, 3, AArch64::ST1Threev4s_POST);
7721 return;
7722 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7723 SelectPostStore(Node, 3, AArch64::ST1Threev1d_POST);
7724 return;
7725 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7726 SelectPostStore(Node, 3, AArch64::ST1Threev2d_POST);
7727 return;
7728 }
7729 break;
7730 }
7731 case AArch64ISD::ST1x4post: {
7732 VT = Node->getOperand(1).getValueType();
7733 if (VT == MVT::v8i8) {
7734 SelectPostStore(Node, 4, AArch64::ST1Fourv8b_POST);
7735 return;
7736 } else if (VT == MVT::v16i8) {
7737 SelectPostStore(Node, 4, AArch64::ST1Fourv16b_POST);
7738 return;
7739 } else if (VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16) {
7740 SelectPostStore(Node, 4, AArch64::ST1Fourv4h_POST);
7741 return;
7742 } else if (VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v8bf16) {
7743 SelectPostStore(Node, 4, AArch64::ST1Fourv8h_POST);
7744 return;
7745 } else if (VT == MVT::v2i32 || VT == MVT::v2f32) {
7746 SelectPostStore(Node, 4, AArch64::ST1Fourv2s_POST);
7747 return;
7748 } else if (VT == MVT::v4i32 || VT == MVT::v4f32) {
7749 SelectPostStore(Node, 4, AArch64::ST1Fourv4s_POST);
7750 return;
7751 } else if (VT == MVT::v1i64 || VT == MVT::v1f64) {
7752 SelectPostStore(Node, 4, AArch64::ST1Fourv1d_POST);
7753 return;
7754 } else if (VT == MVT::v2i64 || VT == MVT::v2f64) {
7755 SelectPostStore(Node, 4, AArch64::ST1Fourv2d_POST);
7756 return;
7757 }
7758 break;
7759 }
7760 case AArch64ISD::ST2LANEpost: {
7761 VT = Node->getOperand(1).getValueType();
7762 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7763 SelectPostStoreLane(Node, 2, AArch64::ST2i8_POST);
7764 return;
7765 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7766 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7767 SelectPostStoreLane(Node, 2, AArch64::ST2i16_POST);
7768 return;
7769 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7770 VT == MVT::v2f32) {
7771 SelectPostStoreLane(Node, 2, AArch64::ST2i32_POST);
7772 return;
7773 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7774 VT == MVT::v1f64) {
7775 SelectPostStoreLane(Node, 2, AArch64::ST2i64_POST);
7776 return;
7777 }
7778 break;
7779 }
7780 case AArch64ISD::ST3LANEpost: {
7781 VT = Node->getOperand(1).getValueType();
7782 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7783 SelectPostStoreLane(Node, 3, AArch64::ST3i8_POST);
7784 return;
7785 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7786 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7787 SelectPostStoreLane(Node, 3, AArch64::ST3i16_POST);
7788 return;
7789 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7790 VT == MVT::v2f32) {
7791 SelectPostStoreLane(Node, 3, AArch64::ST3i32_POST);
7792 return;
7793 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7794 VT == MVT::v1f64) {
7795 SelectPostStoreLane(Node, 3, AArch64::ST3i64_POST);
7796 return;
7797 }
7798 break;
7799 }
7800 case AArch64ISD::ST4LANEpost: {
7801 VT = Node->getOperand(1).getValueType();
7802 if (VT == MVT::v16i8 || VT == MVT::v8i8) {
7803 SelectPostStoreLane(Node, 4, AArch64::ST4i8_POST);
7804 return;
7805 } else if (VT == MVT::v8i16 || VT == MVT::v4i16 || VT == MVT::v4f16 ||
7806 VT == MVT::v8f16 || VT == MVT::v4bf16 || VT == MVT::v8bf16) {
7807 SelectPostStoreLane(Node, 4, AArch64::ST4i16_POST);
7808 return;
7809 } else if (VT == MVT::v4i32 || VT == MVT::v2i32 || VT == MVT::v4f32 ||
7810 VT == MVT::v2f32) {
7811 SelectPostStoreLane(Node, 4, AArch64::ST4i32_POST);
7812 return;
7813 } else if (VT == MVT::v2i64 || VT == MVT::v1i64 || VT == MVT::v2f64 ||
7814 VT == MVT::v1f64) {
7815 SelectPostStoreLane(Node, 4, AArch64::ST4i64_POST);
7816 return;
7817 }
7818 break;
7819 }
7820 }
7821
7822 // Select the default instruction
7823 SelectCode(Node);
7824}
7825
7826/// createAArch64ISelDag - This pass converts a legalized DAG into a
7827/// AArch64-specific DAG, ready for instruction scheduling.
7829 CodeGenOptLevel OptLevel) {
7830 return new AArch64DAGToDAGISelLegacy(TM, OptLevel);
7831}
7832
7833/// When \p PredVT is a scalable vector predicate in the form
7834/// MVT::nx<M>xi1, it builds the correspondent scalable vector of
7835/// integers MVT::nx<M>xi<bits> s.t. M x bits = 128. When targeting
7836/// structured vectors (NumVec >1), the output data type is
7837/// MVT::nx<M*NumVec>xi<bits> s.t. M x bits = 128. If the input
7838/// PredVT is not in the form MVT::nx<M>xi1, it returns an invalid
7839/// EVT.
7841 unsigned NumVec) {
7842 assert(NumVec > 0 && NumVec < 5 && "Invalid number of vectors.");
7843 if (!PredVT.isScalableVectorOf(MVT::i1))
7844 return EVT();
7845
7846 if (PredVT != MVT::nxv16i1 && PredVT != MVT::nxv8i1 &&
7847 PredVT != MVT::nxv4i1 && PredVT != MVT::nxv2i1)
7848 return EVT();
7849
7850 ElementCount EC = PredVT.getVectorElementCount();
7851 EVT ScalarVT =
7852 EVT::getIntegerVT(Ctx, AArch64::SVEBitsPerBlock / EC.getKnownMinValue());
7853 EVT MemVT = EVT::getVectorVT(Ctx, ScalarVT, EC * NumVec);
7854
7855 return MemVT;
7856}
7857
7858/// Builds an integer vector type large enough to hold \p NumVec instances
7859/// of \p VecVT.
7860static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec) {
7862 VecVT.getVectorElementCount() * NumVec);
7863}
7864
7865/// Return the EVT of the data associated to a memory operation in \p
7866/// Root. If such EVT cannot be retrieved, it returns an invalid EVT.
7868 if (auto *MemIntr = dyn_cast<MemIntrinsicSDNode>(Root))
7869 return MemIntr->getMemoryVT();
7870
7871 if (isa<MemSDNode>(Root)) {
7872 EVT MemVT = cast<MemSDNode>(Root)->getMemoryVT();
7873
7874 EVT DataVT;
7875 if (auto *Load = dyn_cast<LoadSDNode>(Root))
7876 DataVT = Load->getValueType(0);
7877 else if (auto *Load = dyn_cast<MaskedLoadSDNode>(Root))
7878 DataVT = Load->getValueType(0);
7879 else if (auto *Store = dyn_cast<StoreSDNode>(Root))
7880 DataVT = Store->getValue().getValueType();
7881 else if (auto *Store = dyn_cast<MaskedStoreSDNode>(Root))
7882 DataVT = Store->getValue().getValueType();
7883 else
7884 llvm_unreachable("Unexpected MemSDNode!");
7885
7886 return DataVT.changeVectorElementType(Ctx, MemVT.getVectorElementType());
7887 }
7888
7889 const unsigned Opcode = Root->getOpcode();
7890 // For custom ISD nodes, we have to look at them individually to extract the
7891 // type of the data moved to/from memory.
7892 switch (Opcode) {
7893 case AArch64ISD::LD1_MERGE_ZERO:
7894 case AArch64ISD::LD1S_MERGE_ZERO:
7895 case AArch64ISD::LDNF1_MERGE_ZERO:
7896 case AArch64ISD::LDNF1S_MERGE_ZERO:
7897 return cast<VTSDNode>(Root->getOperand(3))->getVT();
7898 case AArch64ISD::ST1_PRED:
7899 return cast<VTSDNode>(Root->getOperand(4))->getVT();
7900 default:
7901 break;
7902 }
7903
7904 if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN)
7905 return EVT();
7906
7907 switch (Root->getConstantOperandVal(1)) {
7908 default:
7909 return EVT();
7910 case Intrinsic::aarch64_sme_ldr:
7911 case Intrinsic::aarch64_sme_str:
7912 return MVT::nxv16i8;
7913 case Intrinsic::aarch64_sve_prf:
7914 // We are using an SVE prefetch intrinsic. Type must be inferred from the
7915 // width of the predicate.
7917 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1);
7918 case Intrinsic::aarch64_sve_ld2_sret:
7919 case Intrinsic::aarch64_sve_ld2q_sret:
7921 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/2);
7922 case Intrinsic::aarch64_sve_st2q:
7924 Ctx, Root->getOperand(4)->getValueType(0), /*NumVec=*/2);
7925 case Intrinsic::aarch64_sve_ld3_sret:
7926 case Intrinsic::aarch64_sve_ld3q_sret:
7928 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/3);
7929 case Intrinsic::aarch64_sve_st3q:
7931 Ctx, Root->getOperand(5)->getValueType(0), /*NumVec=*/3);
7932 case Intrinsic::aarch64_sve_ld4_sret:
7933 case Intrinsic::aarch64_sve_ld4q_sret:
7935 Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/4);
7936 case Intrinsic::aarch64_sve_st4q:
7938 Ctx, Root->getOperand(6)->getValueType(0), /*NumVec=*/4);
7939 case Intrinsic::aarch64_sve_ld1_pn_x2:
7940 case Intrinsic::aarch64_sve_ldnt1_pn_x2:
7941 return getMultipleVectorType(Ctx, Root->getValueType(0),
7942 /*NumVec=*/2);
7943 case Intrinsic::aarch64_sve_ld1_pn_x4:
7944 case Intrinsic::aarch64_sve_ldnt1_pn_x4:
7945 return getMultipleVectorType(Ctx, Root->getValueType(0),
7946 /*NumVec=*/4);
7947 case Intrinsic::aarch64_sve_st1_pn_x2:
7948 case Intrinsic::aarch64_sve_stnt1_pn_x2:
7949 return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
7950 /*NumVec=*/2);
7951 case Intrinsic::aarch64_sve_st1_pn_x4:
7952 case Intrinsic::aarch64_sve_stnt1_pn_x4:
7953 return getMultipleVectorType(Ctx, Root->getOperand(2).getValueType(),
7954 /*NumVec=*/4);
7955 case Intrinsic::aarch64_sve_ld1udq:
7956 case Intrinsic::aarch64_sve_st1dq:
7957 return EVT(MVT::nxv1i64);
7958 case Intrinsic::aarch64_sve_ld1uwq:
7959 case Intrinsic::aarch64_sve_st1wq:
7960 return EVT(MVT::nxv1i32);
7961 }
7962}
7963
7964/// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode:
7965/// Base + OffImm * sizeof(MemVT) for Min >= OffImm <= Max
7966/// where Root is the memory access using N for its address.
7967template <int64_t Min, int64_t Max>
7968bool AArch64DAGToDAGISel::SelectAddrModeIndexedSVE(SDNode *Root, SDValue N,
7969 SDValue &Base,
7970 SDValue &OffImm) {
7971 const EVT MemVT = getMemVTFromNode(*(CurDAG->getContext()), Root);
7972 const DataLayout &DL = CurDAG->getDataLayout();
7973 const MachineFrameInfo &MFI = MF->getFrameInfo();
7974
7975 if (N.getOpcode() == ISD::FrameIndex) {
7976 int FI = cast<FrameIndexSDNode>(N)->getIndex();
7977 // We can only encode VL scaled offsets, so only fold in frame indexes
7978 // referencing SVE objects.
7979 if (MFI.hasScalableStackID(FI)) {
7980 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
7981 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
7982 return true;
7983 }
7984
7985 return false;
7986 }
7987
7988 if (MemVT == EVT())
7989 return false;
7990
7991 if (N.getOpcode() != ISD::ADD)
7992 return false;
7993
7994 SDValue VScale = N.getOperand(1);
7995 int64_t MulImm = std::numeric_limits<int64_t>::max();
7996 if (VScale.getOpcode() == ISD::VSCALE) {
7997 MulImm = cast<ConstantSDNode>(VScale.getOperand(0))->getSExtValue();
7998 } else if (auto C = dyn_cast<ConstantSDNode>(VScale)) {
7999 int64_t ByteOffset = C->getSExtValue();
8000 const auto KnownVScale =
8002
8003 if (!KnownVScale || ByteOffset % KnownVScale != 0)
8004 return false;
8005
8006 MulImm = ByteOffset / KnownVScale;
8007 } else
8008 return false;
8009
8010 TypeSize TS = MemVT.getSizeInBits();
8011 int64_t MemWidthBytes = static_cast<int64_t>(TS.getKnownMinValue()) / 8;
8012
8013 if ((MulImm % MemWidthBytes) != 0)
8014 return false;
8015
8016 int64_t Offset = MulImm / MemWidthBytes;
8018 return false;
8019
8020 Base = N.getOperand(0);
8021 if (Base.getOpcode() == ISD::FrameIndex) {
8022 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
8023 // We can only encode VL scaled offsets, so only fold in frame indexes
8024 // referencing SVE objects.
8025 if (MFI.hasScalableStackID(FI))
8026 Base = CurDAG->getTargetFrameIndex(FI, TLI->getPointerTy(DL));
8027 }
8028
8029 OffImm = CurDAG->getTargetConstant(Offset, SDLoc(N), MVT::i64);
8030 return true;
8031}
8032
8033/// Select register plus register addressing mode for SVE, with scaled
8034/// offset.
8035bool AArch64DAGToDAGISel::SelectSVERegRegAddrMode(SDValue N, unsigned Scale,
8036 SDValue &Base,
8037 SDValue &Offset) {
8038 if (N.getOpcode() != ISD::ADD)
8039 return false;
8040
8041 // Process an ADD node.
8042 const SDValue LHS = N.getOperand(0);
8043 const SDValue RHS = N.getOperand(1);
8044
8045 // 8 bit data does not come with the SHL node, so it is treated
8046 // separately.
8047 if (Scale == 0) {
8048 Base = LHS;
8049 Offset = RHS;
8050 return true;
8051 }
8052
8053 if (auto C = dyn_cast<ConstantSDNode>(RHS)) {
8054 int64_t ImmOff = C->getSExtValue();
8055 unsigned Size = 1 << Scale;
8056
8057 // To use the reg+reg addressing mode, the immediate must be a multiple of
8058 // the vector element's byte size.
8059 if (ImmOff % Size)
8060 return false;
8061
8062 SDLoc DL(N);
8063 Base = LHS;
8064 Offset = CurDAG->getTargetConstant(ImmOff >> Scale, DL, MVT::i64);
8065 SDValue Ops[] = {Offset};
8066 SDNode *MI = CurDAG->getMachineNode(AArch64::MOVi64imm, DL, MVT::i64, Ops);
8067 Offset = SDValue(MI, 0);
8068 return true;
8069 }
8070
8071 // Check if the RHS is a shift node with a constant.
8072 if (RHS.getOpcode() != ISD::SHL)
8073 return false;
8074
8075 const SDValue ShiftRHS = RHS.getOperand(1);
8076 if (auto *C = dyn_cast<ConstantSDNode>(ShiftRHS))
8077 if (C->getZExtValue() == Scale) {
8078 Base = LHS;
8079 Offset = RHS.getOperand(0);
8080 return true;
8081 }
8082
8083 return false;
8084}
8085
8086bool AArch64DAGToDAGISel::SelectAllActivePredicate(SDValue N) {
8087 const AArch64TargetLowering *TLI =
8088 static_cast<const AArch64TargetLowering *>(getTargetLowering());
8089
8090 return TLI->isAllActivePredicate(*CurDAG, N);
8091}
8092
8093bool AArch64DAGToDAGISel::SelectAnyPredicate(SDValue N) {
8094 return N.getValueType().isScalableVectorOf(MVT::i1);
8095}
8096
8097bool AArch64DAGToDAGISel::SelectSMETileSlice(SDValue N, unsigned MaxSize,
8099 unsigned Scale) {
8100 auto MatchConstantOffset = [&](SDValue CN) -> SDValue {
8101 if (auto *C = dyn_cast<ConstantSDNode>(CN)) {
8102 int64_t ImmOff = C->getSExtValue();
8103 if ((ImmOff > 0 && ImmOff <= MaxSize && (ImmOff % Scale == 0)))
8104 return CurDAG->getTargetConstant(ImmOff / Scale, SDLoc(N), MVT::i64);
8105 }
8106 return SDValue();
8107 };
8108
8109 if (SDValue C = MatchConstantOffset(N)) {
8110 Base = getZeroRegister(*CurDAG, SDLoc(N), MVT::i32);
8111 Offset = C;
8112 return true;
8113 }
8114
8115 // Try to untangle an ADD node into a 'reg + offset'
8116 if (CurDAG->isBaseWithConstantOffset(N)) {
8117 if (SDValue C = MatchConstantOffset(N.getOperand(1))) {
8118 Base = N.getOperand(0);
8119 Offset = C;
8120 return true;
8121 }
8122 }
8123
8124 // By default, just match reg + 0.
8125 Base = N;
8126 Offset = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i64);
8127 return true;
8128}
8129
8130bool AArch64DAGToDAGISel::SelectCmpBranchUImm6Operand(SDNode *P, SDValue N,
8131 SDValue &Imm) {
8133 static_cast<AArch64CC::CondCode>(P->getConstantOperandVal(1));
8134 if (auto *CN = dyn_cast<ConstantSDNode>(N)) {
8135 // Check conservatively if the immediate fits the valid range [0, 64).
8136 // Immediate variants for GE and HS definitely need to be decremented
8137 // when lowering the pseudos later, so an immediate of 1 would become 0.
8138 // For the inverse conditions LT and LO we don't know for sure if they
8139 // will need a decrement but should the decision be made to reverse the
8140 // branch condition, we again end up with the need to decrement.
8141 // The same argument holds for LE, LS, GT and HI and possibly
8142 // incremented immediates. This can lead to slightly less optimal
8143 // codegen, e.g. we never codegen the legal case
8144 // cblt w0, #63, A
8145 // because we could end up with the illegal case
8146 // cbge w0, #64, B
8147 // should the decision to reverse the branch direction be made. For the
8148 // lower bound cases this is no problem since we can express comparisons
8149 // against 0 with either tbz/tnbz or using wzr/xzr.
8150 uint64_t LowerBound = 0, UpperBound = 64;
8151 switch (CC) {
8152 case AArch64CC::GE:
8153 case AArch64CC::HS:
8154 case AArch64CC::LT:
8155 case AArch64CC::LO:
8156 LowerBound = 1;
8157 break;
8158 case AArch64CC::LE:
8159 case AArch64CC::LS:
8160 case AArch64CC::GT:
8161 case AArch64CC::HI:
8162 UpperBound = 63;
8163 break;
8164 default:
8165 break;
8166 }
8167
8168 if (CN->getAPIntValue().uge(LowerBound) &&
8169 CN->getAPIntValue().ult(UpperBound)) {
8170 SDLoc DL(N);
8171 Imm = CurDAG->getTargetConstant(CN->getZExtValue(), DL, N.getValueType());
8172 return true;
8173 }
8174 }
8175
8176 return false;
8177}
8178
8179template <bool MatchCBB>
8180bool AArch64DAGToDAGISel::SelectCmpBranchExtOperand(SDValue N, SDValue &Reg,
8181 SDValue &ExtType) {
8182
8183 // Use an invalid shift-extend value to indicate we don't need to extend later
8184 if (N.getOpcode() == ISD::AssertZext || N.getOpcode() == ISD::AssertSext) {
8185 EVT Ty = cast<VTSDNode>(N.getOperand(1))->getVT();
8186 if (Ty != (MatchCBB ? MVT::i8 : MVT::i16))
8187 return false;
8188 Reg = N.getOperand(0);
8189 ExtType = CurDAG->getSignedTargetConstant(AArch64_AM::InvalidShiftExtend,
8190 SDLoc(N), MVT::i32);
8191 return true;
8192 }
8193
8195
8196 if ((MatchCBB && (ET == AArch64_AM::UXTB || ET == AArch64_AM::SXTB)) ||
8197 (!MatchCBB && (ET == AArch64_AM::UXTH || ET == AArch64_AM::SXTH))) {
8198 Reg = N.getOperand(0);
8199 ExtType =
8200 CurDAG->getTargetConstant(getExtendEncoding(ET), SDLoc(N), MVT::i32);
8201 return true;
8202 }
8203
8204 return false;
8205}
8206
8207/// Try to fold AArch64 CSEL/FCMP patterns to FMAXNM/FMINNM.
8208///
8209/// This is intentionally done in PreprocessISelDAG rather than DAGCombine:
8210/// doing this earlier based on the defining operation of X can be invalidated
8211/// by later DAG combines. At this point the DAG is being prepared for
8212/// instruction selection, so the use of isKnownNeverSNaN(X) applies to the
8213/// final SDValue being selected.
8214/// Only handles FCMP(X, C) with scalar FP types, where C is a non-NaN constant.
8215/// The nsz requirement is needed only when C is zero, to avoid signed-zero
8216/// mismatches. The never-sNaN check is required because AArch64 FMAXNM/FMINNM
8217/// differ from fcmp+fcsel for signaling NaN inputs.
8218bool AArch64DAGToDAGISel::tryFoldCselToFMaxMin(SDNode *N) {
8219 EVT VT = N->getValueType(0);
8220
8221 // Scalar FP only.
8222 if (!VT.isFloatingPoint() || VT.isVector())
8223 return false;
8224
8225 SDValue TVal = N->getOperand(0);
8226 SDValue FVal = N->getOperand(1);
8227 SDValue CCVal = N->getOperand(2);
8228 SDValue Cmp = N->getOperand(3);
8229
8230 if (Cmp.getOpcode() != AArch64ISD::FCMP)
8231 return false;
8232
8233 auto *CC = dyn_cast<ConstantSDNode>(CCVal);
8234 if (!CC)
8235 return false;
8236
8237 SDValue CmpLHS = Cmp.getOperand(0);
8238 SDValue CmpRHS = Cmp.getOperand(1);
8239 unsigned CondCode = CC->getZExtValue();
8240
8241 // Map VT and operation (max/min) to machine opcode.
8242 auto getOpc = [](EVT VT, bool isMax) -> unsigned {
8243 if (VT == MVT::f16)
8244 return isMax ? AArch64::FMAXNMHrr : AArch64::FMINNMHrr;
8245 else if (VT == MVT::f32)
8246 return isMax ? AArch64::FMAXNMSrr : AArch64::FMINNMSrr;
8247 else if (VT == MVT::f64)
8248 return isMax ? AArch64::FMAXNMDrr : AArch64::FMINNMDrr;
8249 else
8250 return 0; // unsupported
8251 };
8252
8253 // Determine whether to use max or min based on condition code and operands.
8254 bool isMax;
8255 if (CondCode == AArch64CC::GT || CondCode == AArch64CC::GE) {
8256 if (TVal == CmpLHS && FVal == CmpRHS)
8257 isMax = true;
8258 else
8259 return false;
8260 } else if (CondCode == AArch64CC::MI || CondCode == AArch64CC::LS) {
8261 if (TVal == CmpLHS && FVal == CmpRHS)
8262 isMax = false;
8263 else
8264 return false;
8265 } else {
8266 return false;
8267 }
8268
8269 // Get the machine opcode for this VT and operation.
8270 unsigned Opc = getOpc(VT, isMax);
8271 if (!Opc)
8272 return false;
8273
8274 // Constant must be non-NaN.
8275 auto *CFP = dyn_cast<ConstantFPSDNode>(CmpRHS);
8276 if (!CFP || CFP->getValueAPF().isNaN())
8277 return false;
8278
8279 // nsz flag required only when constant is zero: fmaxnm(+0,-0)=+0 differs from
8280 // fcmp+select's -0. For non-zero constants, semantics are identical.
8281 if (CFP->isZero() && !N->getFlags().hasNoSignedZeros())
8282 return false;
8283
8284 // Only fold if variable operand is never sNaN.
8285 // This runs after DAG combines, so later combines cannot remove a defining
8286 // operation used by isKnownNeverSNaN().
8287 if (!CurDAG->isKnownNeverSNaN(CmpLHS))
8288 return false;
8289
8290 CurDAG->SelectNodeTo(N, Opc, VT, CmpLHS, CmpRHS);
8291 return true;
8292}
8293
8294void AArch64DAGToDAGISel::PreprocessISelDAG() {
8295 bool MadeChange = false;
8296 for (SDNode &N : llvm::make_early_inc_range(CurDAG->allnodes())) {
8297 if (N.use_empty())
8298 continue;
8299
8301 switch (N.getOpcode()) {
8302 case ISD::SCALAR_TO_VECTOR: {
8303 EVT ScalarTy = N.getValueType(0).getVectorElementType();
8304 if ((ScalarTy == MVT::i32 || ScalarTy == MVT::i64) &&
8305 ScalarTy == N.getOperand(0).getValueType())
8306 Result = addBitcastHints(*CurDAG, N);
8307
8308 break;
8309 }
8310 case AArch64ISD::VSHL: {
8311 // Undo mul(shl(A,C),B) -> shl(mul(A,B),C) canonicalisation when A is an
8312 // extend that can be folded into the shift.
8313 EVT VT = N.getValueType(0);
8314 SDValue A, B, C = N.getOperand(1);
8315 if (sd_match(N.getOperand(0),
8317 m_SExt(m_Value()))),
8318 m_Value(B))))) {
8319 // If both mul operands are extended, preserve the smull/umull idiom.
8320 if (B.getOpcode() == A.getOpcode())
8321 break;
8322 SDLoc DL(&N);
8323 SDValue SHL = CurDAG->getNode(AArch64ISD::VSHL, DL, VT, A, C);
8324 Result = CurDAG->getNode(ISD::MUL, DL, VT, SHL, B);
8325 }
8326 break;
8327 }
8328 default:
8329 break;
8330 }
8331
8332 if (Result) {
8333 LLVM_DEBUG(dbgs() << "AArch64 DAG preprocessing replacing:\nOld: ");
8334 LLVM_DEBUG(N.dump(CurDAG));
8335 LLVM_DEBUG(dbgs() << "\nNew: ");
8336 LLVM_DEBUG(Result.dump(CurDAG));
8337 LLVM_DEBUG(dbgs() << "\n");
8338
8339 CurDAG->ReplaceAllUsesOfValueWith(SDValue(&N, 0), Result);
8340 MadeChange = true;
8341 }
8342 }
8343
8344 if (MadeChange)
8345 CurDAG->RemoveDeadNodes();
8346
8348}
static std::optional< APInt > GetNEONSplatValue(SDValue N, const AArch64Subtarget *Subtarget)
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
static bool isBitfieldExtractOpFromSExtInReg(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms)
static int getIntOperandFromRegisterString(StringRef RegString)
static SDValue NarrowVector(SDValue V128Reg, SelectionDAG &DAG)
NarrowVector - Given a value in the V128 register class, produce the equivalent value in the V64 regi...
static bool isBitfieldDstMask(uint64_t DstMask, const APInt &BitsToBeInserted, unsigned NumberOfIgnoredHighBits, EVT VT)
Does DstMask form a complementary pair with the mask provided by BitsToBeInserted,...
static SDValue narrowIfNeeded(SelectionDAG *CurDAG, SDValue N)
Instructions that accept extend modifiers like UXTW expect the register being extended to be a GPR32,...
static bool isSeveralBitsPositioningOpFromShl(const uint64_t ShlImm, SDValue Op, SDValue &Src, int &DstLSB, int &Width)
static bool isBitfieldPositioningOp(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, SDValue &Src, int &DstLSB, int &Width)
Does this tree qualify as an attempt to move a bitfield into position, essentially "(and (shl VAL,...
static bool allUsesAreShiftAmounts(SDValue N)
static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc, uint64_t &Imm)
static bool tryBitfieldInsertOpFromOrAndImm(SDNode *N, SelectionDAG *CurDAG)
static std::tuple< SDValue, SDValue > extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG)
static SDValue addBitcastHints(SelectionDAG &DAG, SDNode &N)
addBitcastHints - This method adds bitcast hints to the operands of a node to help instruction select...
static void getUsefulBitsFromOrWithShiftedReg(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromAnd(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB, unsigned NumberOfIgnoredLowBits, bool BiggerPattern)
static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, unsigned NumberOfIgnoredLowBits=0, bool BiggerPattern=false)
static bool isShiftedMask(uint64_t Mask, EVT VT)
bool SelectSMETile(unsigned &BaseReg, unsigned TileNum)
static EVT getMemVTFromNode(LLVMContext &Ctx, SDNode *Root)
Return the EVT of the data associated to a memory operation in Root.
static bool checkCVTFixedPointOperandWithFBits(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static bool isWorthFoldingADDlow(SDValue N)
If there's a use of this ADDlow that's not itself a load/store then we'll need to create a real ADD i...
static AArch64_AM::ShiftExtendType getShiftTypeForNode(SDValue N)
getShiftTypeForNode - Translate a shift node to the corresponding ShiftType value.
static bool isSeveralBitsExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &LSB, unsigned &MSB)
static unsigned SelectOpcodeFromVT(EVT VT, ArrayRef< unsigned > Opcodes)
This function selects an opcode from a list of opcodes, which is expected to be the opcode for { 8-bi...
static EVT getPackedVectorTypeFromPredicateType(LLVMContext &Ctx, EVT PredVT, unsigned NumVec)
When PredVT is a scalable vector predicate in the form MVT::nx<M>xi1, it builds the correspondent sca...
static bool checkCVTFixedPointOperandWithFBitsForVectors(SelectionDAG *CurDAG, SDValue N, SDValue &FixedPos, unsigned RegWidth, bool isReciprocal)
static SDValue getZeroRegister(SelectionDAG &DAG, SDLoc DL, EVT VT)
Returns a copy from WZR or XZR.
static bool isPreferredADD(int64_t ImmOff)
static void getUsefulBitsFromBitfieldMoveOpd(SDValue Op, APInt &UsefulBits, uint64_t Imm, uint64_t MSB, unsigned Depth)
static SDValue getLeftShift(SelectionDAG *CurDAG, SDValue Op, int ShlAmount)
Create a machine node performing a notional SHL of Op by ShlAmount.
static bool isWorthFoldingSHL(SDValue V)
Determine whether it is worth it to fold SHL into the addressing mode.
static bool isBitfieldPositioningOpFromAnd(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, unsigned Depth)
static bool isBitfieldExtractOpFromShr(SDNode *N, unsigned &Opc, SDValue &Opd0, unsigned &Immr, unsigned &Imms, bool BiggerPattern)
static bool tryOrrWithShift(SDNode *N, SDValue OrOpd0, SDValue OrOpd1, SDValue Src, SDValue Dst, SelectionDAG *CurDAG, const bool BiggerPattern)
static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, SDValue Orig, unsigned Depth)
static bool isMemOpOrPrefetch(SDNode *N)
static void getUsefulBitsFromUBFM(SDValue Op, APInt &UsefulBits, unsigned Depth)
static bool tryBitfieldInsertOpFromOr(SDNode *N, const APInt &UsefulBits, SelectionDAG *CurDAG)
static APInt DecodeFMOVImm(uint64_t Imm, unsigned RegWidth)
static void getUsefulBitsFromAndWithImmediate(SDValue Op, APInt &UsefulBits, unsigned Depth)
static std::optional< APInt > DecodeNEONSplat(SDValue N, const AArch64Subtarget *Subtarget)
static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth=0)
static bool isIntImmediateEq(SDValue N, const uint64_t ImmExpected)
static EVT getMultipleVectorType(LLVMContext &Ctx, EVT VecVT, unsigned NumVec)
Builds an integer vector type large enough to hold NumVec instances of VecVT.
static AArch64_AM::ShiftExtendType getExtendTypeForNode(SDValue N, bool IsLoadStore=false)
getExtendTypeForNode - Translate an extend node to the corresponding ExtendType value.
static bool isIntImmediate(const SDNode *N, uint64_t &Imm)
isIntImmediate - This method tests to see if the node is a constant operand.
static bool isWorthFoldingIntoOrrWithShift(SDValue Dst, SelectionDAG *CurDAG, SDValue &ShiftedOperand, uint64_t &EncodedShiftImm)
static bool isValidAsScaledImmediate(int64_t Offset, unsigned Range, unsigned Size)
Check if the immediate offset is valid as a scaled immediate.
static bool isBitfieldPositioningOpFromShl(SelectionDAG *CurDAG, SDValue Op, bool BiggerPattern, const uint64_t NonZeroBits, SDValue &Src, int &DstLSB, int &Width)
return SDValue()
static SDValue WidenVector(SDValue V64Reg, SelectionDAG &DAG)
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 regis...
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
static Register createDTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of D-registers using the registers in Regs.
static Register createQTuple(ArrayRef< Register > Regs, MachineIRBuilder &MIB)
Create a tuple of Q-registers using the registers in Regs.
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
AMDGPU Register Bank Select
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
dxil translate DXIL Translate Metadata
#define DEBUG_TYPE
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
std::pair< Instruction::BinaryOps, Value * > OffsetOp
Find all possible pairs (BinOp, RHS) that BinOp V, RHS can be simplified.
static constexpr Value * getValue(Ty &ValueOrUse)
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
#define R2(n)
Promote Memory to Register
Definition Mem2Reg.cpp:110
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
uint64_t High
OptimizedStructLayoutField Field
#define P(N)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
RISCVCC::CondCode CC
MachineInstr *& Lo
Contains matchers for matching SelectionDAG nodes and values.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
#define LLVM_DEBUG(...)
Definition Debug.h:119
#define PASS_NAME
Value * RHS
Value * LHS
AArch64DAGToDAGISelPass(AArch64TargetMachine &TM)
const AArch64InstrInfo * getInstrInfo() const override
const AArch64TargetLowering * getTargetLowering() const override
bool isStreaming() const
Returns true if the function has a streaming body.
bool isX16X17Safer() const
Returns whether the operating system makes it safer to store sensitive values in x16 and x17 as oppos...
unsigned getSVEVectorSizeInBits() const
Register matchRegisterName(StringRef RegName) const
static const fltSemantics & IEEEsingle()
Definition APFloat.h:304
static const fltSemantics & IEEEdouble()
Definition APFloat.h:305
static const fltSemantics & IEEEhalf()
Definition APFloat.h:302
Class for arbitrary precision integers.
Definition APInt.h:78
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
unsigned popcount() const
Count the number of bits set.
Definition APInt.h:1690
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition APInt.h:254
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition APInt.h:1618
static LLVM_ABI APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition APInt.cpp:648
void flipAllBits()
Toggle every bit to its opposite value.
Definition APInt.h:1472
bool isShiftedMask() const
Return true if this APInt value contains a non-empty sequence of ones with the remainder zero.
Definition APInt.h:506
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition APInt.h:860
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
iterator begin() const
Definition ArrayRef.h:129
const Constant * getConstVal() const
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
const GlobalValue * getGlobal() const
const TargetRegisterClass * getInlineAsmMemoryOperandRegClass(InlineAsm::ConstraintCode C) const override
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
This class is used to represent ISD::LOAD nodes.
unsigned getID() const
getID() - Return the register class ID number.
const MDOperand & getOperand(unsigned I) const
Definition Metadata.h:1443
unsigned getNumOperands() const
Return number of MDNode operands.
Definition Metadata.h:1449
bool equalsStr(StringRef Str) const
Definition Metadata.h:930
Metadata * get() const
Definition Metadata.h:937
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
bool hasScalableStackID(int ObjectIdx) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
A description of a memory reference used in the backend.
const MDNode * getMemCacheHint() const
Return the cache hint metadata for the memory reference.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
iterator_range< user_iterator > users()
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
virtual void PreprocessISelDAG()
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
virtual bool runOnMachineFunction(MachineFunction &mf)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
static constexpr unsigned MaxRecursionDepth
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
void reserve(size_type N)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:728
EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const
Return the EVT corresponding to this LLVM type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
LLVM Value Representation.
Definition Value.h:75
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition Value.h:441
LLVM_ABI Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
Definition Value.cpp:985
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
uint32_t parseGenericRegister(StringRef Name)
static uint64_t decodeLogicalImmediate(uint64_t val, unsigned regSize)
decodeLogicalImmediate - Decode a logical immediate value in the form "N:immr:imms" (where the immr a...
static unsigned getShiftValue(unsigned Imm)
getShiftValue - Extract the shift value.
static bool isLogicalImmediate(uint64_t imm, unsigned regSize)
isLogicalImmediate - Return true if the immediate is valid for a logical immediate instruction of the...
static uint64_t decodeAdvSIMDModImmType12(uint8_t Imm)
constexpr bool isLegalArithImmed(const uint64_t C)
isLegalArithImmed -
static uint64_t decodeAdvSIMDModImmType11(uint8_t Imm)
unsigned getExtendEncoding(AArch64_AM::ShiftExtendType ET)
Mapping from extend bits to required operation: shifter: 000 ==> uxtb 001 ==> uxth 010 ==> uxtw 011 =...
static uint64_t decodeAdvSIMDModImmType10(uint8_t Imm)
static bool isSVELogicalImm(unsigned SizeInBits, uint64_t ImmVal, uint64_t &Encoding)
constexpr unsigned getArithImmedShift(const uint64_t C)
getArithImmedShift - assumes C is a legal immediate for arithmetic instructions and
static bool isSVECpyDupImm(int SizeInBits, int64_t Val, int32_t &Imm, int32_t &Shift)
static AArch64_AM::ShiftExtendType getShiftType(unsigned Imm)
getShiftType - Extract the shift type.
static unsigned getShifterImm(AArch64_AM::ShiftExtendType ST, unsigned Imm)
getShifterImm - Encode the shift type and amount: imm: 6-bit shift amount shifter: 000 ==> lsl 001 ==...
static bool isSignExtendShiftType(AArch64_AM::ShiftExtendType Type)
isSignExtendShiftType - Returns true if Type is sign extending.
void expandMOVImm(uint64_t Imm, unsigned BitSize, SmallVectorImpl< ImmInsnModel > &Insn)
Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more real move-immediate instructions to...
static constexpr unsigned SVEBitsPerBlock
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition ISDOpcodes.h:71
@ CopyFromReg
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition ISDOpcodes.h:232
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
BinaryOpc_match< LHS, RHS, true > m_Mul(const LHS &L, const RHS &R)
Or< Preds... > m_AnyOf(const Preds &...preds)
auto m_SExt(const Opnd &Op)
bool sd_match(SDValue N, Pattern &&P)
UnaryOpc_match< Opnd > m_ZExt(const Opnd &Op)
Value_match m_Value()
Match any valid SDValue.
NUses_match< 1, Value_match > m_OneUse()
Not(const Pred &P) -> Not< Pred >
DiagnosticInfoOptimizationBase::Argument NV
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
friend class Instruction
Iterator for Instructions in a `BasicBlock.
Definition BasicBlock.h:73
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
unsigned CheckFixedPointOperandConstant(APFloat &FVal, unsigned RegWidth, bool isReciprocal)
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition STLExtras.h:649
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition MathExtras.h:268
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition MathExtras.h:274
OutputIt transform(R &&Range, OutputIt d_first, UnaryFunction F)
Wrapper function around std::transform to apply a function to a range and store the result elsewhere.
Definition STLExtras.h:2042
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
LLVM_ABI bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:262
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
LLVM_ATTRIBUTE_VISIBILITY_DEFAULT AnalysisKey InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key
LLVM_ABI raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
FunctionPass * createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOptLevel OptLevel)
createAArch64ISelDag - This pass converts a legalized DAG into a AArch64-specific DAG,...
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
AArch64MemoryHint toAArch64MemoryHint(Int I)
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
Extended Value Type.
Definition ValueTypes.h:35
bool isScalableVectorOf(EVT EltVT) const
Return true if this is a scalable vector with matching element type.
Definition ValueTypes.h:207
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition ValueTypes.h:144
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
ElementCount getVectorElementCount() const
Definition ValueTypes.h:388
EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:509
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition ValueTypes.h:397
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
EVT changeVectorElementType(LLVMContext &Context, EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:98
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition ValueTypes.h:245
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition ValueTypes.h:202
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition ValueTypes.h:240
Matching combinators.