Bug Summary

File:llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
Warning:line 3304, column 20
1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name ARMISelDAGToDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/ARM -I /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM -I include -I /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/build-llvm -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-11-10-160236-22541-1 -x c++ /build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp

→
1//===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the ARM target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "ARM.h"
14#include "ARMBaseInstrInfo.h"
15#include "ARMTargetMachine.h"
16#include "MCTargetDesc/ARMAddressingModes.h"
17#include "Utils/ARMBaseInfo.h"
18#include "llvm/ADT/APSInt.h"
19#include "llvm/ADT/StringSwitch.h"
20#include "llvm/CodeGen/MachineFrameInfo.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineRegisterInfo.h"
24#include "llvm/CodeGen/SelectionDAG.h"
25#include "llvm/CodeGen/SelectionDAGISel.h"
26#include "llvm/CodeGen/TargetLowering.h"
27#include "llvm/IR/CallingConv.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/IntrinsicsARM.h"
33#include "llvm/IR/LLVMContext.h"
34#include "llvm/Support/CommandLine.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Target/TargetOptions.h"
38
39using namespace llvm;
40
41#define DEBUG_TYPE"arm-isel" "arm-isel"
42
43static cl::opt<bool>
44DisableShifterOp("disable-shifter-op", cl::Hidden,
45 cl::desc("Disable isel of shifter-op"),
46 cl::init(false));
47
48//===--------------------------------------------------------------------===//
49/// ARMDAGToDAGISel - ARM specific code to select ARM machine
50/// instructions for SelectionDAG operations.
51///
52namespace {
53
54class ARMDAGToDAGISel : public SelectionDAGISel {
55 /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
56 /// make the right decision when generating code for different targets.
57 const ARMSubtarget *Subtarget;
58
59public:
60 explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
61 : SelectionDAGISel(tm, OptLevel) {}
62
63 bool runOnMachineFunction(MachineFunction &MF) override {
64 // Reset the subtarget each time through.
65 Subtarget = &MF.getSubtarget<ARMSubtarget>();
66 SelectionDAGISel::runOnMachineFunction(MF);
67 return true;
68 }
69
70 StringRef getPassName() const override { return "ARM Instruction Selection"; }
71
72 void PreprocessISelDAG() override;
73
74 /// getI32Imm - Return a target constant of type i32 with the specified
75 /// value.
76 inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
77 return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
78 }
79
80 void Select(SDNode *N) override;
81
82 /// Return true as some complex patterns, like those that call
83 /// canExtractShiftFromMul can modify the DAG inplace.
84 bool ComplexPatternFuncMutatesDAG() const override { return true; }
85
86 bool hasNoVMLxHazardUse(SDNode *N) const;
87 bool isShifterOpProfitable(const SDValue &Shift,
88 ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
89 bool SelectRegShifterOperand(SDValue N, SDValue &A,
90 SDValue &B, SDValue &C,
91 bool CheckProfitability = true);
92 bool SelectImmShifterOperand(SDValue N, SDValue &A,
93 SDValue &B, bool CheckProfitability = true);
94 bool SelectShiftRegShifterOperand(SDValue N, SDValue &A, SDValue &B,
95 SDValue &C) {
96 // Don't apply the profitability check
97 return SelectRegShifterOperand(N, A, B, C, false);
98 }
99 bool SelectShiftImmShifterOperand(SDValue N, SDValue &A, SDValue &B) {
100 // Don't apply the profitability check
101 return SelectImmShifterOperand(N, A, B, false);
102 }
103 bool SelectShiftImmShifterOperandOneUse(SDValue N, SDValue &A, SDValue &B) {
104 if (!N.hasOneUse())
105 return false;
106 return SelectImmShifterOperand(N, A, B, false);
107 }
108
109 bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
110
111 bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
112 bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
113
114 bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
115 const ConstantSDNode *CN = cast<ConstantSDNode>(N);
116 Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
117 Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
118 return true;
119 }
120
121 bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
122 SDValue &Offset, SDValue &Opc);
123 bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
124 SDValue &Offset, SDValue &Opc);
125 bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
126 SDValue &Offset, SDValue &Opc);
127 bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
128 bool SelectAddrMode3(SDValue N, SDValue &Base,
129 SDValue &Offset, SDValue &Opc);
130 bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
131 SDValue &Offset, SDValue &Opc);
132 bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
133 bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
134 bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
135 bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
136 bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
137
138 bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
139
140 // Thumb Addressing Modes:
141 bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
142 bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
143 bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
144 SDValue &OffImm);
145 bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
146 SDValue &OffImm);
147 bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
148 SDValue &OffImm);
149 bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
150 SDValue &OffImm);
151 bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
152 template <unsigned Shift>
153 bool SelectTAddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
154
155 // Thumb 2 Addressing Modes:
156 bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
157 template <unsigned Shift>
158 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base, SDValue &OffImm);
159 bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
160 SDValue &OffImm);
161 bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
162 SDValue &OffImm);
163 template <unsigned Shift>
164 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm);
165 bool SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N, SDValue &OffImm,
166 unsigned Shift);
167 template <unsigned Shift>
168 bool SelectT2AddrModeImm7(SDValue N, SDValue &Base, SDValue &OffImm);
169 bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
170 SDValue &OffReg, SDValue &ShImm);
171 bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
172
173 template<int Min, int Max>
174 bool SelectImmediateInRange(SDValue N, SDValue &OffImm);
175
176 inline bool is_so_imm(unsigned Imm) const {
177 return ARM_AM::getSOImmVal(Imm) != -1;
178 }
179
180 inline bool is_so_imm_not(unsigned Imm) const {
181 return ARM_AM::getSOImmVal(~Imm) != -1;
182 }
183
184 inline bool is_t2_so_imm(unsigned Imm) const {
185 return ARM_AM::getT2SOImmVal(Imm) != -1;
186 }
187
188 inline bool is_t2_so_imm_not(unsigned Imm) const {
189 return ARM_AM::getT2SOImmVal(~Imm) != -1;
190 }
191
192 // Include the pieces autogenerated from the target description.
193#include "ARMGenDAGISel.inc"
194
195private:
196 void transferMemOperands(SDNode *Src, SDNode *Dst);
197
198 /// Indexed (pre/post inc/dec) load matching code for ARM.
199 bool tryARMIndexedLoad(SDNode *N);
200 bool tryT1IndexedLoad(SDNode *N);
201 bool tryT2IndexedLoad(SDNode *N);
202 bool tryMVEIndexedLoad(SDNode *N);
203 bool tryFMULFixed(SDNode *N, SDLoc dl);
204 bool tryFP_TO_INT(SDNode *N, SDLoc dl);
205 bool transformFixedFloatingPointConversion(SDNode *N, SDNode *FMul,
206 bool IsUnsigned,
207 bool FixedToFloat);
208
209 /// SelectVLD - Select NEON load intrinsics. NumVecs should be
210 /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
211 /// loads of D registers and even subregs and odd subregs of Q registers.
212 /// For NumVecs <= 2, QOpcodes1 is not used.
213 void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
214 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
215 const uint16_t *QOpcodes1);
216
217 /// SelectVST - Select NEON store intrinsics. NumVecs should
218 /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
219 /// stores of D registers and even subregs and odd subregs of Q registers.
220 /// For NumVecs <= 2, QOpcodes1 is not used.
221 void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
222 const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
223 const uint16_t *QOpcodes1);
224
225 /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
226 /// be 2, 3 or 4. The opcode arrays specify the instructions used for
227 /// load/store of D registers and Q registers.
228 void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
229 unsigned NumVecs, const uint16_t *DOpcodes,
230 const uint16_t *QOpcodes);
231
232 /// Helper functions for setting up clusters of MVE predication operands.
233 template <typename SDValueVector>
234 void AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
235 SDValue PredicateMask);
236 template <typename SDValueVector>
237 void AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
238 SDValue PredicateMask, SDValue Inactive);
239
240 template <typename SDValueVector>
241 void AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc);
242 template <typename SDValueVector>
243 void AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc, EVT InactiveTy);
244
245 /// SelectMVE_WB - Select MVE writeback load/store intrinsics.
246 void SelectMVE_WB(SDNode *N, const uint16_t *Opcodes, bool Predicated);
247
248 /// SelectMVE_LongShift - Select MVE 64-bit scalar shift intrinsics.
249 void SelectMVE_LongShift(SDNode *N, uint16_t Opcode, bool Immediate,
250 bool HasSaturationOperand);
251
252 /// SelectMVE_VADCSBC - Select MVE vector add/sub-with-carry intrinsics.
253 void SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
254 uint16_t OpcodeWithNoCarry, bool Add, bool Predicated);
255
256 /// SelectMVE_VSHLC - Select MVE intrinsics for a shift that carries between
257 /// vector lanes.
258 void SelectMVE_VSHLC(SDNode *N, bool Predicated);
259
260 /// Select long MVE vector reductions with two vector operands
261 /// Stride is the number of vector element widths the instruction can operate
262 /// on:
263 /// 2 for long non-rounding variants, vml{a,s}ldav[a][x]: [i16, i32]
264 /// 1 for long rounding variants: vrml{a,s}ldavh[a][x]: [i32]
265 /// Stride is used when addressing the OpcodesS array which contains multiple
266 /// opcodes for each element width.
267 /// TySize is the index into the list of element types listed above
268 void SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated,
269 const uint16_t *OpcodesS, const uint16_t *OpcodesU,
270 size_t Stride, size_t TySize);
271
272 /// Select a 64-bit MVE vector reduction with two vector operands
273 /// arm_mve_vmlldava_[predicated]
274 void SelectMVE_VMLLDAV(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
275 const uint16_t *OpcodesU);
276 /// Select a 72-bit MVE vector rounding reduction with two vector operands
277 /// int_arm_mve_vrmlldavha[_predicated]
278 void SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated, const uint16_t *OpcodesS,
279 const uint16_t *OpcodesU);
280
281 /// SelectMVE_VLD - Select MVE interleaving load intrinsics. NumVecs
282 /// should be 2 or 4. The opcode array specifies the instructions
283 /// used for 8, 16 and 32-bit lane sizes respectively, and each
284 /// pointer points to a set of NumVecs sub-opcodes used for the
285 /// different stages (e.g. VLD20 versus VLD21) of each load family.
286 void SelectMVE_VLD(SDNode *N, unsigned NumVecs,
287 const uint16_t *const *Opcodes, bool HasWriteback);
288
289 /// SelectMVE_VxDUP - Select MVE incrementing-dup instructions. Opcodes is an
290 /// array of 3 elements for the 8, 16 and 32-bit lane sizes.
291 void SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
292 bool Wrapping, bool Predicated);
293
294 /// Select SelectCDE_CXxD - Select CDE dual-GPR instruction (one of CX1D,
295 /// CX1DA, CX2D, CX2DA, CX3, CX3DA).
296 /// \arg \c NumExtraOps number of extra operands besides the coprocossor,
297 /// the accumulator and the immediate operand, i.e. 0
298 /// for CX1*, 1 for CX2*, 2 for CX3*
299 /// \arg \c HasAccum whether the instruction has an accumulator operand
300 void SelectCDE_CXxD(SDNode *N, uint16_t Opcode, size_t NumExtraOps,
301 bool HasAccum);
302
303 /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
304 /// should be 1, 2, 3 or 4. The opcode array specifies the instructions used
305 /// for loading D registers.
306 void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
307 unsigned NumVecs, const uint16_t *DOpcodes,
308 const uint16_t *QOpcodes0 = nullptr,
309 const uint16_t *QOpcodes1 = nullptr);
310
311 /// Try to select SBFX/UBFX instructions for ARM.
312 bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
313
314 bool tryInsertVectorElt(SDNode *N);
315
316 // Select special operations if node forms integer ABS pattern
317 bool tryABSOp(SDNode *N);
318
319 bool tryReadRegister(SDNode *N);
320 bool tryWriteRegister(SDNode *N);
321
322 bool tryInlineAsm(SDNode *N);
323
324 void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
325
326 void SelectCMP_SWAP(SDNode *N);
327
328 /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
329 /// inline asm expressions.
330 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
331 std::vector<SDValue> &OutOps) override;
332
333 // Form pairs of consecutive R, S, D, or Q registers.
334 SDNode *createGPRPairNode(EVT VT, SDValue V0, SDValue V1);
335 SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
336 SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
337 SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
338
339 // Form sequences of 4 consecutive S, D, or Q registers.
340 SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
341 SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
342 SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
343
344 // Get the alignment operand for a NEON VLD or VST instruction.
345 SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
346 bool is64BitVector);
347
348 /// Checks if N is a multiplication by a constant where we can extract out a
349 /// power of two from the constant so that it can be used in a shift, but only
350 /// if it simplifies the materialization of the constant. Returns true if it
351 /// is, and assigns to PowerOfTwo the power of two that should be extracted
352 /// out and to NewMulConst the new constant to be multiplied by.
353 bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
354 unsigned &PowerOfTwo, SDValue &NewMulConst) const;
355
356 /// Replace N with M in CurDAG, in a way that also ensures that M gets
357 /// selected when N would have been selected.
358 void replaceDAGValue(const SDValue &N, SDValue M);
359};
360}
361
362/// isInt32Immediate - This method tests to see if the node is a 32-bit constant
363/// operand. If so Imm will receive the 32-bit value.
364static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
365 if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
366 Imm = cast<ConstantSDNode>(N)->getZExtValue();
367 return true;
368 }
369 return false;
370}
371
372// isInt32Immediate - This method tests to see if a constant operand.
373// If so Imm will receive the 32 bit value.
374static bool isInt32Immediate(SDValue N, unsigned &Imm) {
375 return isInt32Immediate(N.getNode(), Imm);
376}
377
378// isOpcWithIntImmediate - This method tests to see if the node is a specific
379// opcode and that it has a immediate integer right operand.
380// If so Imm will receive the 32 bit value.
381static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
382 return N->getOpcode() == Opc &&
383 isInt32Immediate(N->getOperand(1).getNode(), Imm);
384}
385
386/// Check whether a particular node is a constant value representable as
387/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
388///
389/// \param ScaledConstant [out] - On success, the pre-scaled constant value.
390static bool isScaledConstantInRange(SDValue Node, int Scale,
391 int RangeMin, int RangeMax,
392 int &ScaledConstant) {
393 assert(Scale > 0 && "Invalid scale!")(static_cast <bool> (Scale > 0 && "Invalid scale!"
) ? void (0) : __assert_fail ("Scale > 0 && \"Invalid scale!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 393, __extension__ __PRETTY_FUNCTION__));
394
395 // Check that this is a constant.
396 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
397 if (!C)
398 return false;
399
400 ScaledConstant = (int) C->getZExtValue();
401 if ((ScaledConstant % Scale) != 0)
402 return false;
403
404 ScaledConstant /= Scale;
405 return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
406}
407
408void ARMDAGToDAGISel::PreprocessISelDAG() {
409 if (!Subtarget->hasV6T2Ops())
410 return;
411
412 bool isThumb2 = Subtarget->isThumb();
413 // We use make_early_inc_range to avoid invalidation issues.
414 for (SDNode &N : llvm::make_early_inc_range(CurDAG->allnodes())) {
415 if (N.getOpcode() != ISD::ADD)
416 continue;
417
418 // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
419 // leading zeros, followed by consecutive set bits, followed by 1 or 2
420 // trailing zeros, e.g. 1020.
421 // Transform the expression to
422 // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
423 // of trailing zeros of c2. The left shift would be folded as an shifter
424 // operand of 'add' and the 'and' and 'srl' would become a bits extraction
425 // node (UBFX).
426
427 SDValue N0 = N.getOperand(0);
428 SDValue N1 = N.getOperand(1);
429 unsigned And_imm = 0;
430 if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
431 if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
432 std::swap(N0, N1);
433 }
434 if (!And_imm)
435 continue;
436
437 // Check if the AND mask is an immediate of the form: 000.....1111111100
438 unsigned TZ = countTrailingZeros(And_imm);
439 if (TZ != 1 && TZ != 2)
440 // Be conservative here. Shifter operands aren't always free. e.g. On
441 // Swift, left shifter operand of 1 / 2 for free but others are not.
442 // e.g.
443 // ubfx r3, r1, #16, #8
444 // ldr.w r3, [r0, r3, lsl #2]
445 // vs.
446 // mov.w r9, #1020
447 // and.w r2, r9, r1, lsr #14
448 // ldr r2, [r0, r2]
449 continue;
450 And_imm >>= TZ;
451 if (And_imm & (And_imm + 1))
452 continue;
453
454 // Look for (and (srl X, c1), c2).
455 SDValue Srl = N1.getOperand(0);
456 unsigned Srl_imm = 0;
457 if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
458 (Srl_imm <= 2))
459 continue;
460
461 // Make sure first operand is not a shifter operand which would prevent
462 // folding of the left shift.
463 SDValue CPTmp0;
464 SDValue CPTmp1;
465 SDValue CPTmp2;
466 if (isThumb2) {
467 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
468 continue;
469 } else {
470 if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
471 SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
472 continue;
473 }
474
475 // Now make the transformation.
476 Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
477 Srl.getOperand(0),
478 CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
479 MVT::i32));
480 N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
481 Srl,
482 CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
483 N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
484 N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
485 CurDAG->UpdateNodeOperands(&N, N0, N1);
486 }
487}
488
489/// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
490/// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
491/// least on current ARM implementations) which should be avoidded.
492bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
493 if (OptLevel == CodeGenOpt::None)
494 return true;
495
496 if (!Subtarget->hasVMLxHazards())
497 return true;
498
499 if (!N->hasOneUse())
500 return false;
501
502 SDNode *Use = *N->use_begin();
503 if (Use->getOpcode() == ISD::CopyToReg)
504 return true;
505 if (Use->isMachineOpcode()) {
506 const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
507 CurDAG->getSubtarget().getInstrInfo());
508
509 const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
510 if (MCID.mayStore())
511 return true;
512 unsigned Opcode = MCID.getOpcode();
513 if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
514 return true;
515 // vmlx feeding into another vmlx. We actually want to unfold
516 // the use later in the MLxExpansion pass. e.g.
517 // vmla
518 // vmla (stall 8 cycles)
519 //
520 // vmul (5 cycles)
521 // vadd (5 cycles)
522 // vmla
523 // This adds up to about 18 - 19 cycles.
524 //
525 // vmla
526 // vmul (stall 4 cycles)
527 // vadd adds up to about 14 cycles.
528 return TII->isFpMLxInstruction(Opcode);
529 }
530
531 return false;
532}
533
534bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
535 ARM_AM::ShiftOpc ShOpcVal,
536 unsigned ShAmt) {
537 if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
538 return true;
539 if (Shift.hasOneUse())
540 return true;
541 // R << 2 is free.
542 return ShOpcVal == ARM_AM::lsl &&
543 (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
544}
545
546bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
547 unsigned MaxShift,
548 unsigned &PowerOfTwo,
549 SDValue &NewMulConst) const {
550 assert(N.getOpcode() == ISD::MUL)(static_cast <bool> (N.getOpcode() == ISD::MUL) ? void (
0) : __assert_fail ("N.getOpcode() == ISD::MUL", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 550, __extension__ __PRETTY_FUNCTION__));
551 assert(MaxShift > 0)(static_cast <bool> (MaxShift > 0) ? void (0) : __assert_fail
("MaxShift > 0", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 551, __extension__ __PRETTY_FUNCTION__));
552
553 // If the multiply is used in more than one place then changing the constant
554 // will make other uses incorrect, so don't.
555 if (!N.hasOneUse()) return false;
556 // Check if the multiply is by a constant
557 ConstantSDNode *MulConst = dyn_cast<ConstantSDNode>(N.getOperand(1));
558 if (!MulConst) return false;
559 // If the constant is used in more than one place then modifying it will mean
560 // we need to materialize two constants instead of one, which is a bad idea.
561 if (!MulConst->hasOneUse()) return false;
562 unsigned MulConstVal = MulConst->getZExtValue();
563 if (MulConstVal == 0) return false;
564
565 // Find the largest power of 2 that MulConstVal is a multiple of
566 PowerOfTwo = MaxShift;
567 while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
568 --PowerOfTwo;
569 if (PowerOfTwo == 0) return false;
570 }
571
572 // Only optimise if the new cost is better
573 unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
574 NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
575 unsigned OldCost = ConstantMaterializationCost(MulConstVal, Subtarget);
576 unsigned NewCost = ConstantMaterializationCost(NewMulConstVal, Subtarget);
577 return NewCost < OldCost;
578}
579
580void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
581 CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
582 ReplaceUses(N, M);
583}
584
585bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
586 SDValue &BaseReg,
587 SDValue &Opc,
588 bool CheckProfitability) {
589 if (DisableShifterOp)
590 return false;
591
592 // If N is a multiply-by-constant and it's profitable to extract a shift and
593 // use it in a shifted operand do so.
594 if (N.getOpcode() == ISD::MUL) {
595 unsigned PowerOfTwo = 0;
596 SDValue NewMulConst;
597 if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
598 HandleSDNode Handle(N);
599 SDLoc Loc(N);
600 replaceDAGValue(N.getOperand(1), NewMulConst);
601 BaseReg = Handle.getValue();
602 Opc = CurDAG->getTargetConstant(
603 ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
604 return true;
605 }
606 }
607
608 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
609
610 // Don't match base register only case. That is matched to a separate
611 // lower complexity pattern with explicit register operand.
612 if (ShOpcVal == ARM_AM::no_shift) return false;
613
614 BaseReg = N.getOperand(0);
615 unsigned ShImmVal = 0;
616 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
617 if (!RHS) return false;
618 ShImmVal = RHS->getZExtValue() & 31;
619 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
620 SDLoc(N), MVT::i32);
621 return true;
622}
623
624bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
625 SDValue &BaseReg,
626 SDValue &ShReg,
627 SDValue &Opc,
628 bool CheckProfitability) {
629 if (DisableShifterOp)
630 return false;
631
632 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
633
634 // Don't match base register only case. That is matched to a separate
635 // lower complexity pattern with explicit register operand.
636 if (ShOpcVal == ARM_AM::no_shift) return false;
637
638 BaseReg = N.getOperand(0);
639 unsigned ShImmVal = 0;
640 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
641 if (RHS) return false;
642
643 ShReg = N.getOperand(1);
644 if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
645 return false;
646 Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
647 SDLoc(N), MVT::i32);
648 return true;
649}
650
651// Determine whether an ISD::OR's operands are suitable to turn the operation
652// into an addition, which often has more compact encodings.
653bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
654 assert(Parent->getOpcode() == ISD::OR && "unexpected parent")(static_cast <bool> (Parent->getOpcode() == ISD::OR &&
"unexpected parent") ? void (0) : __assert_fail ("Parent->getOpcode() == ISD::OR && \"unexpected parent\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 654, __extension__ __PRETTY_FUNCTION__));
655 Out = N;
656 return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
657}
658
659
660bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
661 SDValue &Base,
662 SDValue &OffImm) {
663 // Match simple R + imm12 operands.
664
665 // Base only.
666 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
667 !CurDAG->isBaseWithConstantOffset(N)) {
668 if (N.getOpcode() == ISD::FrameIndex) {
669 // Match frame index.
670 int FI = cast<FrameIndexSDNode>(N)->getIndex();
671 Base = CurDAG->getTargetFrameIndex(
672 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
673 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
674 return true;
675 }
676
677 if (N.getOpcode() == ARMISD::Wrapper &&
678 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
679 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
680 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
681 Base = N.getOperand(0);
682 } else
683 Base = N;
684 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
685 return true;
686 }
687
688 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
689 int RHSC = (int)RHS->getSExtValue();
690 if (N.getOpcode() == ISD::SUB)
691 RHSC = -RHSC;
692
693 if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
694 Base = N.getOperand(0);
695 if (Base.getOpcode() == ISD::FrameIndex) {
696 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
697 Base = CurDAG->getTargetFrameIndex(
698 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
699 }
700 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
701 return true;
702 }
703 }
704
705 // Base only.
706 Base = N;
707 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
708 return true;
709}
710
711
712
713bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
714 SDValue &Opc) {
715 if (N.getOpcode() == ISD::MUL &&
716 ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
717 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
718 // X * [3,5,9] -> X + X * [2,4,8] etc.
719 int RHSC = (int)RHS->getZExtValue();
720 if (RHSC & 1) {
721 RHSC = RHSC & ~1;
722 ARM_AM::AddrOpc AddSub = ARM_AM::add;
723 if (RHSC < 0) {
724 AddSub = ARM_AM::sub;
725 RHSC = - RHSC;
726 }
727 if (isPowerOf2_32(RHSC)) {
728 unsigned ShAmt = Log2_32(RHSC);
729 Base = Offset = N.getOperand(0);
730 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
731 ARM_AM::lsl),
732 SDLoc(N), MVT::i32);
733 return true;
734 }
735 }
736 }
737 }
738
739 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
740 // ISD::OR that is equivalent to an ISD::ADD.
741 !CurDAG->isBaseWithConstantOffset(N))
742 return false;
743
744 // Leave simple R +/- imm12 operands for LDRi12
745 if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
746 int RHSC;
747 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
748 -0x1000+1, 0x1000, RHSC)) // 12 bits.
749 return false;
750 }
751
752 // Otherwise this is R +/- [possibly shifted] R.
753 ARM_AM::AddrOpc AddSub = N.getOpcode() == ISD::SUB ? ARM_AM::sub:ARM_AM::add;
754 ARM_AM::ShiftOpc ShOpcVal =
755 ARM_AM::getShiftOpcForNode(N.getOperand(1).getOpcode());
756 unsigned ShAmt = 0;
757
758 Base = N.getOperand(0);
759 Offset = N.getOperand(1);
760
761 if (ShOpcVal != ARM_AM::no_shift) {
762 // Check to see if the RHS of the shift is a constant, if not, we can't fold
763 // it.
764 if (ConstantSDNode *Sh =
765 dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
766 ShAmt = Sh->getZExtValue();
767 if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
768 Offset = N.getOperand(1).getOperand(0);
769 else {
770 ShAmt = 0;
771 ShOpcVal = ARM_AM::no_shift;
772 }
773 } else {
774 ShOpcVal = ARM_AM::no_shift;
775 }
776 }
777
778 // Try matching (R shl C) + (R).
779 if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
780 !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
781 N.getOperand(0).hasOneUse())) {
782 ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
783 if (ShOpcVal != ARM_AM::no_shift) {
784 // Check to see if the RHS of the shift is a constant, if not, we can't
785 // fold it.
786 if (ConstantSDNode *Sh =
787 dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
788 ShAmt = Sh->getZExtValue();
789 if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
790 Offset = N.getOperand(0).getOperand(0);
791 Base = N.getOperand(1);
792 } else {
793 ShAmt = 0;
794 ShOpcVal = ARM_AM::no_shift;
795 }
796 } else {
797 ShOpcVal = ARM_AM::no_shift;
798 }
799 }
800 }
801
802 // If Offset is a multiply-by-constant and it's profitable to extract a shift
803 // and use it in a shifted operand do so.
804 if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
805 unsigned PowerOfTwo = 0;
806 SDValue NewMulConst;
807 if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
808 HandleSDNode Handle(Offset);
809 replaceDAGValue(Offset.getOperand(1), NewMulConst);
810 Offset = Handle.getValue();
811 ShAmt = PowerOfTwo;
812 ShOpcVal = ARM_AM::lsl;
813 }
814 }
815
816 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
817 SDLoc(N), MVT::i32);
818 return true;
819}
820
821bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
822 SDValue &Offset, SDValue &Opc) {
823 unsigned Opcode = Op->getOpcode();
824 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
825 ? cast<LoadSDNode>(Op)->getAddressingMode()
826 : cast<StoreSDNode>(Op)->getAddressingMode();
827 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
828 ? ARM_AM::add : ARM_AM::sub;
829 int Val;
830 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
831 return false;
832
833 Offset = N;
834 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOpcode());
835 unsigned ShAmt = 0;
836 if (ShOpcVal != ARM_AM::no_shift) {
837 // Check to see if the RHS of the shift is a constant, if not, we can't fold
838 // it.
839 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
840 ShAmt = Sh->getZExtValue();
841 if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
842 Offset = N.getOperand(0);
843 else {
844 ShAmt = 0;
845 ShOpcVal = ARM_AM::no_shift;
846 }
847 } else {
848 ShOpcVal = ARM_AM::no_shift;
849 }
850 }
851
852 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
853 SDLoc(N), MVT::i32);
854 return true;
855}
856
857bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
858 SDValue &Offset, SDValue &Opc) {
859 unsigned Opcode = Op->getOpcode();
860 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
861 ? cast<LoadSDNode>(Op)->getAddressingMode()
862 : cast<StoreSDNode>(Op)->getAddressingMode();
863 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
864 ? ARM_AM::add : ARM_AM::sub;
865 int Val;
866 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
867 if (AddSub == ARM_AM::sub) Val *= -1;
868 Offset = CurDAG->getRegister(0, MVT::i32);
869 Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
870 return true;
871 }
872
873 return false;
874}
875
876
877bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
878 SDValue &Offset, SDValue &Opc) {
879 unsigned Opcode = Op->getOpcode();
880 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
881 ? cast<LoadSDNode>(Op)->getAddressingMode()
882 : cast<StoreSDNode>(Op)->getAddressingMode();
883 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
884 ? ARM_AM::add : ARM_AM::sub;
885 int Val;
886 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
887 Offset = CurDAG->getRegister(0, MVT::i32);
888 Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
889 ARM_AM::no_shift),
890 SDLoc(Op), MVT::i32);
891 return true;
892 }
893
894 return false;
895}
896
897bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
898 Base = N;
899 return true;
900}
901
902bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
903 SDValue &Base, SDValue &Offset,
904 SDValue &Opc) {
905 if (N.getOpcode() == ISD::SUB) {
906 // X - C is canonicalize to X + -C, no need to handle it here.
907 Base = N.getOperand(0);
908 Offset = N.getOperand(1);
909 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
910 MVT::i32);
911 return true;
912 }
913
914 if (!CurDAG->isBaseWithConstantOffset(N)) {
915 Base = N;
916 if (N.getOpcode() == ISD::FrameIndex) {
917 int FI = cast<FrameIndexSDNode>(N)->getIndex();
918 Base = CurDAG->getTargetFrameIndex(
919 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
920 }
921 Offset = CurDAG->getRegister(0, MVT::i32);
922 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
923 MVT::i32);
924 return true;
925 }
926
927 // If the RHS is +/- imm8, fold into addr mode.
928 int RHSC;
929 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
930 -256 + 1, 256, RHSC)) { // 8 bits.
931 Base = N.getOperand(0);
932 if (Base.getOpcode() == ISD::FrameIndex) {
933 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
934 Base = CurDAG->getTargetFrameIndex(
935 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
936 }
937 Offset = CurDAG->getRegister(0, MVT::i32);
938
939 ARM_AM::AddrOpc AddSub = ARM_AM::add;
940 if (RHSC < 0) {
941 AddSub = ARM_AM::sub;
942 RHSC = -RHSC;
943 }
944 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
945 MVT::i32);
946 return true;
947 }
948
949 Base = N.getOperand(0);
950 Offset = N.getOperand(1);
951 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
952 MVT::i32);
953 return true;
954}
955
956bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
957 SDValue &Offset, SDValue &Opc) {
958 unsigned Opcode = Op->getOpcode();
959 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
960 ? cast<LoadSDNode>(Op)->getAddressingMode()
961 : cast<StoreSDNode>(Op)->getAddressingMode();
962 ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
963 ? ARM_AM::add : ARM_AM::sub;
964 int Val;
965 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
966 Offset = CurDAG->getRegister(0, MVT::i32);
967 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
968 MVT::i32);
969 return true;
970 }
971
972 Offset = N;
973 Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
974 MVT::i32);
975 return true;
976}
977
978bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
979 bool FP16) {
980 if (!CurDAG->isBaseWithConstantOffset(N)) {
981 Base = N;
982 if (N.getOpcode() == ISD::FrameIndex) {
983 int FI = cast<FrameIndexSDNode>(N)->getIndex();
984 Base = CurDAG->getTargetFrameIndex(
985 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
986 } else if (N.getOpcode() == ARMISD::Wrapper &&
987 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
988 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
989 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
990 Base = N.getOperand(0);
991 }
992 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
993 SDLoc(N), MVT::i32);
994 return true;
995 }
996
997 // If the RHS is +/- imm8, fold into addr mode.
998 int RHSC;
999 const int Scale = FP16 ? 2 : 4;
1000
1001 if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) {
1002 Base = N.getOperand(0);
1003 if (Base.getOpcode() == ISD::FrameIndex) {
1004 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1005 Base = CurDAG->getTargetFrameIndex(
1006 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1007 }
1008
1009 ARM_AM::AddrOpc AddSub = ARM_AM::add;
1010 if (RHSC < 0) {
1011 AddSub = ARM_AM::sub;
1012 RHSC = -RHSC;
1013 }
1014
1015 if (FP16)
1016 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
1017 SDLoc(N), MVT::i32);
1018 else
1019 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
1020 SDLoc(N), MVT::i32);
1021
1022 return true;
1023 }
1024
1025 Base = N;
1026
1027 if (FP16)
1028 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
1029 SDLoc(N), MVT::i32);
1030 else
1031 Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
1032 SDLoc(N), MVT::i32);
1033
1034 return true;
1035}
1036
1037bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
1038 SDValue &Base, SDValue &Offset) {
1039 return IsAddressingMode5(N, Base, Offset, /*FP16=*/ false);
1040}
1041
1042bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
1043 SDValue &Base, SDValue &Offset) {
1044 return IsAddressingMode5(N, Base, Offset, /*FP16=*/ true);
1045}
1046
1047bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
1048 SDValue &Align) {
1049 Addr = N;
1050
1051 unsigned Alignment = 0;
1052
1053 MemSDNode *MemN = cast<MemSDNode>(Parent);
1054
1055 if (isa<LSBaseSDNode>(MemN) ||
1056 ((MemN->getOpcode() == ARMISD::VST1_UPD ||
1057 MemN->getOpcode() == ARMISD::VLD1_UPD) &&
1058 MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
1059 // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
1060 // The maximum alignment is equal to the memory size being referenced.
1061 unsigned MMOAlign = MemN->getAlignment();
1062 unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
1063 if (MMOAlign >= MemSize && MemSize > 1)
1064 Alignment = MemSize;
1065 } else {
1066 // All other uses of addrmode6 are for intrinsics. For now just record
1067 // the raw alignment value; it will be refined later based on the legal
1068 // alignment operands for the intrinsic.
1069 Alignment = MemN->getAlignment();
1070 }
1071
1072 Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
1073 return true;
1074}
1075
1076bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1077 SDValue &Offset) {
1078 LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1079 ISD::MemIndexedMode AM = LdSt->getAddressingMode();
1080 if (AM != ISD::POST_INC)
1081 return false;
1082 Offset = N;
1083 if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1084 if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1085 Offset = CurDAG->getRegister(0, MVT::i32);
1086 }
1087 return true;
1088}
1089
1090bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1091 SDValue &Offset, SDValue &Label) {
1092 if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1093 Offset = N.getOperand(0);
1094 SDValue N1 = N.getOperand(1);
1095 Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1096 SDLoc(N), MVT::i32);
1097 return true;
1098 }
1099
1100 return false;
1101}
1102
1103
1104//===----------------------------------------------------------------------===//
1105// Thumb Addressing Modes
1106//===----------------------------------------------------------------------===//
1107
1108static bool shouldUseZeroOffsetLdSt(SDValue N) {
1109 // Negative numbers are difficult to materialise in thumb1. If we are
1110 // selecting the add of a negative, instead try to select ri with a zero
1111 // offset, so create the add node directly which will become a sub.
1112 if (N.getOpcode() != ISD::ADD)
1113 return false;
1114
1115 // Look for an imm which is not legal for ld/st, but is legal for sub.
1116 if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1)))
1117 return C->getSExtValue() < 0 && C->getSExtValue() >= -255;
1118
1119 return false;
1120}
1121
1122bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1123 SDValue &Offset) {
1124 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1125 ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N);
1126 if (!NC || !NC->isZero())
1127 return false;
1128
1129 Base = Offset = N;
1130 return true;
1131 }
1132
1133 Base = N.getOperand(0);
1134 Offset = N.getOperand(1);
1135 return true;
1136}
1137
1138bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1139 SDValue &Offset) {
1140 if (shouldUseZeroOffsetLdSt(N))
1141 return false; // Select ri instead
1142 return SelectThumbAddrModeRRSext(N, Base, Offset);
1143}
1144
1145bool
1146ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1147 SDValue &Base, SDValue &OffImm) {
1148 if (shouldUseZeroOffsetLdSt(N)) {
1149 Base = N;
1150 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1151 return true;
1152 }
1153
1154 if (!CurDAG->isBaseWithConstantOffset(N)) {
1155 if (N.getOpcode() == ISD::ADD) {
1156 return false; // We want to select register offset instead
1157 } else if (N.getOpcode() == ARMISD::Wrapper &&
1158 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
1159 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
1160 N.getOperand(0).getOpcode() != ISD::TargetConstantPool &&
1161 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
1162 Base = N.getOperand(0);
1163 } else {
1164 Base = N;
1165 }
1166
1167 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1168 return true;
1169 }
1170
1171 // If the RHS is + imm5 * scale, fold into addr mode.
1172 int RHSC;
1173 if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1174 Base = N.getOperand(0);
1175 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1176 return true;
1177 }
1178
1179 // Offset is too large, so use register offset instead.
1180 return false;
1181}
1182
1183bool
1184ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1185 SDValue &OffImm) {
1186 return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1187}
1188
1189bool
1190ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1191 SDValue &OffImm) {
1192 return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1193}
1194
1195bool
1196ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1197 SDValue &OffImm) {
1198 return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1199}
1200
1201bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1202 SDValue &Base, SDValue &OffImm) {
1203 if (N.getOpcode() == ISD::FrameIndex) {
1204 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1205 // Only multiples of 4 are allowed for the offset, so the frame object
1206 // alignment must be at least 4.
1207 MachineFrameInfo &MFI = MF->getFrameInfo();
1208 if (MFI.getObjectAlign(FI) < Align(4))
1209 MFI.setObjectAlignment(FI, Align(4));
1210 Base = CurDAG->getTargetFrameIndex(
1211 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1212 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1213 return true;
1214 }
1215
1216 if (!CurDAG->isBaseWithConstantOffset(N))
1217 return false;
1218
1219 if (N.getOperand(0).getOpcode() == ISD::FrameIndex) {
1220 // If the RHS is + imm8 * scale, fold into addr mode.
1221 int RHSC;
1222 if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1223 Base = N.getOperand(0);
1224 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1225 // Make sure the offset is inside the object, or we might fail to
1226 // allocate an emergency spill slot. (An out-of-range access is UB, but
1227 // it could show up anyway.)
1228 MachineFrameInfo &MFI = MF->getFrameInfo();
1229 if (RHSC * 4 < MFI.getObjectSize(FI)) {
1230 // For LHS+RHS to result in an offset that's a multiple of 4 the object
1231 // indexed by the LHS must be 4-byte aligned.
1232 if (!MFI.isFixedObjectIndex(FI) && MFI.getObjectAlign(FI) < Align(4))
1233 MFI.setObjectAlignment(FI, Align(4));
1234 if (MFI.getObjectAlign(FI) >= Align(4)) {
1235 Base = CurDAG->getTargetFrameIndex(
1236 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1237 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1238 return true;
1239 }
1240 }
1241 }
1242 }
1243
1244 return false;
1245}
1246
1247template <unsigned Shift>
1248bool ARMDAGToDAGISel::SelectTAddrModeImm7(SDValue N, SDValue &Base,
1249 SDValue &OffImm) {
1250 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1251 int RHSC;
1252 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80,
1253 RHSC)) {
1254 Base = N.getOperand(0);
1255 if (N.getOpcode() == ISD::SUB)
1256 RHSC = -RHSC;
1257 OffImm =
1258 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
1259 return true;
1260 }
1261 }
1262
1263 // Base only.
1264 Base = N;
1265 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1266 return true;
1267}
1268
1269
1270//===----------------------------------------------------------------------===//
1271// Thumb 2 Addressing Modes
1272//===----------------------------------------------------------------------===//
1273
1274
1275bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1276 SDValue &Base, SDValue &OffImm) {
1277 // Match simple R + imm12 operands.
1278
1279 // Base only.
1280 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1281 !CurDAG->isBaseWithConstantOffset(N)) {
1282 if (N.getOpcode() == ISD::FrameIndex) {
1283 // Match frame index.
1284 int FI = cast<FrameIndexSDNode>(N)->getIndex();
1285 Base = CurDAG->getTargetFrameIndex(
1286 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1287 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1288 return true;
1289 }
1290
1291 if (N.getOpcode() == ARMISD::Wrapper &&
1292 N.getOperand(0).getOpcode() != ISD::TargetGlobalAddress &&
1293 N.getOperand(0).getOpcode() != ISD::TargetExternalSymbol &&
1294 N.getOperand(0).getOpcode() != ISD::TargetGlobalTLSAddress) {
1295 Base = N.getOperand(0);
1296 if (Base.getOpcode() == ISD::TargetConstantPool)
1297 return false; // We want to select t2LDRpci instead.
1298 } else
1299 Base = N;
1300 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1301 return true;
1302 }
1303
1304 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1305 if (SelectT2AddrModeImm8(N, Base, OffImm))
1306 // Let t2LDRi8 handle (R - imm8).
1307 return false;
1308
1309 int RHSC = (int)RHS->getZExtValue();
1310 if (N.getOpcode() == ISD::SUB)
1311 RHSC = -RHSC;
1312
1313 if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1314 Base = N.getOperand(0);
1315 if (Base.getOpcode() == ISD::FrameIndex) {
1316 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1317 Base = CurDAG->getTargetFrameIndex(
1318 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1319 }
1320 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1321 return true;
1322 }
1323 }
1324
1325 // Base only.
1326 Base = N;
1327 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1328 return true;
1329}
1330
1331template <unsigned Shift>
1332bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N, SDValue &Base,
1333 SDValue &OffImm) {
1334 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1335 int RHSC;
1336 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -255, 256, RHSC)) {
1337 Base = N.getOperand(0);
1338 if (Base.getOpcode() == ISD::FrameIndex) {
1339 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1340 Base = CurDAG->getTargetFrameIndex(
1341 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1342 }
1343
1344 if (N.getOpcode() == ISD::SUB)
1345 RHSC = -RHSC;
1346 OffImm =
1347 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
1348 return true;
1349 }
1350 }
1351
1352 // Base only.
1353 Base = N;
1354 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1355 return true;
1356}
1357
1358bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1359 SDValue &Base, SDValue &OffImm) {
1360 // Match simple R - imm8 operands.
1361 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1362 !CurDAG->isBaseWithConstantOffset(N))
1363 return false;
1364
1365 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1366 int RHSC = (int)RHS->getSExtValue();
1367 if (N.getOpcode() == ISD::SUB)
1368 RHSC = -RHSC;
1369
1370 if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1371 Base = N.getOperand(0);
1372 if (Base.getOpcode() == ISD::FrameIndex) {
1373 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1374 Base = CurDAG->getTargetFrameIndex(
1375 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1376 }
1377 OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1378 return true;
1379 }
1380 }
1381
1382 return false;
1383}
1384
1385bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1386 SDValue &OffImm){
1387 unsigned Opcode = Op->getOpcode();
1388 ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1389 ? cast<LoadSDNode>(Op)->getAddressingMode()
1390 : cast<StoreSDNode>(Op)->getAddressingMode();
1391 int RHSC;
1392 if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1393 OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1394 ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
1395 : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1396 return true;
1397 }
1398
1399 return false;
1400}
1401
1402template <unsigned Shift>
1403bool ARMDAGToDAGISel::SelectT2AddrModeImm7(SDValue N, SDValue &Base,
1404 SDValue &OffImm) {
1405 if (N.getOpcode() == ISD::SUB || CurDAG->isBaseWithConstantOffset(N)) {
1406 int RHSC;
1407 if (isScaledConstantInRange(N.getOperand(1), 1 << Shift, -0x7f, 0x80,
1408 RHSC)) {
1409 Base = N.getOperand(0);
1410 if (Base.getOpcode() == ISD::FrameIndex) {
1411 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1412 Base = CurDAG->getTargetFrameIndex(
1413 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1414 }
1415
1416 if (N.getOpcode() == ISD::SUB)
1417 RHSC = -RHSC;
1418 OffImm =
1419 CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32);
1420 return true;
1421 }
1422 }
1423
1424 // Base only.
1425 Base = N;
1426 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1427 return true;
1428}
1429
1430template <unsigned Shift>
1431bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1432 SDValue &OffImm) {
1433 return SelectT2AddrModeImm7Offset(Op, N, OffImm, Shift);
1434}
1435
1436bool ARMDAGToDAGISel::SelectT2AddrModeImm7Offset(SDNode *Op, SDValue N,
1437 SDValue &OffImm,
1438 unsigned Shift) {
1439 unsigned Opcode = Op->getOpcode();
1440 ISD::MemIndexedMode AM;
1441 switch (Opcode) {
1442 case ISD::LOAD:
1443 AM = cast<LoadSDNode>(Op)->getAddressingMode();
1444 break;
1445 case ISD::STORE:
1446 AM = cast<StoreSDNode>(Op)->getAddressingMode();
1447 break;
1448 case ISD::MLOAD:
1449 AM = cast<MaskedLoadSDNode>(Op)->getAddressingMode();
1450 break;
1451 case ISD::MSTORE:
1452 AM = cast<MaskedStoreSDNode>(Op)->getAddressingMode();
1453 break;
1454 default:
1455 llvm_unreachable("Unexpected Opcode for Imm7Offset")::llvm::llvm_unreachable_internal("Unexpected Opcode for Imm7Offset"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 1455);
1456 }
1457
1458 int RHSC;
1459 // 7 bit constant, shifted by Shift.
1460 if (isScaledConstantInRange(N, 1 << Shift, 0, 0x80, RHSC)) {
1461 OffImm =
1462 ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1463 ? CurDAG->getTargetConstant(RHSC * (1 << Shift), SDLoc(N), MVT::i32)
1464 : CurDAG->getTargetConstant(-RHSC * (1 << Shift), SDLoc(N),
1465 MVT::i32);
1466 return true;
1467 }
1468 return false;
1469}
1470
1471template <int Min, int Max>
1472bool ARMDAGToDAGISel::SelectImmediateInRange(SDValue N, SDValue &OffImm) {
1473 int Val;
1474 if (isScaledConstantInRange(N, 1, Min, Max, Val)) {
1475 OffImm = CurDAG->getTargetConstant(Val, SDLoc(N), MVT::i32);
1476 return true;
1477 }
1478 return false;
1479}
1480
1481bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1482 SDValue &Base,
1483 SDValue &OffReg, SDValue &ShImm) {
1484 // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1485 if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1486 return false;
1487
1488 // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1489 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1490 int RHSC = (int)RHS->getZExtValue();
1491 if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1492 return false;
1493 else if (RHSC < 0 && RHSC >= -255) // 8 bits
1494 return false;
1495 }
1496
1497 // Look for (R + R) or (R + (R << [1,2,3])).
1498 unsigned ShAmt = 0;
1499 Base = N.getOperand(0);
1500 OffReg = N.getOperand(1);
1501
1502 // Swap if it is ((R << c) + R).
1503 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(OffReg.getOpcode());
1504 if (ShOpcVal != ARM_AM::lsl) {
1505 ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1506 if (ShOpcVal == ARM_AM::lsl)
1507 std::swap(Base, OffReg);
1508 }
1509
1510 if (ShOpcVal == ARM_AM::lsl) {
1511 // Check to see if the RHS of the shift is a constant, if not, we can't fold
1512 // it.
1513 if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1514 ShAmt = Sh->getZExtValue();
1515 if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1516 OffReg = OffReg.getOperand(0);
1517 else {
1518 ShAmt = 0;
1519 }
1520 }
1521 }
1522
1523 // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1524 // and use it in a shifted operand do so.
1525 if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
1526 unsigned PowerOfTwo = 0;
1527 SDValue NewMulConst;
1528 if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1529 HandleSDNode Handle(OffReg);
1530 replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1531 OffReg = Handle.getValue();
1532 ShAmt = PowerOfTwo;
1533 }
1534 }
1535
1536 ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1537
1538 return true;
1539}
1540
1541bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1542 SDValue &OffImm) {
1543 // This *must* succeed since it's used for the irreplaceable ldrex and strex
1544 // instructions.
1545 Base = N;
1546 OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1547
1548 if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1549 return true;
1550
1551 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1));
1552 if (!RHS)
1553 return true;
1554
1555 uint32_t RHSC = (int)RHS->getZExtValue();
1556 if (RHSC > 1020 || RHSC % 4 != 0)
1557 return true;
1558
1559 Base = N.getOperand(0);
1560 if (Base.getOpcode() == ISD::FrameIndex) {
1561 int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1562 Base = CurDAG->getTargetFrameIndex(
1563 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1564 }
1565
1566 OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1567 return true;
1568}
1569
1570//===--------------------------------------------------------------------===//
1571
1572/// getAL - Returns a ARMCC::AL immediate node.
1573static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1574 return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1575}
1576
1577void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1578 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1579 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
1580}
1581
1582bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1583 LoadSDNode *LD = cast<LoadSDNode>(N);
1584 ISD::MemIndexedMode AM = LD->getAddressingMode();
1585 if (AM == ISD::UNINDEXED)
1586 return false;
1587
1588 EVT LoadedVT = LD->getMemoryVT();
1589 SDValue Offset, AMOpc;
1590 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1591 unsigned Opcode = 0;
1592 bool Match = false;
1593 if (LoadedVT == MVT::i32 && isPre &&
1594 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1595 Opcode = ARM::LDR_PRE_IMM;
1596 Match = true;
1597 } else if (LoadedVT == MVT::i32 && !isPre &&
1598 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1599 Opcode = ARM::LDR_POST_IMM;
1600 Match = true;
1601 } else if (LoadedVT == MVT::i32 &&
1602 SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1603 Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1604 Match = true;
1605
1606 } else if (LoadedVT == MVT::i16 &&
1607 SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1608 Match = true;
1609 Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1610 ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1611 : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1612 } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1613 if (LD->getExtensionType() == ISD::SEXTLOAD) {
1614 if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1615 Match = true;
1616 Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1617 }
1618 } else {
1619 if (isPre &&
1620 SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1621 Match = true;
1622 Opcode = ARM::LDRB_PRE_IMM;
1623 } else if (!isPre &&
1624 SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1625 Match = true;
1626 Opcode = ARM::LDRB_POST_IMM;
1627 } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1628 Match = true;
1629 Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1630 }
1631 }
1632 }
1633
1634 if (Match) {
1635 if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1636 SDValue Chain = LD->getChain();
1637 SDValue Base = LD->getBasePtr();
1638 SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1639 CurDAG->getRegister(0, MVT::i32), Chain };
1640 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1641 MVT::Other, Ops);
1642 transferMemOperands(N, New);
1643 ReplaceNode(N, New);
1644 return true;
1645 } else {
1646 SDValue Chain = LD->getChain();
1647 SDValue Base = LD->getBasePtr();
1648 SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1649 CurDAG->getRegister(0, MVT::i32), Chain };
1650 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1651 MVT::Other, Ops);
1652 transferMemOperands(N, New);
1653 ReplaceNode(N, New);
1654 return true;
1655 }
1656 }
1657
1658 return false;
1659}
1660
1661bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1662 LoadSDNode *LD = cast<LoadSDNode>(N);
1663 EVT LoadedVT = LD->getMemoryVT();
1664 ISD::MemIndexedMode AM = LD->getAddressingMode();
1665 if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
1666 LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
1667 return false;
1668
1669 auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1670 if (!COffs || COffs->getZExtValue() != 4)
1671 return false;
1672
1673 // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1674 // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1675 // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1676 // ISel.
1677 SDValue Chain = LD->getChain();
1678 SDValue Base = LD->getBasePtr();
1679 SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1680 CurDAG->getRegister(0, MVT::i32), Chain };
1681 SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1682 MVT::i32, MVT::Other, Ops);
1683 transferMemOperands(N, New);
1684 ReplaceNode(N, New);
1685 return true;
1686}
1687
1688bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1689 LoadSDNode *LD = cast<LoadSDNode>(N);
1690 ISD::MemIndexedMode AM = LD->getAddressingMode();
1691 if (AM == ISD::UNINDEXED)
1692 return false;
1693
1694 EVT LoadedVT = LD->getMemoryVT();
1695 bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1696 SDValue Offset;
1697 bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1698 unsigned Opcode = 0;
1699 bool Match = false;
1700 if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1701 switch (LoadedVT.getSimpleVT().SimpleTy) {
1702 case MVT::i32:
1703 Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1704 break;
1705 case MVT::i16:
1706 if (isSExtLd)
1707 Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1708 else
1709 Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1710 break;
1711 case MVT::i8:
1712 case MVT::i1:
1713 if (isSExtLd)
1714 Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1715 else
1716 Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1717 break;
1718 default:
1719 return false;
1720 }
1721 Match = true;
1722 }
1723
1724 if (Match) {
1725 SDValue Chain = LD->getChain();
1726 SDValue Base = LD->getBasePtr();
1727 SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1728 CurDAG->getRegister(0, MVT::i32), Chain };
1729 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1730 MVT::Other, Ops);
1731 transferMemOperands(N, New);
1732 ReplaceNode(N, New);
1733 return true;
1734 }
1735
1736 return false;
1737}
1738
1739bool ARMDAGToDAGISel::tryMVEIndexedLoad(SDNode *N) {
1740 EVT LoadedVT;
1741 unsigned Opcode = 0;
1742 bool isSExtLd, isPre;
1743 Align Alignment;
1744 ARMVCC::VPTCodes Pred;
1745 SDValue PredReg;
1746 SDValue Chain, Base, Offset;
1747
1748 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(N)) {
1749 ISD::MemIndexedMode AM = LD->getAddressingMode();
1750 if (AM == ISD::UNINDEXED)
1751 return false;
1752 LoadedVT = LD->getMemoryVT();
1753 if (!LoadedVT.isVector())
1754 return false;
1755
1756 Chain = LD->getChain();
1757 Base = LD->getBasePtr();
1758 Offset = LD->getOffset();
1759 Alignment = LD->getAlign();
1760 isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1761 isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1762 Pred = ARMVCC::None;
1763 PredReg = CurDAG->getRegister(0, MVT::i32);
1764 } else if (MaskedLoadSDNode *LD = dyn_cast<MaskedLoadSDNode>(N)) {
1765 ISD::MemIndexedMode AM = LD->getAddressingMode();
1766 if (AM == ISD::UNINDEXED)
1767 return false;
1768 LoadedVT = LD->getMemoryVT();
1769 if (!LoadedVT.isVector())
1770 return false;
1771
1772 Chain = LD->getChain();
1773 Base = LD->getBasePtr();
1774 Offset = LD->getOffset();
1775 Alignment = LD->getAlign();
1776 isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1777 isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1778 Pred = ARMVCC::Then;
1779 PredReg = LD->getMask();
1780 } else
1781 llvm_unreachable("Expected a Load or a Masked Load!")::llvm::llvm_unreachable_internal("Expected a Load or a Masked Load!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 1781);
1782
1783 // We allow LE non-masked loads to change the type (for example use a vldrb.8
1784 // as opposed to a vldrw.32). This can allow extra addressing modes or
1785 // alignments for what is otherwise an equivalent instruction.
1786 bool CanChangeType = Subtarget->isLittle() && !isa<MaskedLoadSDNode>(N);
1787
1788 SDValue NewOffset;
1789 if (Alignment >= Align(2) && LoadedVT == MVT::v4i16 &&
1790 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1)) {
1791 if (isSExtLd)
1792 Opcode = isPre ? ARM::MVE_VLDRHS32_pre : ARM::MVE_VLDRHS32_post;
1793 else
1794 Opcode = isPre ? ARM::MVE_VLDRHU32_pre : ARM::MVE_VLDRHU32_post;
1795 } else if (LoadedVT == MVT::v8i8 &&
1796 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) {
1797 if (isSExtLd)
1798 Opcode = isPre ? ARM::MVE_VLDRBS16_pre : ARM::MVE_VLDRBS16_post;
1799 else
1800 Opcode = isPre ? ARM::MVE_VLDRBU16_pre : ARM::MVE_VLDRBU16_post;
1801 } else if (LoadedVT == MVT::v4i8 &&
1802 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0)) {
1803 if (isSExtLd)
1804 Opcode = isPre ? ARM::MVE_VLDRBS32_pre : ARM::MVE_VLDRBS32_post;
1805 else
1806 Opcode = isPre ? ARM::MVE_VLDRBU32_pre : ARM::MVE_VLDRBU32_post;
1807 } else if (Alignment >= Align(4) &&
1808 (CanChangeType || LoadedVT == MVT::v4i32 ||
1809 LoadedVT == MVT::v4f32) &&
1810 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 2))
1811 Opcode = isPre ? ARM::MVE_VLDRWU32_pre : ARM::MVE_VLDRWU32_post;
1812 else if (Alignment >= Align(2) &&
1813 (CanChangeType || LoadedVT == MVT::v8i16 ||
1814 LoadedVT == MVT::v8f16) &&
1815 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 1))
1816 Opcode = isPre ? ARM::MVE_VLDRHU16_pre : ARM::MVE_VLDRHU16_post;
1817 else if ((CanChangeType || LoadedVT == MVT::v16i8) &&
1818 SelectT2AddrModeImm7Offset(N, Offset, NewOffset, 0))
1819 Opcode = isPre ? ARM::MVE_VLDRBU8_pre : ARM::MVE_VLDRBU8_post;
1820 else
1821 return false;
1822
1823 SDValue Ops[] = {Base,
1824 NewOffset,
1825 CurDAG->getTargetConstant(Pred, SDLoc(N), MVT::i32),
1826 PredReg,
1827 CurDAG->getRegister(0, MVT::i32), // tp_reg
1828 Chain};
1829 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32,
1830 N->getValueType(0), MVT::Other, Ops);
1831 transferMemOperands(N, New);
1832 ReplaceUses(SDValue(N, 0), SDValue(New, 1));
1833 ReplaceUses(SDValue(N, 1), SDValue(New, 0));
1834 ReplaceUses(SDValue(N, 2), SDValue(New, 2));
1835 CurDAG->RemoveDeadNode(N);
1836 return true;
1837}
1838
1839/// Form a GPRPair pseudo register from a pair of GPR regs.
1840SDNode *ARMDAGToDAGISel::createGPRPairNode(EVT VT, SDValue V0, SDValue V1) {
1841 SDLoc dl(V0.getNode());
1842 SDValue RegClass =
1843 CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1844 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1845 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1846 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1847 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1848}
1849
1850/// Form a D register from a pair of S registers.
1851SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1852 SDLoc dl(V0.getNode());
1853 SDValue RegClass =
1854 CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1855 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1856 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1857 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1858 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1859}
1860
1861/// Form a quad register from a pair of D registers.
1862SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1863 SDLoc dl(V0.getNode());
1864 SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1865 MVT::i32);
1866 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1867 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1868 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1869 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1870}
1871
1872/// Form 4 consecutive D registers from a pair of Q registers.
1873SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1874 SDLoc dl(V0.getNode());
1875 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1876 MVT::i32);
1877 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1878 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1879 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1880 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1881}
1882
1883/// Form 4 consecutive S registers.
1884SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1885 SDValue V2, SDValue V3) {
1886 SDLoc dl(V0.getNode());
1887 SDValue RegClass =
1888 CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1889 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1890 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1891 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1892 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1893 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1894 V2, SubReg2, V3, SubReg3 };
1895 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1896}
1897
1898/// Form 4 consecutive D registers.
1899SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1900 SDValue V2, SDValue V3) {
1901 SDLoc dl(V0.getNode());
1902 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1903 MVT::i32);
1904 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1905 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1906 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1907 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1908 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1909 V2, SubReg2, V3, SubReg3 };
1910 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1911}
1912
1913/// Form 4 consecutive Q registers.
1914SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1915 SDValue V2, SDValue V3) {
1916 SDLoc dl(V0.getNode());
1917 SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1918 MVT::i32);
1919 SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1920 SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1921 SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1922 SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1923 const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1924 V2, SubReg2, V3, SubReg3 };
1925 return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1926}
1927
1928/// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1929/// of a NEON VLD or VST instruction. The supported values depend on the
1930/// number of registers being loaded.
1931SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1932 unsigned NumVecs, bool is64BitVector) {
1933 unsigned NumRegs = NumVecs;
1934 if (!is64BitVector && NumVecs < 3)
1935 NumRegs *= 2;
1936
1937 unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1938 if (Alignment >= 32 && NumRegs == 4)
1939 Alignment = 32;
1940 else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1941 Alignment = 16;
1942 else if (Alignment >= 8)
1943 Alignment = 8;
1944 else
1945 Alignment = 0;
1946
1947 return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1948}
1949
1950static bool isVLDfixed(unsigned Opc)
1951{
1952 switch (Opc) {
1953 default: return false;
1954 case ARM::VLD1d8wb_fixed : return true;
1955 case ARM::VLD1d16wb_fixed : return true;
1956 case ARM::VLD1d64Qwb_fixed : return true;
1957 case ARM::VLD1d32wb_fixed : return true;
1958 case ARM::VLD1d64wb_fixed : return true;
1959 case ARM::VLD1d8TPseudoWB_fixed : return true;
1960 case ARM::VLD1d16TPseudoWB_fixed : return true;
1961 case ARM::VLD1d32TPseudoWB_fixed : return true;
1962 case ARM::VLD1d64TPseudoWB_fixed : return true;
1963 case ARM::VLD1d8QPseudoWB_fixed : return true;
1964 case ARM::VLD1d16QPseudoWB_fixed : return true;
1965 case ARM::VLD1d32QPseudoWB_fixed : return true;
1966 case ARM::VLD1d64QPseudoWB_fixed : return true;
1967 case ARM::VLD1q8wb_fixed : return true;
1968 case ARM::VLD1q16wb_fixed : return true;
1969 case ARM::VLD1q32wb_fixed : return true;
1970 case ARM::VLD1q64wb_fixed : return true;
1971 case ARM::VLD1DUPd8wb_fixed : return true;
1972 case ARM::VLD1DUPd16wb_fixed : return true;
1973 case ARM::VLD1DUPd32wb_fixed : return true;
1974 case ARM::VLD1DUPq8wb_fixed : return true;
1975 case ARM::VLD1DUPq16wb_fixed : return true;
1976 case ARM::VLD1DUPq32wb_fixed : return true;
1977 case ARM::VLD2d8wb_fixed : return true;
1978 case ARM::VLD2d16wb_fixed : return true;
1979 case ARM::VLD2d32wb_fixed : return true;
1980 case ARM::VLD2q8PseudoWB_fixed : return true;
1981 case ARM::VLD2q16PseudoWB_fixed : return true;
1982 case ARM::VLD2q32PseudoWB_fixed : return true;
1983 case ARM::VLD2DUPd8wb_fixed : return true;
1984 case ARM::VLD2DUPd16wb_fixed : return true;
1985 case ARM::VLD2DUPd32wb_fixed : return true;
1986 case ARM::VLD2DUPq8OddPseudoWB_fixed: return true;
1987 case ARM::VLD2DUPq16OddPseudoWB_fixed: return true;
1988 case ARM::VLD2DUPq32OddPseudoWB_fixed: return true;
1989 }
1990}
1991
1992static bool isVSTfixed(unsigned Opc)
1993{
1994 switch (Opc) {
1995 default: return false;
1996 case ARM::VST1d8wb_fixed : return true;
1997 case ARM::VST1d16wb_fixed : return true;
1998 case ARM::VST1d32wb_fixed : return true;
1999 case ARM::VST1d64wb_fixed : return true;
2000 case ARM::VST1q8wb_fixed : return true;
2001 case ARM::VST1q16wb_fixed : return true;
2002 case ARM::VST1q32wb_fixed : return true;
2003 case ARM::VST1q64wb_fixed : return true;
2004 case ARM::VST1d8TPseudoWB_fixed : return true;
2005 case ARM::VST1d16TPseudoWB_fixed : return true;
2006 case ARM::VST1d32TPseudoWB_fixed : return true;
2007 case ARM::VST1d64TPseudoWB_fixed : return true;
2008 case ARM::VST1d8QPseudoWB_fixed : return true;
2009 case ARM::VST1d16QPseudoWB_fixed : return true;
2010 case ARM::VST1d32QPseudoWB_fixed : return true;
2011 case ARM::VST1d64QPseudoWB_fixed : return true;
2012 case ARM::VST2d8wb_fixed : return true;
2013 case ARM::VST2d16wb_fixed : return true;
2014 case ARM::VST2d32wb_fixed : return true;
2015 case ARM::VST2q8PseudoWB_fixed : return true;
2016 case ARM::VST2q16PseudoWB_fixed : return true;
2017 case ARM::VST2q32PseudoWB_fixed : return true;
2018 }
2019}
2020
2021// Get the register stride update opcode of a VLD/VST instruction that
2022// is otherwise equivalent to the given fixed stride updating instruction.
2023static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
2024 assert((isVLDfixed(Opc) || isVSTfixed(Opc))(static_cast <bool> ((isVLDfixed(Opc) || isVSTfixed(Opc
)) && "Incorrect fixed stride updating instruction.")
? void (0) : __assert_fail ("(isVLDfixed(Opc) || isVSTfixed(Opc)) && \"Incorrect fixed stride updating instruction.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2025, __extension__ __PRETTY_FUNCTION__))
2025 && "Incorrect fixed stride updating instruction.")(static_cast <bool> ((isVLDfixed(Opc) || isVSTfixed(Opc
)) && "Incorrect fixed stride updating instruction.")
? void (0) : __assert_fail ("(isVLDfixed(Opc) || isVSTfixed(Opc)) && \"Incorrect fixed stride updating instruction.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2025, __extension__ __PRETTY_FUNCTION__));
2026 switch (Opc) {
2027 default: break;
2028 case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
2029 case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
2030 case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
2031 case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
2032 case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
2033 case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
2034 case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
2035 case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
2036 case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
2037 case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
2038 case ARM::VLD1d8TPseudoWB_fixed: return ARM::VLD1d8TPseudoWB_register;
2039 case ARM::VLD1d16TPseudoWB_fixed: return ARM::VLD1d16TPseudoWB_register;
2040 case ARM::VLD1d32TPseudoWB_fixed: return ARM::VLD1d32TPseudoWB_register;
2041 case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
2042 case ARM::VLD1d8QPseudoWB_fixed: return ARM::VLD1d8QPseudoWB_register;
2043 case ARM::VLD1d16QPseudoWB_fixed: return ARM::VLD1d16QPseudoWB_register;
2044 case ARM::VLD1d32QPseudoWB_fixed: return ARM::VLD1d32QPseudoWB_register;
2045 case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
2046 case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
2047 case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
2048 case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
2049 case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
2050 case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
2051 case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
2052 case ARM::VLD2DUPq8OddPseudoWB_fixed: return ARM::VLD2DUPq8OddPseudoWB_register;
2053 case ARM::VLD2DUPq16OddPseudoWB_fixed: return ARM::VLD2DUPq16OddPseudoWB_register;
2054 case ARM::VLD2DUPq32OddPseudoWB_fixed: return ARM::VLD2DUPq32OddPseudoWB_register;
2055
2056 case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
2057 case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
2058 case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
2059 case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
2060 case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
2061 case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
2062 case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
2063 case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
2064 case ARM::VST1d8TPseudoWB_fixed: return ARM::VST1d8TPseudoWB_register;
2065 case ARM::VST1d16TPseudoWB_fixed: return ARM::VST1d16TPseudoWB_register;
2066 case ARM::VST1d32TPseudoWB_fixed: return ARM::VST1d32TPseudoWB_register;
2067 case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
2068 case ARM::VST1d8QPseudoWB_fixed: return ARM::VST1d8QPseudoWB_register;
2069 case ARM::VST1d16QPseudoWB_fixed: return ARM::VST1d16QPseudoWB_register;
2070 case ARM::VST1d32QPseudoWB_fixed: return ARM::VST1d32QPseudoWB_register;
2071 case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
2072
2073 case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
2074 case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
2075 case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
2076 case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
2077 case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
2078 case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
2079
2080 case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
2081 case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
2082 case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
2083 case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
2084 case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
2085 case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
2086
2087 case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
2088 case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
2089 case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
2090 }
2091 return Opc; // If not one we handle, return it unchanged.
2092}
2093
2094/// Returns true if the given increment is a Constant known to be equal to the
2095/// access size performed by a NEON load/store. This means the "[rN]!" form can
2096/// be used.
2097static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
2098 auto C = dyn_cast<ConstantSDNode>(Inc);
2099 return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
2100}
2101
2102void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
2103 const uint16_t *DOpcodes,
2104 const uint16_t *QOpcodes0,
2105 const uint16_t *QOpcodes1) {
2106 assert(Subtarget->hasNEON())(static_cast <bool> (Subtarget->hasNEON()) ? void (0
) : __assert_fail ("Subtarget->hasNEON()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2106, __extension__ __PRETTY_FUNCTION__));
2107 assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range")(static_cast <bool> (NumVecs >= 1 && NumVecs
<= 4 && "VLD NumVecs out-of-range") ? void (0) : __assert_fail
("NumVecs >= 1 && NumVecs <= 4 && \"VLD NumVecs out-of-range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2107, __extension__ __PRETTY_FUNCTION__));
2108 SDLoc dl(N);
2109
2110 SDValue MemAddr, Align;
2111 bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2112 // nodes are not intrinsics.
2113 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2114 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2115 return;
2116
2117 SDValue Chain = N->getOperand(0);
2118 EVT VT = N->getValueType(0);
2119 bool is64BitVector = VT.is64BitVector();
2120 Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
2121
2122 unsigned OpcodeIndex;
2123 switch (VT.getSimpleVT().SimpleTy) {
2124 default: llvm_unreachable("unhandled vld type")::llvm::llvm_unreachable_internal("unhandled vld type", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2124);
2125 // Double-register operations:
2126 case MVT::v8i8: OpcodeIndex = 0; break;
2127 case MVT::v4f16:
2128 case MVT::v4bf16:
2129 case MVT::v4i16: OpcodeIndex = 1; break;
2130 case MVT::v2f32:
2131 case MVT::v2i32: OpcodeIndex = 2; break;
2132 case MVT::v1i64: OpcodeIndex = 3; break;
2133 // Quad-register operations:
2134 case MVT::v16i8: OpcodeIndex = 0; break;
2135 case MVT::v8f16:
2136 case MVT::v8bf16:
2137 case MVT::v8i16: OpcodeIndex = 1; break;
2138 case MVT::v4f32:
2139 case MVT::v4i32: OpcodeIndex = 2; break;
2140 case MVT::v2f64:
2141 case MVT::v2i64: OpcodeIndex = 3; break;
2142 }
2143
2144 EVT ResTy;
2145 if (NumVecs == 1)
2146 ResTy = VT;
2147 else {
2148 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2149 if (!is64BitVector)
2150 ResTyElts *= 2;
2151 ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2152 }
2153 std::vector<EVT> ResTys;
2154 ResTys.push_back(ResTy);
2155 if (isUpdating)
2156 ResTys.push_back(MVT::i32);
2157 ResTys.push_back(MVT::Other);
2158
2159 SDValue Pred = getAL(CurDAG, dl);
2160 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2161 SDNode *VLd;
2162 SmallVector<SDValue, 7> Ops;
2163
2164 // Double registers and VLD1/VLD2 quad registers are directly supported.
2165 if (is64BitVector || NumVecs <= 2) {
2166 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2167 QOpcodes0[OpcodeIndex]);
2168 Ops.push_back(MemAddr);
2169 Ops.push_back(Align);
2170 if (isUpdating) {
2171 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2172 bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
2173 if (!IsImmUpdate) {
2174 // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
2175 // check for the opcode rather than the number of vector elements.
2176 if (isVLDfixed(Opc))
2177 Opc = getVLDSTRegisterUpdateOpcode(Opc);
2178 Ops.push_back(Inc);
2179 // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
2180 // the operands if not such an opcode.
2181 } else if (!isVLDfixed(Opc))
2182 Ops.push_back(Reg0);
2183 }
2184 Ops.push_back(Pred);
2185 Ops.push_back(Reg0);
2186 Ops.push_back(Chain);
2187 VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2188
2189 } else {
2190 // Otherwise, quad registers are loaded with two separate instructions,
2191 // where one loads the even registers and the other loads the odd registers.
2192 EVT AddrTy = MemAddr.getValueType();
2193
2194 // Load the even subregs. This is always an updating load, so that it
2195 // provides the address to the second load for the odd subregs.
2196 SDValue ImplDef =
2197 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2198 const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
2199 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2200 ResTy, AddrTy, MVT::Other, OpsA);
2201 Chain = SDValue(VLdA, 2);
2202
2203 // Load the odd subregs.
2204 Ops.push_back(SDValue(VLdA, 1));
2205 Ops.push_back(Align);
2206 if (isUpdating) {
2207 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2208 assert(isa<ConstantSDNode>(Inc.getNode()) &&(static_cast <bool> (isa<ConstantSDNode>(Inc.getNode
()) && "only constant post-increment update allowed for VLD3/4"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(Inc.getNode()) && \"only constant post-increment update allowed for VLD3/4\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2209, __extension__ __PRETTY_FUNCTION__))
2209 "only constant post-increment update allowed for VLD3/4")(static_cast <bool> (isa<ConstantSDNode>(Inc.getNode
()) && "only constant post-increment update allowed for VLD3/4"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(Inc.getNode()) && \"only constant post-increment update allowed for VLD3/4\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2209, __extension__ __PRETTY_FUNCTION__));
2210 (void)Inc;
2211 Ops.push_back(Reg0);
2212 }
2213 Ops.push_back(SDValue(VLdA, 0));
2214 Ops.push_back(Pred);
2215 Ops.push_back(Reg0);
2216 Ops.push_back(Chain);
2217 VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
2218 }
2219
2220 // Transfer memoperands.
2221 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2222 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp});
2223
2224 if (NumVecs == 1) {
2225 ReplaceNode(N, VLd);
2226 return;
2227 }
2228
2229 // Extract out the subregisters.
2230 SDValue SuperReg = SDValue(VLd, 0);
2231 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2232 ARM::qsub_3 == ARM::qsub_0 + 3,
2233 "Unexpected subreg numbering");
2234 unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
2235 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2236 ReplaceUses(SDValue(N, Vec),
2237 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2238 ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
2239 if (isUpdating)
2240 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
2241 CurDAG->RemoveDeadNode(N);
2242}
2243
2244void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
2245 const uint16_t *DOpcodes,
2246 const uint16_t *QOpcodes0,
2247 const uint16_t *QOpcodes1) {
2248 assert(Subtarget->hasNEON())(static_cast <bool> (Subtarget->hasNEON()) ? void (0
) : __assert_fail ("Subtarget->hasNEON()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2248, __extension__ __PRETTY_FUNCTION__));
2249 assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range")(static_cast <bool> (NumVecs >= 1 && NumVecs
<= 4 && "VST NumVecs out-of-range") ? void (0) : __assert_fail
("NumVecs >= 1 && NumVecs <= 4 && \"VST NumVecs out-of-range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2249, __extension__ __PRETTY_FUNCTION__));
2250 SDLoc dl(N);
2251
2252 SDValue MemAddr, Align;
2253 bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2254 // nodes are not intrinsics.
2255 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2256 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2257 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2258 return;
2259
2260 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2261
2262 SDValue Chain = N->getOperand(0);
2263 EVT VT = N->getOperand(Vec0Idx).getValueType();
2264 bool is64BitVector = VT.is64BitVector();
2265 Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
2266
2267 unsigned OpcodeIndex;
2268 switch (VT.getSimpleVT().SimpleTy) {
2269 default: llvm_unreachable("unhandled vst type")::llvm::llvm_unreachable_internal("unhandled vst type", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2269);
2270 // Double-register operations:
2271 case MVT::v8i8: OpcodeIndex = 0; break;
2272 case MVT::v4f16:
2273 case MVT::v4bf16:
2274 case MVT::v4i16: OpcodeIndex = 1; break;
2275 case MVT::v2f32:
2276 case MVT::v2i32: OpcodeIndex = 2; break;
2277 case MVT::v1i64: OpcodeIndex = 3; break;
2278 // Quad-register operations:
2279 case MVT::v16i8: OpcodeIndex = 0; break;
2280 case MVT::v8f16:
2281 case MVT::v8bf16:
2282 case MVT::v8i16: OpcodeIndex = 1; break;
2283 case MVT::v4f32:
2284 case MVT::v4i32: OpcodeIndex = 2; break;
2285 case MVT::v2f64:
2286 case MVT::v2i64: OpcodeIndex = 3; break;
2287 }
2288
2289 std::vector<EVT> ResTys;
2290 if (isUpdating)
2291 ResTys.push_back(MVT::i32);
2292 ResTys.push_back(MVT::Other);
2293
2294 SDValue Pred = getAL(CurDAG, dl);
2295 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2296 SmallVector<SDValue, 7> Ops;
2297
2298 // Double registers and VST1/VST2 quad registers are directly supported.
2299 if (is64BitVector || NumVecs <= 2) {
2300 SDValue SrcReg;
2301 if (NumVecs == 1) {
2302 SrcReg = N->getOperand(Vec0Idx);
2303 } else if (is64BitVector) {
2304 // Form a REG_SEQUENCE to force register allocation.
2305 SDValue V0 = N->getOperand(Vec0Idx + 0);
2306 SDValue V1 = N->getOperand(Vec0Idx + 1);
2307 if (NumVecs == 2)
2308 SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2309 else {
2310 SDValue V2 = N->getOperand(Vec0Idx + 2);
2311 // If it's a vst3, form a quad D-register and leave the last part as
2312 // an undef.
2313 SDValue V3 = (NumVecs == 3)
2314 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
2315 : N->getOperand(Vec0Idx + 3);
2316 SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2317 }
2318 } else {
2319 // Form a QQ register.
2320 SDValue Q0 = N->getOperand(Vec0Idx);
2321 SDValue Q1 = N->getOperand(Vec0Idx + 1);
2322 SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
2323 }
2324
2325 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2326 QOpcodes0[OpcodeIndex]);
2327 Ops.push_back(MemAddr);
2328 Ops.push_back(Align);
2329 if (isUpdating) {
2330 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2331 bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
2332 if (!IsImmUpdate) {
2333 // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
2334 // check for the opcode rather than the number of vector elements.
2335 if (isVSTfixed(Opc))
2336 Opc = getVLDSTRegisterUpdateOpcode(Opc);
2337 Ops.push_back(Inc);
2338 }
2339 // VST1/VST2 fixed increment does not need Reg0 so only include it in
2340 // the operands if not such an opcode.
2341 else if (!isVSTfixed(Opc))
2342 Ops.push_back(Reg0);
2343 }
2344 Ops.push_back(SrcReg);
2345 Ops.push_back(Pred);
2346 Ops.push_back(Reg0);
2347 Ops.push_back(Chain);
2348 SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2349
2350 // Transfer memoperands.
2351 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp});
2352
2353 ReplaceNode(N, VSt);
2354 return;
2355 }
2356
2357 // Otherwise, quad registers are stored with two separate instructions,
2358 // where one stores the even registers and the other stores the odd registers.
2359
2360 // Form the QQQQ REG_SEQUENCE.
2361 SDValue V0 = N->getOperand(Vec0Idx + 0);
2362 SDValue V1 = N->getOperand(Vec0Idx + 1);
2363 SDValue V2 = N->getOperand(Vec0Idx + 2);
2364 SDValue V3 = (NumVecs == 3)
2365 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2366 : N->getOperand(Vec0Idx + 3);
2367 SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2368
2369 // Store the even D registers. This is always an updating store, so that it
2370 // provides the address to the second store for the odd subregs.
2371 const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2372 SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2373 MemAddr.getValueType(),
2374 MVT::Other, OpsA);
2375 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp});
2376 Chain = SDValue(VStA, 1);
2377
2378 // Store the odd D registers.
2379 Ops.push_back(SDValue(VStA, 0));
2380 Ops.push_back(Align);
2381 if (isUpdating) {
2382 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2383 assert(isa<ConstantSDNode>(Inc.getNode()) &&(static_cast <bool> (isa<ConstantSDNode>(Inc.getNode
()) && "only constant post-increment update allowed for VST3/4"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(Inc.getNode()) && \"only constant post-increment update allowed for VST3/4\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2384, __extension__ __PRETTY_FUNCTION__))
2384 "only constant post-increment update allowed for VST3/4")(static_cast <bool> (isa<ConstantSDNode>(Inc.getNode
()) && "only constant post-increment update allowed for VST3/4"
) ? void (0) : __assert_fail ("isa<ConstantSDNode>(Inc.getNode()) && \"only constant post-increment update allowed for VST3/4\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2384, __extension__ __PRETTY_FUNCTION__));
2385 (void)Inc;
2386 Ops.push_back(Reg0);
2387 }
2388 Ops.push_back(RegSeq);
2389 Ops.push_back(Pred);
2390 Ops.push_back(Reg0);
2391 Ops.push_back(Chain);
2392 SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2393 Ops);
2394 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp});
2395 ReplaceNode(N, VStB);
2396}
2397
2398void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2399 unsigned NumVecs,
2400 const uint16_t *DOpcodes,
2401 const uint16_t *QOpcodes) {
2402 assert(Subtarget->hasNEON())(static_cast <bool> (Subtarget->hasNEON()) ? void (0
) : __assert_fail ("Subtarget->hasNEON()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2402, __extension__ __PRETTY_FUNCTION__));
2403 assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range")(static_cast <bool> (NumVecs >=2 && NumVecs <=
4 && "VLDSTLane NumVecs out-of-range") ? void (0) : __assert_fail
("NumVecs >=2 && NumVecs <= 4 && \"VLDSTLane NumVecs out-of-range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2403, __extension__ __PRETTY_FUNCTION__));
2404 SDLoc dl(N);
2405
2406 SDValue MemAddr, Align;
2407 bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2408 // nodes are not intrinsics.
2409 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2410 unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2411 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2412 return;
2413
2414 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2415
2416 SDValue Chain = N->getOperand(0);
2417 unsigned Lane =
2418 cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2419 EVT VT = N->getOperand(Vec0Idx).getValueType();
2420 bool is64BitVector = VT.is64BitVector();
2421
2422 unsigned Alignment = 0;
2423 if (NumVecs != 3) {
2424 Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2425 unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2426 if (Alignment > NumBytes)
2427 Alignment = NumBytes;
2428 if (Alignment < 8 && Alignment < NumBytes)
2429 Alignment = 0;
2430 // Alignment must be a power of two; make sure of that.
2431 Alignment = (Alignment & -Alignment);
2432 if (Alignment == 1)
2433 Alignment = 0;
2434 }
2435 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2436
2437 unsigned OpcodeIndex;
2438 switch (VT.getSimpleVT().SimpleTy) {
2439 default: llvm_unreachable("unhandled vld/vst lane type")::llvm::llvm_unreachable_internal("unhandled vld/vst lane type"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2439);
2440 // Double-register operations:
2441 case MVT::v8i8: OpcodeIndex = 0; break;
2442 case MVT::v4f16:
2443 case MVT::v4bf16:
2444 case MVT::v4i16: OpcodeIndex = 1; break;
2445 case MVT::v2f32:
2446 case MVT::v2i32: OpcodeIndex = 2; break;
2447 // Quad-register operations:
2448 case MVT::v8f16:
2449 case MVT::v8bf16:
2450 case MVT::v8i16: OpcodeIndex = 0; break;
2451 case MVT::v4f32:
2452 case MVT::v4i32: OpcodeIndex = 1; break;
2453 }
2454
2455 std::vector<EVT> ResTys;
2456 if (IsLoad) {
2457 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2458 if (!is64BitVector)
2459 ResTyElts *= 2;
2460 ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2461 MVT::i64, ResTyElts));
2462 }
2463 if (isUpdating)
2464 ResTys.push_back(MVT::i32);
2465 ResTys.push_back(MVT::Other);
2466
2467 SDValue Pred = getAL(CurDAG, dl);
2468 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2469
2470 SmallVector<SDValue, 8> Ops;
2471 Ops.push_back(MemAddr);
2472 Ops.push_back(Align);
2473 if (isUpdating) {
2474 SDValue Inc = N->getOperand(AddrOpIdx + 1);
2475 bool IsImmUpdate =
2476 isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2477 Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2478 }
2479
2480 SDValue SuperReg;
2481 SDValue V0 = N->getOperand(Vec0Idx + 0);
2482 SDValue V1 = N->getOperand(Vec0Idx + 1);
2483 if (NumVecs == 2) {
2484 if (is64BitVector)
2485 SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2486 else
2487 SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2488 } else {
2489 SDValue V2 = N->getOperand(Vec0Idx + 2);
2490 SDValue V3 = (NumVecs == 3)
2491 ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2492 : N->getOperand(Vec0Idx + 3);
2493 if (is64BitVector)
2494 SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2495 else
2496 SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2497 }
2498 Ops.push_back(SuperReg);
2499 Ops.push_back(getI32Imm(Lane, dl));
2500 Ops.push_back(Pred);
2501 Ops.push_back(Reg0);
2502 Ops.push_back(Chain);
2503
2504 unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2505 QOpcodes[OpcodeIndex]);
2506 SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2507 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp});
2508 if (!IsLoad) {
2509 ReplaceNode(N, VLdLn);
2510 return;
2511 }
2512
2513 // Extract the subregisters.
2514 SuperReg = SDValue(VLdLn, 0);
2515 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2516 ARM::qsub_3 == ARM::qsub_0 + 3,
2517 "Unexpected subreg numbering");
2518 unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2519 for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2520 ReplaceUses(SDValue(N, Vec),
2521 CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2522 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2523 if (isUpdating)
2524 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2525 CurDAG->RemoveDeadNode(N);
2526}
2527
2528template <typename SDValueVector>
2529void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2530 SDValue PredicateMask) {
2531 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32));
2532 Ops.push_back(PredicateMask);
2533 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg
2534}
2535
2536template <typename SDValueVector>
2537void ARMDAGToDAGISel::AddMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2538 SDValue PredicateMask,
2539 SDValue Inactive) {
2540 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::Then, Loc, MVT::i32));
2541 Ops.push_back(PredicateMask);
2542 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg
2543 Ops.push_back(Inactive);
2544}
2545
2546template <typename SDValueVector>
2547void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc) {
2548 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32));
2549 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2550 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg
2551}
2552
2553template <typename SDValueVector>
2554void ARMDAGToDAGISel::AddEmptyMVEPredicateToOps(SDValueVector &Ops, SDLoc Loc,
2555 EVT InactiveTy) {
2556 Ops.push_back(CurDAG->getTargetConstant(ARMVCC::None, Loc, MVT::i32));
2557 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2558 Ops.push_back(CurDAG->getRegister(0, MVT::i32)); // tp_reg
2559 Ops.push_back(SDValue(
2560 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, InactiveTy), 0));
2561}
2562
2563void ARMDAGToDAGISel::SelectMVE_WB(SDNode *N, const uint16_t *Opcodes,
2564 bool Predicated) {
2565 SDLoc Loc(N);
2566 SmallVector<SDValue, 8> Ops;
2567
2568 uint16_t Opcode;
2569 switch (N->getValueType(1).getVectorElementType().getSizeInBits()) {
2570 case 32:
2571 Opcode = Opcodes[0];
2572 break;
2573 case 64:
2574 Opcode = Opcodes[1];
2575 break;
2576 default:
2577 llvm_unreachable("bad vector element size in SelectMVE_WB")::llvm::llvm_unreachable_internal("bad vector element size in SelectMVE_WB"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2577);
2578 }
2579
2580 Ops.push_back(N->getOperand(2)); // vector of base addresses
2581
2582 int32_t ImmValue = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
2583 Ops.push_back(getI32Imm(ImmValue, Loc)); // immediate offset
2584
2585 if (Predicated)
2586 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4));
2587 else
2588 AddEmptyMVEPredicateToOps(Ops, Loc);
2589
2590 Ops.push_back(N->getOperand(0)); // chain
2591
2592 SmallVector<EVT, 8> VTs;
2593 VTs.push_back(N->getValueType(1));
2594 VTs.push_back(N->getValueType(0));
2595 VTs.push_back(N->getValueType(2));
2596
2597 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), VTs, Ops);
2598 ReplaceUses(SDValue(N, 0), SDValue(New, 1));
2599 ReplaceUses(SDValue(N, 1), SDValue(New, 0));
2600 ReplaceUses(SDValue(N, 2), SDValue(New, 2));
2601 transferMemOperands(N, New);
2602 CurDAG->RemoveDeadNode(N);
2603}
2604
2605void ARMDAGToDAGISel::SelectMVE_LongShift(SDNode *N, uint16_t Opcode,
2606 bool Immediate,
2607 bool HasSaturationOperand) {
2608 SDLoc Loc(N);
2609 SmallVector<SDValue, 8> Ops;
2610
2611 // Two 32-bit halves of the value to be shifted
2612 Ops.push_back(N->getOperand(1));
2613 Ops.push_back(N->getOperand(2));
2614
2615 // The shift count
2616 if (Immediate) {
2617 int32_t ImmValue = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
2618 Ops.push_back(getI32Imm(ImmValue, Loc)); // immediate shift count
2619 } else {
2620 Ops.push_back(N->getOperand(3));
2621 }
2622
2623 // The immediate saturation operand, if any
2624 if (HasSaturationOperand) {
2625 int32_t SatOp = cast<ConstantSDNode>(N->getOperand(4))->getZExtValue();
2626 int SatBit = (SatOp == 64 ? 0 : 1);
2627 Ops.push_back(getI32Imm(SatBit, Loc));
2628 }
2629
2630 // MVE scalar shifts are IT-predicable, so include the standard
2631 // predicate arguments.
2632 Ops.push_back(getAL(CurDAG, Loc));
2633 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
2634
2635 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
2636}
2637
2638void ARMDAGToDAGISel::SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
2639 uint16_t OpcodeWithNoCarry,
2640 bool Add, bool Predicated) {
2641 SDLoc Loc(N);
2642 SmallVector<SDValue, 8> Ops;
2643 uint16_t Opcode;
2644
2645 unsigned FirstInputOp = Predicated ? 2 : 1;
2646
2647 // Two input vectors and the input carry flag
2648 Ops.push_back(N->getOperand(FirstInputOp));
2649 Ops.push_back(N->getOperand(FirstInputOp + 1));
2650 SDValue CarryIn = N->getOperand(FirstInputOp + 2);
2651 ConstantSDNode *CarryInConstant = dyn_cast<ConstantSDNode>(CarryIn);
2652 uint32_t CarryMask = 1 << 29;
2653 uint32_t CarryExpected = Add ? 0 : CarryMask;
2654 if (CarryInConstant &&
2655 (CarryInConstant->getZExtValue() & CarryMask) == CarryExpected) {
2656 Opcode = OpcodeWithNoCarry;
2657 } else {
2658 Ops.push_back(CarryIn);
2659 Opcode = OpcodeWithCarry;
2660 }
2661
2662 if (Predicated)
2663 AddMVEPredicateToOps(Ops, Loc,
2664 N->getOperand(FirstInputOp + 3), // predicate
2665 N->getOperand(FirstInputOp - 1)); // inactive
2666 else
2667 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0));
2668
2669 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
2670}
2671
2672void ARMDAGToDAGISel::SelectMVE_VSHLC(SDNode *N, bool Predicated) {
2673 SDLoc Loc(N);
2674 SmallVector<SDValue, 8> Ops;
2675
2676 // One vector input, followed by a 32-bit word of bits to shift in
2677 // and then an immediate shift count
2678 Ops.push_back(N->getOperand(1));
2679 Ops.push_back(N->getOperand(2));
2680 int32_t ImmValue = cast<ConstantSDNode>(N->getOperand(3))->getZExtValue();
2681 Ops.push_back(getI32Imm(ImmValue, Loc)); // immediate shift count
2682
2683 if (Predicated)
2684 AddMVEPredicateToOps(Ops, Loc, N->getOperand(4));
2685 else
2686 AddEmptyMVEPredicateToOps(Ops, Loc);
2687
2688 CurDAG->SelectNodeTo(N, ARM::MVE_VSHLC, N->getVTList(), makeArrayRef(Ops));
2689}
2690
2691static bool SDValueToConstBool(SDValue SDVal) {
2692 assert(isa<ConstantSDNode>(SDVal) && "expected a compile-time constant")(static_cast <bool> (isa<ConstantSDNode>(SDVal) &&
"expected a compile-time constant") ? void (0) : __assert_fail
("isa<ConstantSDNode>(SDVal) && \"expected a compile-time constant\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2692, __extension__ __PRETTY_FUNCTION__));
2693 ConstantSDNode *SDValConstant = dyn_cast<ConstantSDNode>(SDVal);
2694 uint64_t Value = SDValConstant->getZExtValue();
2695 assert((Value == 0 || Value == 1) && "expected value 0 or 1")(static_cast <bool> ((Value == 0 || Value == 1) &&
"expected value 0 or 1") ? void (0) : __assert_fail ("(Value == 0 || Value == 1) && \"expected value 0 or 1\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2695, __extension__ __PRETTY_FUNCTION__));
2696 return Value;
2697}
2698
2699void ARMDAGToDAGISel::SelectBaseMVE_VMLLDAV(SDNode *N, bool Predicated,
2700 const uint16_t *OpcodesS,
2701 const uint16_t *OpcodesU,
2702 size_t Stride, size_t TySize) {
2703 assert(TySize < Stride && "Invalid TySize")(static_cast <bool> (TySize < Stride && "Invalid TySize"
) ? void (0) : __assert_fail ("TySize < Stride && \"Invalid TySize\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2703, __extension__ __PRETTY_FUNCTION__));
2704 bool IsUnsigned = SDValueToConstBool(N->getOperand(1));
2705 bool IsSub = SDValueToConstBool(N->getOperand(2));
2706 bool IsExchange = SDValueToConstBool(N->getOperand(3));
2707 if (IsUnsigned) {
2708 assert(!IsSub &&(static_cast <bool> (!IsSub && "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist"
) ? void (0) : __assert_fail ("!IsSub && \"Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2709, __extension__ __PRETTY_FUNCTION__))
2709 "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist")(static_cast <bool> (!IsSub && "Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist"
) ? void (0) : __assert_fail ("!IsSub && \"Unsigned versions of vmlsldav[a]/vrmlsldavh[a] do not exist\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2709, __extension__ __PRETTY_FUNCTION__));
2710 assert(!IsExchange &&(static_cast <bool> (!IsExchange && "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist"
) ? void (0) : __assert_fail ("!IsExchange && \"Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2711, __extension__ __PRETTY_FUNCTION__))
2711 "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist")(static_cast <bool> (!IsExchange && "Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist"
) ? void (0) : __assert_fail ("!IsExchange && \"Unsigned versions of vmlaldav[a]x/vrmlaldavh[a]x do not exist\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2711, __extension__ __PRETTY_FUNCTION__));
2712 }
2713
2714 auto OpIsZero = [N](size_t OpNo) {
2715 if (ConstantSDNode *OpConst = dyn_cast<ConstantSDNode>(N->getOperand(OpNo)))
2716 if (OpConst->getZExtValue() == 0)
2717 return true;
2718 return false;
2719 };
2720
2721 // If the input accumulator value is not zero, select an instruction with
2722 // accumulator, otherwise select an instruction without accumulator
2723 bool IsAccum = !(OpIsZero(4) && OpIsZero(5));
2724
2725 const uint16_t *Opcodes = IsUnsigned ? OpcodesU : OpcodesS;
2726 if (IsSub)
2727 Opcodes += 4 * Stride;
2728 if (IsExchange)
2729 Opcodes += 2 * Stride;
2730 if (IsAccum)
2731 Opcodes += Stride;
2732 uint16_t Opcode = Opcodes[TySize];
2733
2734 SDLoc Loc(N);
2735 SmallVector<SDValue, 8> Ops;
2736 // Push the accumulator operands, if they are used
2737 if (IsAccum) {
2738 Ops.push_back(N->getOperand(4));
2739 Ops.push_back(N->getOperand(5));
2740 }
2741 // Push the two vector operands
2742 Ops.push_back(N->getOperand(6));
2743 Ops.push_back(N->getOperand(7));
2744
2745 if (Predicated)
2746 AddMVEPredicateToOps(Ops, Loc, N->getOperand(8));
2747 else
2748 AddEmptyMVEPredicateToOps(Ops, Loc);
2749
2750 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
2751}
2752
2753void ARMDAGToDAGISel::SelectMVE_VMLLDAV(SDNode *N, bool Predicated,
2754 const uint16_t *OpcodesS,
2755 const uint16_t *OpcodesU) {
2756 EVT VecTy = N->getOperand(6).getValueType();
2757 size_t SizeIndex;
2758 switch (VecTy.getVectorElementType().getSizeInBits()) {
2759 case 16:
2760 SizeIndex = 0;
2761 break;
2762 case 32:
2763 SizeIndex = 1;
2764 break;
2765 default:
2766 llvm_unreachable("bad vector element size")::llvm::llvm_unreachable_internal("bad vector element size", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2766);
2767 }
2768
2769 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 2, SizeIndex);
2770}
2771
2772void ARMDAGToDAGISel::SelectMVE_VRMLLDAVH(SDNode *N, bool Predicated,
2773 const uint16_t *OpcodesS,
2774 const uint16_t *OpcodesU) {
2775 assert((static_cast <bool> (N->getOperand(6).getValueType()
.getVectorElementType().getSizeInBits() == 32 && "bad vector element size"
) ? void (0) : __assert_fail ("N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == 32 && \"bad vector element size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
2776 N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() ==(static_cast <bool> (N->getOperand(6).getValueType()
.getVectorElementType().getSizeInBits() == 32 && "bad vector element size"
) ? void (0) : __assert_fail ("N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == 32 && \"bad vector element size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
2777 32 &&(static_cast <bool> (N->getOperand(6).getValueType()
.getVectorElementType().getSizeInBits() == 32 && "bad vector element size"
) ? void (0) : __assert_fail ("N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == 32 && \"bad vector element size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__))
2778 "bad vector element size")(static_cast <bool> (N->getOperand(6).getValueType()
.getVectorElementType().getSizeInBits() == 32 && "bad vector element size"
) ? void (0) : __assert_fail ("N->getOperand(6).getValueType().getVectorElementType().getSizeInBits() == 32 && \"bad vector element size\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2778, __extension__ __PRETTY_FUNCTION__));
2779 SelectBaseMVE_VMLLDAV(N, Predicated, OpcodesS, OpcodesU, 1, 0);
2780}
2781
2782void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs,
2783 const uint16_t *const *Opcodes,
2784 bool HasWriteback) {
2785 EVT VT = N->getValueType(0);
2786 SDLoc Loc(N);
2787
2788 const uint16_t *OurOpcodes;
2789 switch (VT.getVectorElementType().getSizeInBits()) {
2790 case 8:
2791 OurOpcodes = Opcodes[0];
2792 break;
2793 case 16:
2794 OurOpcodes = Opcodes[1];
2795 break;
2796 case 32:
2797 OurOpcodes = Opcodes[2];
2798 break;
2799 default:
2800 llvm_unreachable("bad vector element size in SelectMVE_VLD")::llvm::llvm_unreachable_internal("bad vector element size in SelectMVE_VLD"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2800);
2801 }
2802
2803 EVT DataTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, NumVecs * 2);
2804 SmallVector<EVT, 4> ResultTys = {DataTy, MVT::Other};
2805 unsigned PtrOperand = HasWriteback ? 1 : 2;
2806
2807 auto Data = SDValue(
2808 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, Loc, DataTy), 0);
2809 SDValue Chain = N->getOperand(0);
2810 // Add a MVE_VLDn instruction for each Vec, except the last
2811 for (unsigned Stage = 0; Stage < NumVecs - 1; ++Stage) {
2812 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain};
2813 auto LoadInst =
2814 CurDAG->getMachineNode(OurOpcodes[Stage], Loc, ResultTys, Ops);
2815 Data = SDValue(LoadInst, 0);
2816 Chain = SDValue(LoadInst, 1);
2817 transferMemOperands(N, LoadInst);
2818 }
2819 // The last may need a writeback on it
2820 if (HasWriteback)
2821 ResultTys = {DataTy, MVT::i32, MVT::Other};
2822 SDValue Ops[] = {Data, N->getOperand(PtrOperand), Chain};
2823 auto LoadInst =
2824 CurDAG->getMachineNode(OurOpcodes[NumVecs - 1], Loc, ResultTys, Ops);
2825 transferMemOperands(N, LoadInst);
2826
2827 unsigned i;
2828 for (i = 0; i < NumVecs; i++)
2829 ReplaceUses(SDValue(N, i),
2830 CurDAG->getTargetExtractSubreg(ARM::qsub_0 + i, Loc, VT,
2831 SDValue(LoadInst, 0)));
2832 if (HasWriteback)
2833 ReplaceUses(SDValue(N, i++), SDValue(LoadInst, 1));
2834 ReplaceUses(SDValue(N, i), SDValue(LoadInst, HasWriteback ? 2 : 1));
2835 CurDAG->RemoveDeadNode(N);
2836}
2837
2838void ARMDAGToDAGISel::SelectMVE_VxDUP(SDNode *N, const uint16_t *Opcodes,
2839 bool Wrapping, bool Predicated) {
2840 EVT VT = N->getValueType(0);
2841 SDLoc Loc(N);
2842
2843 uint16_t Opcode;
2844 switch (VT.getScalarSizeInBits()) {
2845 case 8:
2846 Opcode = Opcodes[0];
2847 break;
2848 case 16:
2849 Opcode = Opcodes[1];
2850 break;
2851 case 32:
2852 Opcode = Opcodes[2];
2853 break;
2854 default:
2855 llvm_unreachable("bad vector element size in SelectMVE_VxDUP")::llvm::llvm_unreachable_internal("bad vector element size in SelectMVE_VxDUP"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2855);
2856 }
2857
2858 SmallVector<SDValue, 8> Ops;
2859 unsigned OpIdx = 1;
2860
2861 SDValue Inactive;
2862 if (Predicated)
2863 Inactive = N->getOperand(OpIdx++);
2864
2865 Ops.push_back(N->getOperand(OpIdx++)); // base
2866 if (Wrapping)
2867 Ops.push_back(N->getOperand(OpIdx++)); // limit
2868
2869 SDValue ImmOp = N->getOperand(OpIdx++); // step
2870 int ImmValue = cast<ConstantSDNode>(ImmOp)->getZExtValue();
2871 Ops.push_back(getI32Imm(ImmValue, Loc));
2872
2873 if (Predicated)
2874 AddMVEPredicateToOps(Ops, Loc, N->getOperand(OpIdx), Inactive);
2875 else
2876 AddEmptyMVEPredicateToOps(Ops, Loc, N->getValueType(0));
2877
2878 CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
2879}
2880
2881void ARMDAGToDAGISel::SelectCDE_CXxD(SDNode *N, uint16_t Opcode,
2882 size_t NumExtraOps, bool HasAccum) {
2883 bool IsBigEndian = CurDAG->getDataLayout().isBigEndian();
2884 SDLoc Loc(N);
2885 SmallVector<SDValue, 8> Ops;
2886
2887 unsigned OpIdx = 1;
2888
2889 // Convert and append the immediate operand designating the coprocessor.
2890 SDValue ImmCorpoc = N->getOperand(OpIdx++);
2891 uint32_t ImmCoprocVal = cast<ConstantSDNode>(ImmCorpoc)->getZExtValue();
2892 Ops.push_back(getI32Imm(ImmCoprocVal, Loc));
2893
2894 // For accumulating variants copy the low and high order parts of the
2895 // accumulator into a register pair and add it to the operand vector.
2896 if (HasAccum) {
2897 SDValue AccLo = N->getOperand(OpIdx++);
2898 SDValue AccHi = N->getOperand(OpIdx++);
2899 if (IsBigEndian)
2900 std::swap(AccLo, AccHi);
2901 Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, AccLo, AccHi), 0));
2902 }
2903
2904 // Copy extra operands as-is.
2905 for (size_t I = 0; I < NumExtraOps; I++)
2906 Ops.push_back(N->getOperand(OpIdx++));
2907
2908 // Convert and append the immediate operand
2909 SDValue Imm = N->getOperand(OpIdx);
2910 uint32_t ImmVal = cast<ConstantSDNode>(Imm)->getZExtValue();
2911 Ops.push_back(getI32Imm(ImmVal, Loc));
2912
2913 // Accumulating variants are IT-predicable, add predicate operands.
2914 if (HasAccum) {
2915 SDValue Pred = getAL(CurDAG, Loc);
2916 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2917 Ops.push_back(Pred);
2918 Ops.push_back(PredReg);
2919 }
2920
2921 // Create the CDE intruction
2922 SDNode *InstrNode = CurDAG->getMachineNode(Opcode, Loc, MVT::Untyped, Ops);
2923 SDValue ResultPair = SDValue(InstrNode, 0);
2924
2925 // The original intrinsic had two outputs, and the output of the dual-register
2926 // CDE instruction is a register pair. We need to extract the two subregisters
2927 // and replace all uses of the original outputs with the extracted
2928 // subregisters.
2929 uint16_t SubRegs[2] = {ARM::gsub_0, ARM::gsub_1};
2930 if (IsBigEndian)
2931 std::swap(SubRegs[0], SubRegs[1]);
2932
2933 for (size_t ResIdx = 0; ResIdx < 2; ResIdx++) {
2934 if (SDValue(N, ResIdx).use_empty())
2935 continue;
2936 SDValue SubReg = CurDAG->getTargetExtractSubreg(SubRegs[ResIdx], Loc,
2937 MVT::i32, ResultPair);
2938 ReplaceUses(SDValue(N, ResIdx), SubReg);
2939 }
2940
2941 CurDAG->RemoveDeadNode(N);
2942}
2943
2944void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2945 bool isUpdating, unsigned NumVecs,
2946 const uint16_t *DOpcodes,
2947 const uint16_t *QOpcodes0,
2948 const uint16_t *QOpcodes1) {
2949 assert(Subtarget->hasNEON())(static_cast <bool> (Subtarget->hasNEON()) ? void (0
) : __assert_fail ("Subtarget->hasNEON()", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2949, __extension__ __PRETTY_FUNCTION__));
2950 assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range")(static_cast <bool> (NumVecs >= 1 && NumVecs
<= 4 && "VLDDup NumVecs out-of-range") ? void (0)
: __assert_fail ("NumVecs >= 1 && NumVecs <= 4 && \"VLDDup NumVecs out-of-range\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2950, __extension__ __PRETTY_FUNCTION__));
2951 SDLoc dl(N);
2952
2953 SDValue MemAddr, Align;
2954 unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2955 if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2956 return;
2957
2958 SDValue Chain = N->getOperand(0);
2959 EVT VT = N->getValueType(0);
2960 bool is64BitVector = VT.is64BitVector();
2961
2962 unsigned Alignment = 0;
2963 if (NumVecs != 3) {
2964 Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2965 unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2966 if (Alignment > NumBytes)
2967 Alignment = NumBytes;
2968 if (Alignment < 8 && Alignment < NumBytes)
2969 Alignment = 0;
2970 // Alignment must be a power of two; make sure of that.
2971 Alignment = (Alignment & -Alignment);
2972 if (Alignment == 1)
2973 Alignment = 0;
2974 }
2975 Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2976
2977 unsigned OpcodeIndex;
2978 switch (VT.getSimpleVT().SimpleTy) {
2979 default: llvm_unreachable("unhandled vld-dup type")::llvm::llvm_unreachable_internal("unhandled vld-dup type", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 2979);
2980 case MVT::v8i8:
2981 case MVT::v16i8: OpcodeIndex = 0; break;
2982 case MVT::v4i16:
2983 case MVT::v8i16:
2984 case MVT::v4f16:
2985 case MVT::v8f16:
2986 case MVT::v4bf16:
2987 case MVT::v8bf16:
2988 OpcodeIndex = 1; break;
2989 case MVT::v2f32:
2990 case MVT::v2i32:
2991 case MVT::v4f32:
2992 case MVT::v4i32: OpcodeIndex = 2; break;
2993 case MVT::v1f64:
2994 case MVT::v1i64: OpcodeIndex = 3; break;
2995 }
2996
2997 unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2998 if (!is64BitVector)
2999 ResTyElts *= 2;
3000 EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
3001
3002 std::vector<EVT> ResTys;
3003 ResTys.push_back(ResTy);
3004 if (isUpdating)
3005 ResTys.push_back(MVT::i32);
3006 ResTys.push_back(MVT::Other);
3007
3008 SDValue Pred = getAL(CurDAG, dl);
3009 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3010
3011 SmallVector<SDValue, 6> Ops;
3012 Ops.push_back(MemAddr);
3013 Ops.push_back(Align);
3014 unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex]
3015 : (NumVecs == 1) ? QOpcodes0[OpcodeIndex]
3016 : QOpcodes1[OpcodeIndex];
3017 if (isUpdating) {
3018 SDValue Inc = N->getOperand(2);
3019 bool IsImmUpdate =
3020 isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
3021 if (IsImmUpdate) {
3022 if (!isVLDfixed(Opc))
3023 Ops.push_back(Reg0);
3024 } else {
3025 if (isVLDfixed(Opc))
3026 Opc = getVLDSTRegisterUpdateOpcode(Opc);
3027 Ops.push_back(Inc);
3028 }
3029 }
3030 if (is64BitVector || NumVecs == 1) {
3031 // Double registers and VLD1 quad registers are directly supported.
3032 } else if (NumVecs == 2) {
3033 const SDValue OpsA[] = {MemAddr, Align, Pred, Reg0, Chain};
3034 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy,
3035 MVT::Other, OpsA);
3036 Chain = SDValue(VLdA, 1);
3037 } else {
3038 SDValue ImplDef = SDValue(
3039 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
3040 const SDValue OpsA[] = {MemAddr, Align, ImplDef, Pred, Reg0, Chain};
3041 SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, ResTy,
3042 MVT::Other, OpsA);
3043 Ops.push_back(SDValue(VLdA, 0));
3044 Chain = SDValue(VLdA, 1);
3045 }
3046
3047 Ops.push_back(Pred);
3048 Ops.push_back(Reg0);
3049 Ops.push_back(Chain);
3050
3051 SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
3052
3053 // Transfer memoperands.
3054 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3055 CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
3056
3057 // Extract the subregisters.
3058 if (NumVecs == 1) {
3059 ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
3060 } else {
3061 SDValue SuperReg = SDValue(VLdDup, 0);
3062 static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
3063 unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
3064 for (unsigned Vec = 0; Vec != NumVecs; ++Vec) {
3065 ReplaceUses(SDValue(N, Vec),
3066 CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
3067 }
3068 }
3069 ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
3070 if (isUpdating)
3071 ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
3072 CurDAG->RemoveDeadNode(N);
3073}
3074
3075bool ARMDAGToDAGISel::tryInsertVectorElt(SDNode *N) {
3076 if (!Subtarget->hasMVEIntegerOps())
3077 return false;
3078
3079 SDLoc dl(N);
3080
3081 // We are trying to use VMOV/VMOVX/VINS to more efficiently lower insert and
3082 // extracts of v8f16 and v8i16 vectors. Check that we have two adjacent
3083 // inserts of the correct type:
3084 SDValue Ins1 = SDValue(N, 0);
3085 SDValue Ins2 = N->getOperand(0);
3086 EVT VT = Ins1.getValueType();
3087 if (Ins2.getOpcode() != ISD::INSERT_VECTOR_ELT || !Ins2.hasOneUse() ||
3088 !isa<ConstantSDNode>(Ins1.getOperand(2)) ||
3089 !isa<ConstantSDNode>(Ins2.getOperand(2)) ||
3090 (VT != MVT::v8f16 && VT != MVT::v8i16) || (Ins2.getValueType() != VT))
3091 return false;
3092
3093 unsigned Lane1 = Ins1.getConstantOperandVal(2);
3094 unsigned Lane2 = Ins2.getConstantOperandVal(2);
3095 if (Lane2 % 2 != 0 || Lane1 != Lane2 + 1)
3096 return false;
3097
3098 // If the inserted values will be able to use T/B already, leave it to the
3099 // existing tablegen patterns. For example VCVTT/VCVTB.
3100 SDValue Val1 = Ins1.getOperand(1);
3101 SDValue Val2 = Ins2.getOperand(1);
3102 if (Val1.getOpcode() == ISD::FP_ROUND || Val2.getOpcode() == ISD::FP_ROUND)
3103 return false;
3104
3105 // Check if the inserted values are both extracts.
3106 if ((Val1.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
3107 Val1.getOpcode() == ARMISD::VGETLANEu) &&
3108 (Val2.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
3109 Val2.getOpcode() == ARMISD::VGETLANEu) &&
3110 isa<ConstantSDNode>(Val1.getOperand(1)) &&
3111 isa<ConstantSDNode>(Val2.getOperand(1)) &&
3112 (Val1.getOperand(0).getValueType() == MVT::v8f16 ||
3113 Val1.getOperand(0).getValueType() == MVT::v8i16) &&
3114 (Val2.getOperand(0).getValueType() == MVT::v8f16 ||
3115 Val2.getOperand(0).getValueType() == MVT::v8i16)) {
3116 unsigned ExtractLane1 = Val1.getConstantOperandVal(1);
3117 unsigned ExtractLane2 = Val2.getConstantOperandVal(1);
3118
3119 // If the two extracted lanes are from the same place and adjacent, this
3120 // simplifies into a f32 lane move.
3121 if (Val1.getOperand(0) == Val2.getOperand(0) && ExtractLane2 % 2 == 0 &&
3122 ExtractLane1 == ExtractLane2 + 1) {
3123 SDValue NewExt = CurDAG->getTargetExtractSubreg(
3124 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val1.getOperand(0));
3125 SDValue NewIns = CurDAG->getTargetInsertSubreg(
3126 ARM::ssub_0 + Lane2 / 2, dl, VT, Ins2.getOperand(0),
3127 NewExt);
3128 ReplaceUses(Ins1, NewIns);
3129 return true;
3130 }
3131
3132 // Else v8i16 pattern of an extract and an insert, with a optional vmovx for
3133 // extracting odd lanes.
3134 if (VT == MVT::v8i16) {
3135 SDValue Inp1 = CurDAG->getTargetExtractSubreg(
3136 ARM::ssub_0 + ExtractLane1 / 2, dl, MVT::f32, Val1.getOperand(0));
3137 SDValue Inp2 = CurDAG->getTargetExtractSubreg(
3138 ARM::ssub_0 + ExtractLane2 / 2, dl, MVT::f32, Val2.getOperand(0));
3139 if (ExtractLane1 % 2 != 0)
3140 Inp1 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp1), 0);
3141 if (ExtractLane2 % 2 != 0)
3142 Inp2 = SDValue(CurDAG->getMachineNode(ARM::VMOVH, dl, MVT::f32, Inp2), 0);
3143 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Inp2, Inp1);
3144 SDValue NewIns =
3145 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3146 Ins2.getOperand(0), SDValue(VINS, 0));
3147 ReplaceUses(Ins1, NewIns);
3148 return true;
3149 }
3150 }
3151
3152 // The inserted values are not extracted - if they are f16 then insert them
3153 // directly using a VINS.
3154 if (VT == MVT::v8f16) {
3155 SDNode *VINS = CurDAG->getMachineNode(ARM::VINSH, dl, MVT::f32, Val2, Val1);
3156 SDValue NewIns =
3157 CurDAG->getTargetInsertSubreg(ARM::ssub_0 + Lane2 / 2, dl, MVT::v4f32,
3158 Ins2.getOperand(0), SDValue(VINS, 0));
3159 ReplaceUses(Ins1, NewIns);
3160 return true;
3161 }
3162
3163 return false;
3164}
3165
3166bool ARMDAGToDAGISel::transformFixedFloatingPointConversion(SDNode *N,
3167 SDNode *FMul,
3168 bool IsUnsigned,
3169 bool FixedToFloat) {
3170 auto Type = N->getValueType(0);
3171 unsigned ScalarBits = Type.getScalarSizeInBits();
3172 if (ScalarBits > 32)
3173 return false;
3174
3175 SDNodeFlags FMulFlags = FMul->getFlags();
3176 // The fixed-point vcvt and vcvt+vmul are not always equivalent if inf is
3177 // allowed in 16 bit unsigned floats
3178 if (ScalarBits == 16 && !FMulFlags.hasNoInfs() && IsUnsigned)
3179 return false;
3180
3181 SDValue ImmNode = FMul->getOperand(1);
3182 SDValue VecVal = FMul->getOperand(0);
3183 if (VecVal->getOpcode() == ISD::UINT_TO_FP ||
3184 VecVal->getOpcode() == ISD::SINT_TO_FP)
3185 VecVal = VecVal->getOperand(0);
3186
3187 if (VecVal.getValueType().getScalarSizeInBits() != ScalarBits)
3188 return false;
3189
3190 if (ImmNode.getOpcode() == ISD::BITCAST) {
3191 if (ImmNode.getValueType().getScalarSizeInBits() != ScalarBits)
3192 return false;
3193 ImmNode = ImmNode.getOperand(0);
3194 }
3195
3196 if (ImmNode.getValueType().getScalarSizeInBits() != ScalarBits)
3197 return false;
3198
3199 APFloat ImmAPF(0.0f);
3200 switch (ImmNode.getOpcode()) {
3201 case ARMISD::VMOVIMM:
3202 case ARMISD::VDUP: {
3203 if (!isa<ConstantSDNode>(ImmNode.getOperand(0)))
3204 return false;
3205 unsigned Imm = ImmNode.getConstantOperandVal(0);
3206 if (ImmNode.getOpcode() == ARMISD::VMOVIMM)
3207 Imm = ARM_AM::decodeVMOVModImm(Imm, ScalarBits);
3208 ImmAPF =
3209 APFloat(ScalarBits == 32 ? APFloat::IEEEsingle() : APFloat::IEEEhalf(),
3210 APInt(ScalarBits, Imm));
3211 break;
3212 }
3213 case ARMISD::VMOVFPIMM: {
3214 ImmAPF = APFloat(ARM_AM::getFPImmFloat(ImmNode.getConstantOperandVal(0)));
3215 break;
3216 }
3217 default:
3218 return false;
3219 }
3220
3221 // Where n is the number of fractional bits, multiplying by 2^n will convert
3222 // from float to fixed and multiplying by 2^-n will convert from fixed to
3223 // float. Taking log2 of the factor (after taking the inverse in the case of
3224 // float to fixed) will give n.
3225 APFloat ToConvert = ImmAPF;
3226 if (FixedToFloat) {
3227 if (!ImmAPF.getExactInverse(&ToConvert))
3228 return false;
3229 }
3230 APSInt Converted(64, 0);
3231 bool IsExact;
3232 ToConvert.convertToInteger(Converted, llvm::RoundingMode::NearestTiesToEven,
3233 &IsExact);
3234 if (!IsExact || !Converted.isPowerOf2())
3235 return false;
3236
3237 unsigned FracBits = Converted.logBase2();
3238 if (FracBits > ScalarBits)
3239 return false;
3240
3241 SmallVector<SDValue, 3> Ops{
3242 VecVal, CurDAG->getConstant(FracBits, SDLoc(N), MVT::i32)};
3243 AddEmptyMVEPredicateToOps(Ops, SDLoc(N), Type);
3244
3245 unsigned int Opcode;
3246 switch (ScalarBits) {
3247 case 16:
3248 if (FixedToFloat)
3249 Opcode = IsUnsigned ? ARM::MVE_VCVTf16u16_fix : ARM::MVE_VCVTf16s16_fix;
3250 else
3251 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3252 break;
3253 case 32:
3254 if (FixedToFloat)
3255 Opcode = IsUnsigned ? ARM::MVE_VCVTf32u32_fix : ARM::MVE_VCVTf32s32_fix;
3256 else
3257 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3258 break;
3259 default:
3260 llvm_unreachable("unexpected number of scalar bits")::llvm::llvm_unreachable_internal("unexpected number of scalar bits"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3260);
3261 break;
3262 }
3263
3264 ReplaceNode(N, CurDAG->getMachineNode(Opcode, SDLoc(N), Type, Ops));
3265 return true;
3266}
3267
3268bool ARMDAGToDAGISel::tryFP_TO_INT(SDNode *N, SDLoc dl) {
3269 // Transform a floating-point to fixed-point conversion to a VCVT
3270 if (!Subtarget->hasMVEFloatOps())
1
Assuming the condition is false
→
2
←
Taking false branch
→
3271 return false;
3272 EVT Type = N->getValueType(0);
3273 if (!Type.isVector())
3
←
Calling 'EVT::isVector'
→
6
←
Returning from 'EVT::isVector'
→
7
←
Assuming the condition is false
→
8
←
Taking false branch
→
3274 return false;
3275 unsigned int ScalarBits = Type.getScalarSizeInBits();
3276
3277 bool IsUnsigned = N->getOpcode() == ISD::FP_TO_UINT;
9
←
Assuming the condition is false
→
3278 SDNode *Node = N->getOperand(0).getNode();
3279
3280 // floating-point to fixed-point with one fractional bit gets turned into an
3281 // FP_TO_[U|S]INT(FADD (x, x)) rather than an FP_TO_[U|S]INT(FMUL (x, y))
3282 if (Node->getOpcode() == ISD::FADD) {
10
←
Assuming the condition is true
→
11
←
Taking true branch
→
3283 if (Node->getOperand(0) != Node->getOperand(1))
12
←
Calling 'SDValue::operator!='
→
19
←
Returning from 'SDValue::operator!='
→
20
←
Taking false branch
→
3284 return false;
3285 SDNodeFlags Flags = Node->getFlags();
3286 // The fixed-point vcvt and vcvt+vmul are not always equivalent if inf is
3287 // allowed in 16 bit unsigned floats
3288 if (ScalarBits == 16 && !Flags.hasNoInfs() && IsUnsigned)
21
←
Assuming 'ScalarBits' is not equal to 16
→
3289 return false;
3290
3291 unsigned Opcode;
22
←
'Opcode' declared without an initial value
→
3292 switch (ScalarBits) {
23
←
'Default' branch taken. Execution continues on line 3300
→
3293 case 16:
3294 Opcode = IsUnsigned ? ARM::MVE_VCVTu16f16_fix : ARM::MVE_VCVTs16f16_fix;
3295 break;
3296 case 32:
3297 Opcode = IsUnsigned ? ARM::MVE_VCVTu32f32_fix : ARM::MVE_VCVTs32f32_fix;
3298 break;
3299 }
3300 SmallVector<SDValue, 3> Ops{Node->getOperand(0),
3301 CurDAG->getConstant(1, dl, MVT::i32)};
3302 AddEmptyMVEPredicateToOps(Ops, dl, Type);
3303
3304 ReplaceNode(N, CurDAG->getMachineNode(Opcode, dl, Type, Ops));
24
←
1st function call argument is an uninitialized value
3305 return true;
3306 }
3307
3308 if (Node->getOpcode() != ISD::FMUL)
3309 return false;
3310
3311 return transformFixedFloatingPointConversion(N, Node, IsUnsigned, false);
3312}
3313
3314bool ARMDAGToDAGISel::tryFMULFixed(SDNode *N, SDLoc dl) {
3315 // Transform a fixed-point to floating-point conversion to a VCVT
3316 if (!Subtarget->hasMVEFloatOps())
3317 return false;
3318 auto Type = N->getValueType(0);
3319 if (!Type.isVector())
3320 return false;
3321
3322 auto LHS = N->getOperand(0);
3323 if (LHS.getOpcode() != ISD::SINT_TO_FP && LHS.getOpcode() != ISD::UINT_TO_FP)
3324 return false;
3325
3326 return transformFixedFloatingPointConversion(
3327 N, N, LHS.getOpcode() == ISD::UINT_TO_FP, true);
3328}
3329
3330bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
3331 if (!Subtarget->hasV6T2Ops())
3332 return false;
3333
3334 unsigned Opc = isSigned
3335 ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
3336 : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
3337 SDLoc dl(N);
3338
3339 // For unsigned extracts, check for a shift right and mask
3340 unsigned And_imm = 0;
3341 if (N->getOpcode() == ISD::AND) {
3342 if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
3343
3344 // The immediate is a mask of the low bits iff imm & (imm+1) == 0
3345 if (And_imm & (And_imm + 1))
3346 return false;
3347
3348 unsigned Srl_imm = 0;
3349 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL,
3350 Srl_imm)) {
3351 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!")(static_cast <bool> (Srl_imm > 0 && Srl_imm <
32 && "bad amount in shift node!") ? void (0) : __assert_fail
("Srl_imm > 0 && Srl_imm < 32 && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3351, __extension__ __PRETTY_FUNCTION__));
3352
3353 // Mask off the unnecessary bits of the AND immediate; normally
3354 // DAGCombine will do this, but that might not happen if
3355 // targetShrinkDemandedConstant chooses a different immediate.
3356 And_imm &= -1U >> Srl_imm;
3357
3358 // Note: The width operand is encoded as width-1.
3359 unsigned Width = countTrailingOnes(And_imm) - 1;
3360 unsigned LSB = Srl_imm;
3361
3362 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3363
3364 if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
3365 // It's cheaper to use a right shift to extract the top bits.
3366 if (Subtarget->isThumb()) {
3367 Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
3368 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3369 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3370 getAL(CurDAG, dl), Reg0, Reg0 };
3371 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3372 return true;
3373 }
3374
3375 // ARM models shift instructions as MOVsi with shifter operand.
3376 ARM_AM::ShiftOpc ShOpcVal = ARM_AM::getShiftOpcForNode(ISD::SRL);
3377 SDValue ShOpc =
3378 CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
3379 MVT::i32);
3380 SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
3381 getAL(CurDAG, dl), Reg0, Reg0 };
3382 CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
3383 return true;
3384 }
3385
3386 assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx")(static_cast <bool> (LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx") ? void (0) : __assert_fail
("LSB + Width + 1 <= 32 && \"Shouldn't create an invalid ubfx\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3386, __extension__ __PRETTY_FUNCTION__));
3387 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3388 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3389 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3390 getAL(CurDAG, dl), Reg0 };
3391 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3392 return true;
3393 }
3394 }
3395 return false;
3396 }
3397
3398 // Otherwise, we're looking for a shift of a shift
3399 unsigned Shl_imm = 0;
3400 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
3401 assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!")(static_cast <bool> (Shl_imm > 0 && Shl_imm <
32 && "bad amount in shift node!") ? void (0) : __assert_fail
("Shl_imm > 0 && Shl_imm < 32 && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3401, __extension__ __PRETTY_FUNCTION__));
3402 unsigned Srl_imm = 0;
3403 if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
3404 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!")(static_cast <bool> (Srl_imm > 0 && Srl_imm <
32 && "bad amount in shift node!") ? void (0) : __assert_fail
("Srl_imm > 0 && Srl_imm < 32 && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3404, __extension__ __PRETTY_FUNCTION__));
3405 // Note: The width operand is encoded as width-1.
3406 unsigned Width = 32 - Srl_imm - 1;
3407 int LSB = Srl_imm - Shl_imm;
3408 if (LSB < 0)
3409 return false;
3410 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3411 assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx")(static_cast <bool> (LSB + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx") ? void (0) : __assert_fail
("LSB + Width + 1 <= 32 && \"Shouldn't create an invalid ubfx\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3411, __extension__ __PRETTY_FUNCTION__));
3412 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3413 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3414 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3415 getAL(CurDAG, dl), Reg0 };
3416 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3417 return true;
3418 }
3419 }
3420
3421 // Or we are looking for a shift of an and, with a mask operand
3422 if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
3423 isShiftedMask_32(And_imm)) {
3424 unsigned Srl_imm = 0;
3425 unsigned LSB = countTrailingZeros(And_imm);
3426 // Shift must be the same as the ands lsb
3427 if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
3428 assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!")(static_cast <bool> (Srl_imm > 0 && Srl_imm <
32 && "bad amount in shift node!") ? void (0) : __assert_fail
("Srl_imm > 0 && Srl_imm < 32 && \"bad amount in shift node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3428, __extension__ __PRETTY_FUNCTION__));
3429 unsigned MSB = 31 - countLeadingZeros(And_imm);
3430 // Note: The width operand is encoded as width-1.
3431 unsigned Width = MSB - LSB;
3432 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3433 assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx")(static_cast <bool> (Srl_imm + Width + 1 <= 32 &&
"Shouldn't create an invalid ubfx") ? void (0) : __assert_fail
("Srl_imm + Width + 1 <= 32 && \"Shouldn't create an invalid ubfx\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3433, __extension__ __PRETTY_FUNCTION__));
3434 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3435 CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
3436 CurDAG->getTargetConstant(Width, dl, MVT::i32),
3437 getAL(CurDAG, dl), Reg0 };
3438 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3439 return true;
3440 }
3441 }
3442
3443 if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
3444 unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
3445 unsigned LSB = 0;
3446 if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
3447 !isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRA, LSB))
3448 return false;
3449
3450 if (LSB + Width > 32)
3451 return false;
3452
3453 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3454 assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx")(static_cast <bool> (LSB + Width <= 32 && "Shouldn't create an invalid ubfx"
) ? void (0) : __assert_fail ("LSB + Width <= 32 && \"Shouldn't create an invalid ubfx\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3454, __extension__ __PRETTY_FUNCTION__));
3455 SDValue Ops[] = { N->getOperand(0).getOperand(0),
3456 CurDAG->getTargetConstant(LSB, dl, MVT::i32),
3457 CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
3458 getAL(CurDAG, dl), Reg0 };
3459 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3460 return true;
3461 }
3462
3463 return false;
3464}
3465
3466/// Target-specific DAG combining for ISD::XOR.
3467/// Target-independent combining lowers SELECT_CC nodes of the form
3468/// select_cc setg[ge] X, 0, X, -X
3469/// select_cc setgt X, -1, X, -X
3470/// select_cc setl[te] X, 0, -X, X
3471/// select_cc setlt X, 1, -X, X
3472/// which represent Integer ABS into:
3473/// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
3474/// ARM instruction selection detects the latter and matches it to
3475/// ARM::ABS or ARM::t2ABS machine node.
3476bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
3477 SDValue XORSrc0 = N->getOperand(0);
3478 SDValue XORSrc1 = N->getOperand(1);
3479 EVT VT = N->getValueType(0);
3480
3481 if (Subtarget->isThumb1Only())
3482 return false;
3483
3484 if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
3485 return false;
3486
3487 SDValue ADDSrc0 = XORSrc0.getOperand(0);
3488 SDValue ADDSrc1 = XORSrc0.getOperand(1);
3489 SDValue SRASrc0 = XORSrc1.getOperand(0);
3490 SDValue SRASrc1 = XORSrc1.getOperand(1);
3491 ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
3492 EVT XType = SRASrc0.getValueType();
3493 unsigned Size = XType.getSizeInBits() - 1;
3494
3495 if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
3496 XType.isInteger() && SRAConstant != nullptr &&
3497 Size == SRAConstant->getZExtValue()) {
3498 unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
3499 CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
3500 return true;
3501 }
3502
3503 return false;
3504}
3505
3506/// We've got special pseudo-instructions for these
3507void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
3508 unsigned Opcode;
3509 EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
3510 if (MemTy == MVT::i8)
3511 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_8 : ARM::CMP_SWAP_8;
3512 else if (MemTy == MVT::i16)
3513 Opcode = Subtarget->isThumb() ? ARM::tCMP_SWAP_16 : ARM::CMP_SWAP_16;
3514 else if (MemTy == MVT::i32)
3515 Opcode = ARM::CMP_SWAP_32;
3516 else
3517 llvm_unreachable("Unknown AtomicCmpSwap type")::llvm::llvm_unreachable_internal("Unknown AtomicCmpSwap type"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3517);
3518
3519 SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
3520 N->getOperand(0)};
3521 SDNode *CmpSwap = CurDAG->getMachineNode(
3522 Opcode, SDLoc(N),
3523 CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
3524
3525 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
3526 CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
3527
3528 ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
3529 ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
3530 CurDAG->RemoveDeadNode(N);
3531}
3532
3533static Optional<std::pair<unsigned, unsigned>>
3534getContiguousRangeOfSetBits(const APInt &A) {
3535 unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
3536 unsigned LastOne = A.countTrailingZeros();
3537 if (A.countPopulation() != (FirstOne - LastOne + 1))
3538 return Optional<std::pair<unsigned,unsigned>>();
3539 return std::make_pair(FirstOne, LastOne);
3540}
3541
3542void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
3543 assert(N->getOpcode() == ARMISD::CMPZ)(static_cast <bool> (N->getOpcode() == ARMISD::CMPZ)
? void (0) : __assert_fail ("N->getOpcode() == ARMISD::CMPZ"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3543, __extension__ __PRETTY_FUNCTION__));
3544 SwitchEQNEToPLMI = false;
3545
3546 if (!Subtarget->isThumb())
3547 // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
3548 // LSR don't exist as standalone instructions - they need the barrel shifter.
3549 return;
3550
3551 // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
3552 SDValue And = N->getOperand(0);
3553 if (!And->hasOneUse())
3554 return;
3555
3556 SDValue Zero = N->getOperand(1);
3557 if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isZero() ||
3558 And->getOpcode() != ISD::AND)
3559 return;
3560 SDValue X = And.getOperand(0);
3561 auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
3562
3563 if (!C)
3564 return;
3565 auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
3566 if (!Range)
3567 return;
3568
3569 // There are several ways to lower this:
3570 SDNode *NewN;
3571 SDLoc dl(N);
3572
3573 auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
3574 if (Subtarget->isThumb2()) {
3575 Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
3576 SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3577 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3578 CurDAG->getRegister(0, MVT::i32) };
3579 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3580 } else {
3581 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
3582 CurDAG->getTargetConstant(Imm, dl, MVT::i32),
3583 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3584 return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3585 }
3586 };
3587
3588 if (Range->second == 0) {
3589 // 1. Mask includes the LSB -> Simply shift the top N bits off
3590 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3591 ReplaceNode(And.getNode(), NewN);
3592 } else if (Range->first == 31) {
3593 // 2. Mask includes the MSB -> Simply shift the bottom N bits off
3594 NewN = EmitShift(ARM::tLSRri, X, Range->second);
3595 ReplaceNode(And.getNode(), NewN);
3596 } else if (Range->first == Range->second) {
3597 // 3. Only one bit is set. We can shift this into the sign bit and use a
3598 // PL/MI comparison.
3599 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3600 ReplaceNode(And.getNode(), NewN);
3601
3602 SwitchEQNEToPLMI = true;
3603 } else if (!Subtarget->hasV6T2Ops()) {
3604 // 4. Do a double shift to clear bottom and top bits, but only in
3605 // thumb-1 mode as in thumb-2 we can use UBFX.
3606 NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
3607 NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
3608 Range->second + (31 - Range->first));
3609 ReplaceNode(And.getNode(), NewN);
3610 }
3611
3612}
3613
3614void ARMDAGToDAGISel::Select(SDNode *N) {
3615 SDLoc dl(N);
3616
3617 if (N->isMachineOpcode()) {
3618 N->setNodeId(-1);
3619 return; // Already selected.
3620 }
3621
3622 switch (N->getOpcode()) {
3623 default: break;
3624 case ISD::STORE: {
3625 // For Thumb1, match an sp-relative store in C++. This is a little
3626 // unfortunate, but I don't think I can make the chain check work
3627 // otherwise. (The chain of the store has to be the same as the chain
3628 // of the CopyFromReg, or else we can't replace the CopyFromReg with
3629 // a direct reference to "SP".)
3630 //
3631 // This is only necessary on Thumb1 because Thumb1 sp-relative stores use
3632 // a different addressing mode from other four-byte stores.
3633 //
3634 // This pattern usually comes up with call arguments.
3635 StoreSDNode *ST = cast<StoreSDNode>(N);
3636 SDValue Ptr = ST->getBasePtr();
3637 if (Subtarget->isThumb1Only() && ST->isUnindexed()) {
3638 int RHSC = 0;
3639 if (Ptr.getOpcode() == ISD::ADD &&
3640 isScaledConstantInRange(Ptr.getOperand(1), /*Scale=*/4, 0, 256, RHSC))
3641 Ptr = Ptr.getOperand(0);
3642
3643 if (Ptr.getOpcode() == ISD::CopyFromReg &&
3644 cast<RegisterSDNode>(Ptr.getOperand(1))->getReg() == ARM::SP &&
3645 Ptr.getOperand(0) == ST->getChain()) {
3646 SDValue Ops[] = {ST->getValue(),
3647 CurDAG->getRegister(ARM::SP, MVT::i32),
3648 CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
3649 getAL(CurDAG, dl),
3650 CurDAG->getRegister(0, MVT::i32),
3651 ST->getChain()};
3652 MachineSDNode *ResNode =
3653 CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
3654 MachineMemOperand *MemOp = ST->getMemOperand();
3655 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
3656 ReplaceNode(N, ResNode);
3657 return;
3658 }
3659 }
3660 break;
3661 }
3662 case ISD::WRITE_REGISTER:
3663 if (tryWriteRegister(N))
3664 return;
3665 break;
3666 case ISD::READ_REGISTER:
3667 if (tryReadRegister(N))
3668 return;
3669 break;
3670 case ISD::INLINEASM:
3671 case ISD::INLINEASM_BR:
3672 if (tryInlineAsm(N))
3673 return;
3674 break;
3675 case ISD::XOR:
3676 // Select special operations if XOR node forms integer ABS pattern
3677 if (tryABSOp(N))
3678 return;
3679 // Other cases are autogenerated.
3680 break;
3681 case ISD::Constant: {
3682 unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
3683 // If we can't materialize the constant we need to use a literal pool
3684 if (ConstantMaterializationCost(Val, Subtarget) > 2) {
3685 SDValue CPIdx = CurDAG->getTargetConstantPool(
3686 ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
3687 TLI->getPointerTy(CurDAG->getDataLayout()));
3688
3689 SDNode *ResNode;
3690 if (Subtarget->isThumb()) {
3691 SDValue Ops[] = {
3692 CPIdx,
3693 getAL(CurDAG, dl),
3694 CurDAG->getRegister(0, MVT::i32),
3695 CurDAG->getEntryNode()
3696 };
3697 ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
3698 Ops);
3699 } else {
3700 SDValue Ops[] = {
3701 CPIdx,
3702 CurDAG->getTargetConstant(0, dl, MVT::i32),
3703 getAL(CurDAG, dl),
3704 CurDAG->getRegister(0, MVT::i32),
3705 CurDAG->getEntryNode()
3706 };
3707 ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
3708 Ops);
3709 }
3710 // Annotate the Node with memory operand information so that MachineInstr
3711 // queries work properly. This e.g. gives the register allocation the
3712 // required information for rematerialization.
3713 MachineFunction& MF = CurDAG->getMachineFunction();
3714 MachineMemOperand *MemOp =
3715 MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
3716 MachineMemOperand::MOLoad, 4, Align(4));
3717
3718 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
3719
3720 ReplaceNode(N, ResNode);
3721 return;
3722 }
3723
3724 // Other cases are autogenerated.
3725 break;
3726 }
3727 case ISD::FrameIndex: {
3728 // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
3729 int FI = cast<FrameIndexSDNode>(N)->getIndex();
3730 SDValue TFI = CurDAG->getTargetFrameIndex(
3731 FI, TLI->getPointerTy(CurDAG->getDataLayout()));
3732 if (Subtarget->isThumb1Only()) {
3733 // Set the alignment of the frame object to 4, to avoid having to generate
3734 // more than one ADD
3735 MachineFrameInfo &MFI = MF->getFrameInfo();
3736 if (MFI.getObjectAlign(FI) < Align(4))
3737 MFI.setObjectAlignment(FI, Align(4));
3738 CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
3739 CurDAG->getTargetConstant(0, dl, MVT::i32));
3740 return;
3741 } else {
3742 unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
3743 ARM::t2ADDri : ARM::ADDri);
3744 SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
3745 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3746 CurDAG->getRegister(0, MVT::i32) };
3747 CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
3748 return;
3749 }
3750 }
3751 case ISD::INSERT_VECTOR_ELT: {
3752 if (tryInsertVectorElt(N))
3753 return;
3754 break;
3755 }
3756 case ISD::SRL:
3757 if (tryV6T2BitfieldExtractOp(N, false))
3758 return;
3759 break;
3760 case ISD::SIGN_EXTEND_INREG:
3761 case ISD::SRA:
3762 if (tryV6T2BitfieldExtractOp(N, true))
3763 return;
3764 break;
3765 case ISD::FP_TO_UINT:
3766 case ISD::FP_TO_SINT:
3767 if (tryFP_TO_INT(N, dl))
3768 return;
3769 break;
3770 case ISD::FMUL:
3771 if (tryFMULFixed(N, dl))
3772 return;
3773 break;
3774 case ISD::MUL:
3775 if (Subtarget->isThumb1Only())
3776 break;
3777 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
3778 unsigned RHSV = C->getZExtValue();
3779 if (!RHSV) break;
3780 if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
3781 unsigned ShImm = Log2_32(RHSV-1);
3782 if (ShImm >= 32)
3783 break;
3784 SDValue V = N->getOperand(0);
3785 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
3786 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3787 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3788 if (Subtarget->isThumb()) {
3789 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3790 CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
3791 return;
3792 } else {
3793 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
3794 Reg0 };
3795 CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
3796 return;
3797 }
3798 }
3799 if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
3800 unsigned ShImm = Log2_32(RHSV+1);
3801 if (ShImm >= 32)
3802 break;
3803 SDValue V = N->getOperand(0);
3804 ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
3805 SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
3806 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
3807 if (Subtarget->isThumb()) {
3808 SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
3809 CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
3810 return;
3811 } else {
3812 SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
3813 Reg0 };
3814 CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
3815 return;
3816 }
3817 }
3818 }
3819 break;
3820 case ISD::AND: {
3821 // Check for unsigned bitfield extract
3822 if (tryV6T2BitfieldExtractOp(N, false))
3823 return;
3824
3825 // If an immediate is used in an AND node, it is possible that the immediate
3826 // can be more optimally materialized when negated. If this is the case we
3827 // can negate the immediate and use a BIC instead.
3828 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
3829 if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
3830 uint32_t Imm = (uint32_t) N1C->getZExtValue();
3831
3832 // In Thumb2 mode, an AND can take a 12-bit immediate. If this
3833 // immediate can be negated and fit in the immediate operand of
3834 // a t2BIC, don't do any manual transform here as this can be
3835 // handled by the generic ISel machinery.
3836 bool PreferImmediateEncoding =
3837 Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
3838 if (!PreferImmediateEncoding &&
3839 ConstantMaterializationCost(Imm, Subtarget) >
3840 ConstantMaterializationCost(~Imm, Subtarget)) {
3841 // The current immediate costs more to materialize than a negated
3842 // immediate, so negate the immediate and use a BIC.
3843 SDValue NewImm =
3844 CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
3845 // If the new constant didn't exist before, reposition it in the topological
3846 // ordering so it is just before N. Otherwise, don't touch its location.
3847 if (NewImm->getNodeId() == -1)
3848 CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
3849
3850 if (!Subtarget->hasThumb2()) {
3851 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
3852 N->getOperand(0), NewImm, getAL(CurDAG, dl),
3853 CurDAG->getRegister(0, MVT::i32)};
3854 ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
3855 return;
3856 } else {
3857 SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
3858 CurDAG->getRegister(0, MVT::i32),
3859 CurDAG->getRegister(0, MVT::i32)};
3860 ReplaceNode(N,
3861 CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
3862 return;
3863 }
3864 }
3865 }
3866
3867 // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
3868 // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
3869 // are entirely contributed by c2 and lower 16-bits are entirely contributed
3870 // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
3871 // Select it to: "movt x, ((c1 & 0xffff) >> 16)
3872 EVT VT = N->getValueType(0);
3873 if (VT != MVT::i32)
3874 break;
3875 unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
3876 ? ARM::t2MOVTi16
3877 : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
3878 if (!Opc)
3879 break;
3880 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
3881 N1C = dyn_cast<ConstantSDNode>(N1);
3882 if (!N1C)
3883 break;
3884 if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
3885 SDValue N2 = N0.getOperand(1);
3886 ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2);
3887 if (!N2C)
3888 break;
3889 unsigned N1CVal = N1C->getZExtValue();
3890 unsigned N2CVal = N2C->getZExtValue();
3891 if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
3892 (N1CVal & 0xffffU) == 0xffffU &&
3893 (N2CVal & 0xffffU) == 0x0U) {
3894 SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
3895 dl, MVT::i32);
3896 SDValue Ops[] = { N0.getOperand(0), Imm16,
3897 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
3898 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
3899 return;
3900 }
3901 }
3902
3903 break;
3904 }
3905 case ARMISD::UMAAL: {
3906 unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
3907 SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
3908 N->getOperand(2), N->getOperand(3),
3909 getAL(CurDAG, dl),
3910 CurDAG->getRegister(0, MVT::i32) };
3911 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
3912 return;
3913 }
3914 case ARMISD::UMLAL:{
3915 if (Subtarget->isThumb()) {
3916 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3917 N->getOperand(3), getAL(CurDAG, dl),
3918 CurDAG->getRegister(0, MVT::i32)};
3919 ReplaceNode(
3920 N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
3921 return;
3922 }else{
3923 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3924 N->getOperand(3), getAL(CurDAG, dl),
3925 CurDAG->getRegister(0, MVT::i32),
3926 CurDAG->getRegister(0, MVT::i32) };
3927 ReplaceNode(N, CurDAG->getMachineNode(
3928 Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
3929 MVT::i32, MVT::i32, Ops));
3930 return;
3931 }
3932 }
3933 case ARMISD::SMLAL:{
3934 if (Subtarget->isThumb()) {
3935 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3936 N->getOperand(3), getAL(CurDAG, dl),
3937 CurDAG->getRegister(0, MVT::i32)};
3938 ReplaceNode(
3939 N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
3940 return;
3941 }else{
3942 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
3943 N->getOperand(3), getAL(CurDAG, dl),
3944 CurDAG->getRegister(0, MVT::i32),
3945 CurDAG->getRegister(0, MVT::i32) };
3946 ReplaceNode(N, CurDAG->getMachineNode(
3947 Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
3948 MVT::i32, MVT::i32, Ops));
3949 return;
3950 }
3951 }
3952 case ARMISD::SUBE: {
3953 if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
3954 break;
3955 // Look for a pattern to match SMMLS
3956 // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
3957 if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
3958 N->getOperand(2).getOpcode() != ARMISD::SUBC ||
3959 !SDValue(N, 1).use_empty())
3960 break;
3961
3962 if (Subtarget->isThumb())
3963 assert(Subtarget->hasThumb2() &&(static_cast <bool> (Subtarget->hasThumb2() &&
"This pattern should not be generated for Thumb") ? void (0)
: __assert_fail ("Subtarget->hasThumb2() && \"This pattern should not be generated for Thumb\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3964, __extension__ __PRETTY_FUNCTION__))
3964 "This pattern should not be generated for Thumb")(static_cast <bool> (Subtarget->hasThumb2() &&
"This pattern should not be generated for Thumb") ? void (0)
: __assert_fail ("Subtarget->hasThumb2() && \"This pattern should not be generated for Thumb\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 3964, __extension__ __PRETTY_FUNCTION__));
3965
3966 SDValue SmulLoHi = N->getOperand(1);
3967 SDValue Subc = N->getOperand(2);
3968 auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
3969
3970 if (!Zero || Zero->getZExtValue() != 0 ||
3971 Subc.getOperand(1) != SmulLoHi.getValue(0) ||
3972 N->getOperand(1) != SmulLoHi.getValue(1) ||
3973 N->getOperand(2) != Subc.getValue(1))
3974 break;
3975
3976 unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
3977 SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
3978 N->getOperand(0), getAL(CurDAG, dl),
3979 CurDAG->getRegister(0, MVT::i32) };
3980 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
3981 return;
3982 }
3983 case ISD::LOAD: {
3984 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N))
3985 return;
3986 if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
3987 if (tryT2IndexedLoad(N))
3988 return;
3989 } else if (Subtarget->isThumb()) {
3990 if (tryT1IndexedLoad(N))
3991 return;
3992 } else if (tryARMIndexedLoad(N))
3993 return;
3994 // Other cases are autogenerated.
3995 break;
3996 }
3997 case ISD::MLOAD:
3998 if (Subtarget->hasMVEIntegerOps() && tryMVEIndexedLoad(N))
3999 return;
4000 // Other cases are autogenerated.
4001 break;
4002 case ARMISD::WLSSETUP: {
4003 SDNode *New = CurDAG->getMachineNode(ARM::t2WhileLoopSetup, dl, MVT::i32,
4004 N->getOperand(0));
4005 ReplaceUses(N, New);
4006 CurDAG->RemoveDeadNode(N);
4007 return;
4008 }
4009 case ARMISD::WLS: {
4010 SDNode *New = CurDAG->getMachineNode(ARM::t2WhileLoopStart, dl, MVT::Other,
4011 N->getOperand(1), N->getOperand(2),
4012 N->getOperand(0));
4013 ReplaceUses(N, New);
4014 CurDAG->RemoveDeadNode(N);
4015 return;
4016 }
4017 case ARMISD::LE: {
4018 SDValue Ops[] = { N->getOperand(1),
4019 N->getOperand(2),
4020 N->getOperand(0) };
4021 unsigned Opc = ARM::t2LoopEnd;
4022 SDNode *New = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops);
4023 ReplaceUses(N, New);
4024 CurDAG->RemoveDeadNode(N);
4025 return;
4026 }
4027 case ARMISD::LDRD: {
4028 if (Subtarget->isThumb2())
4029 break; // TableGen handles isel in this case.
4030 SDValue Base, RegOffset, ImmOffset;
4031 const SDValue &Chain = N->getOperand(0);
4032 const SDValue &Addr = N->getOperand(1);
4033 SelectAddrMode3(Addr, Base, RegOffset, ImmOffset);
4034 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4035 // The register-offset variant of LDRD mandates that the register
4036 // allocated to RegOffset is not reused in any of the remaining operands.
4037 // This restriction is currently not enforced. Therefore emitting this
4038 // variant is explicitly avoided.
4039 Base = Addr;
4040 RegOffset = CurDAG->getRegister(0, MVT::i32);
4041 }
4042 SDValue Ops[] = {Base, RegOffset, ImmOffset, Chain};
4043 SDNode *New = CurDAG->getMachineNode(ARM::LOADDUAL, dl,
4044 {MVT::Untyped, MVT::Other}, Ops);
4045 SDValue Lo = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4046 SDValue(New, 0));
4047 SDValue Hi = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4048 SDValue(New, 0));
4049 transferMemOperands(N, New);
4050 ReplaceUses(SDValue(N, 0), Lo);
4051 ReplaceUses(SDValue(N, 1), Hi);
4052 ReplaceUses(SDValue(N, 2), SDValue(New, 1));
4053 CurDAG->RemoveDeadNode(N);
4054 return;
4055 }
4056 case ARMISD::STRD: {
4057 if (Subtarget->isThumb2())
4058 break; // TableGen handles isel in this case.
4059 SDValue Base, RegOffset, ImmOffset;
4060 const SDValue &Chain = N->getOperand(0);
4061 const SDValue &Addr = N->getOperand(3);
4062 SelectAddrMode3(Addr, Base, RegOffset, ImmOffset);
4063 if (RegOffset != CurDAG->getRegister(0, MVT::i32)) {
4064 // The register-offset variant of STRD mandates that the register
4065 // allocated to RegOffset is not reused in any of the remaining operands.
4066 // This restriction is currently not enforced. Therefore emitting this
4067 // variant is explicitly avoided.
4068 Base = Addr;
4069 RegOffset = CurDAG->getRegister(0, MVT::i32);
4070 }
4071 SDNode *RegPair =
4072 createGPRPairNode(MVT::Untyped, N->getOperand(1), N->getOperand(2));
4073 SDValue Ops[] = {SDValue(RegPair, 0), Base, RegOffset, ImmOffset, Chain};
4074 SDNode *New = CurDAG->getMachineNode(ARM::STOREDUAL, dl, MVT::Other, Ops);
4075 transferMemOperands(N, New);
4076 ReplaceUses(SDValue(N, 0), SDValue(New, 0));
4077 CurDAG->RemoveDeadNode(N);
4078 return;
4079 }
4080 case ARMISD::LOOP_DEC: {
4081 SDValue Ops[] = { N->getOperand(1),
4082 N->getOperand(2),
4083 N->getOperand(0) };
4084 SDNode *Dec =
4085 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4086 CurDAG->getVTList(MVT::i32, MVT::Other), Ops);
4087 ReplaceUses(N, Dec);
4088 CurDAG->RemoveDeadNode(N);
4089 return;
4090 }
4091 case ARMISD::BRCOND: {
4092 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
4093 // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
4094 // Pattern complexity = 6 cost = 1 size = 0
4095
4096 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
4097 // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
4098 // Pattern complexity = 6 cost = 1 size = 0
4099
4100 // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
4101 // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
4102 // Pattern complexity = 6 cost = 1 size = 0
4103
4104 unsigned Opc = Subtarget->isThumb() ?
4105 ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
4106 SDValue Chain = N->getOperand(0);
4107 SDValue N1 = N->getOperand(1);
4108 SDValue N2 = N->getOperand(2);
4109 SDValue N3 = N->getOperand(3);
4110 SDValue InFlag = N->getOperand(4);
4111 assert(N1.getOpcode() == ISD::BasicBlock)(static_cast <bool> (N1.getOpcode() == ISD::BasicBlock)
? void (0) : __assert_fail ("N1.getOpcode() == ISD::BasicBlock"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4111, __extension__ __PRETTY_FUNCTION__));
4112 assert(N2.getOpcode() == ISD::Constant)(static_cast <bool> (N2.getOpcode() == ISD::Constant) ?
void (0) : __assert_fail ("N2.getOpcode() == ISD::Constant",
"/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4112, __extension__ __PRETTY_FUNCTION__));
4113 assert(N3.getOpcode() == ISD::Register)(static_cast <bool> (N3.getOpcode() == ISD::Register) ?
void (0) : __assert_fail ("N3.getOpcode() == ISD::Register",
"/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4113, __extension__ __PRETTY_FUNCTION__));
4114
4115 unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
4116
4117 if (InFlag.getOpcode() == ARMISD::CMPZ) {
4118 if (InFlag.getOperand(0).getOpcode() == ISD::INTRINSIC_W_CHAIN) {
4119 SDValue Int = InFlag.getOperand(0);
4120 uint64_t ID = cast<ConstantSDNode>(Int->getOperand(1))->getZExtValue();
4121
4122 // Handle low-overhead loops.
4123 if (ID == Intrinsic::loop_decrement_reg) {
4124 SDValue Elements = Int.getOperand(2);
4125 SDValue Size = CurDAG->getTargetConstant(
4126 cast<ConstantSDNode>(Int.getOperand(3))->getZExtValue(), dl,
4127 MVT::i32);
4128
4129 SDValue Args[] = { Elements, Size, Int.getOperand(0) };
4130 SDNode *LoopDec =
4131 CurDAG->getMachineNode(ARM::t2LoopDec, dl,
4132 CurDAG->getVTList(MVT::i32, MVT::Other),
4133 Args);
4134 ReplaceUses(Int.getNode(), LoopDec);
4135
4136 SDValue EndArgs[] = { SDValue(LoopDec, 0), N1, Chain };
4137 SDNode *LoopEnd =
4138 CurDAG->getMachineNode(ARM::t2LoopEnd, dl, MVT::Other, EndArgs);
4139
4140 ReplaceUses(N, LoopEnd);
4141 CurDAG->RemoveDeadNode(N);
4142 CurDAG->RemoveDeadNode(InFlag.getNode());
4143 CurDAG->RemoveDeadNode(Int.getNode());
4144 return;
4145 }
4146 }
4147
4148 bool SwitchEQNEToPLMI;
4149 SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
4150 InFlag = N->getOperand(4);
4151
4152 if (SwitchEQNEToPLMI) {
4153 switch ((ARMCC::CondCodes)CC) {
4154 default: llvm_unreachable("CMPZ must be either NE or EQ!")::llvm::llvm_unreachable_internal("CMPZ must be either NE or EQ!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4154);
4155 case ARMCC::NE:
4156 CC = (unsigned)ARMCC::MI;
4157 break;
4158 case ARMCC::EQ:
4159 CC = (unsigned)ARMCC::PL;
4160 break;
4161 }
4162 }
4163 }
4164
4165 SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
4166 SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
4167 SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
4168 MVT::Glue, Ops);
4169 Chain = SDValue(ResNode, 0);
4170 if (N->getNumValues() == 2) {
4171 InFlag = SDValue(ResNode, 1);
4172 ReplaceUses(SDValue(N, 1), InFlag);
4173 }
4174 ReplaceUses(SDValue(N, 0),
4175 SDValue(Chain.getNode(), Chain.getResNo()));
4176 CurDAG->RemoveDeadNode(N);
4177 return;
4178 }
4179
4180 case ARMISD::CMPZ: {
4181 // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
4182 // This allows us to avoid materializing the expensive negative constant.
4183 // The CMPZ #0 is useless and will be peepholed away but we need to keep it
4184 // for its glue output.
4185 SDValue X = N->getOperand(0);
4186 auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
4187 if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
4188 int64_t Addend = -C->getSExtValue();
4189
4190 SDNode *Add = nullptr;
4191 // ADDS can be better than CMN if the immediate fits in a
4192 // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
4193 // Outside that range we can just use a CMN which is 32-bit but has a
4194 // 12-bit immediate range.
4195 if (Addend < 1<<8) {
4196 if (Subtarget->isThumb2()) {
4197 SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4198 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
4199 CurDAG->getRegister(0, MVT::i32) };
4200 Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
4201 } else {
4202 unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
4203 SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
4204 CurDAG->getTargetConstant(Addend, dl, MVT::i32),
4205 getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
4206 Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
4207 }
4208 }
4209 if (Add) {
4210 SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
4211 CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
4212 }
4213 }
4214 // Other cases are autogenerated.
4215 break;
4216 }
4217
4218 case ARMISD::CMOV: {
4219 SDValue InFlag = N->getOperand(4);
4220
4221 if (InFlag.getOpcode() == ARMISD::CMPZ) {
4222 bool SwitchEQNEToPLMI;
4223 SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
4224
4225 if (SwitchEQNEToPLMI) {
4226 SDValue ARMcc = N->getOperand(2);
4227 ARMCC::CondCodes CC =
4228 (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
4229
4230 switch (CC) {
4231 default: llvm_unreachable("CMPZ must be either NE or EQ!")::llvm::llvm_unreachable_internal("CMPZ must be either NE or EQ!"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4231);
4232 case ARMCC::NE:
4233 CC = ARMCC::MI;
4234 break;
4235 case ARMCC::EQ:
4236 CC = ARMCC::PL;
4237 break;
4238 }
4239 SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
4240 SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
4241 N->getOperand(3), N->getOperand(4)};
4242 CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
4243 }
4244
4245 }
4246 // Other cases are autogenerated.
4247 break;
4248 }
4249
4250 case ARMISD::VZIP: {
4251 unsigned Opc = 0;
4252 EVT VT = N->getValueType(0);
4253 switch (VT.getSimpleVT().SimpleTy) {
4254 default: return;
4255 case MVT::v8i8: Opc = ARM::VZIPd8; break;
4256 case MVT::v4f16:
4257 case MVT::v4i16: Opc = ARM::VZIPd16; break;
4258 case MVT::v2f32:
4259 // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
4260 case MVT::v2i32: Opc = ARM::VTRNd32; break;
4261 case MVT::v16i8: Opc = ARM::VZIPq8; break;
4262 case MVT::v8f16:
4263 case MVT::v8i16: Opc = ARM::VZIPq16; break;
4264 case MVT::v4f32:
4265 case MVT::v4i32: Opc = ARM::VZIPq32; break;
4266 }
4267 SDValue Pred = getAL(CurDAG, dl);
4268 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4269 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
4270 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4271 return;
4272 }
4273 case ARMISD::VUZP: {
4274 unsigned Opc = 0;
4275 EVT VT = N->getValueType(0);
4276 switch (VT.getSimpleVT().SimpleTy) {
4277 default: return;
4278 case MVT::v8i8: Opc = ARM::VUZPd8; break;
4279 case MVT::v4f16:
4280 case MVT::v4i16: Opc = ARM::VUZPd16; break;
4281 case MVT::v2f32:
4282 // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
4283 case MVT::v2i32: Opc = ARM::VTRNd32; break;
4284 case MVT::v16i8: Opc = ARM::VUZPq8; break;
4285 case MVT::v8f16:
4286 case MVT::v8i16: Opc = ARM::VUZPq16; break;
4287 case MVT::v4f32:
4288 case MVT::v4i32: Opc = ARM::VUZPq32; break;
4289 }
4290 SDValue Pred = getAL(CurDAG, dl);
4291 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4292 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
4293 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4294 return;
4295 }
4296 case ARMISD::VTRN: {
4297 unsigned Opc = 0;
4298 EVT VT = N->getValueType(0);
4299 switch (VT.getSimpleVT().SimpleTy) {
4300 default: return;
4301 case MVT::v8i8: Opc = ARM::VTRNd8; break;
4302 case MVT::v4f16:
4303 case MVT::v4i16: Opc = ARM::VTRNd16; break;
4304 case MVT::v2f32:
4305 case MVT::v2i32: Opc = ARM::VTRNd32; break;
4306 case MVT::v16i8: Opc = ARM::VTRNq8; break;
4307 case MVT::v8f16:
4308 case MVT::v8i16: Opc = ARM::VTRNq16; break;
4309 case MVT::v4f32:
4310 case MVT::v4i32: Opc = ARM::VTRNq32; break;
4311 }
4312 SDValue Pred = getAL(CurDAG, dl);
4313 SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
4314 SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
4315 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
4316 return;
4317 }
4318 case ARMISD::BUILD_VECTOR: {
4319 EVT VecVT = N->getValueType(0);
4320 EVT EltVT = VecVT.getVectorElementType();
4321 unsigned NumElts = VecVT.getVectorNumElements();
4322 if (EltVT == MVT::f64) {
4323 assert(NumElts == 2 && "unexpected type for BUILD_VECTOR")(static_cast <bool> (NumElts == 2 && "unexpected type for BUILD_VECTOR"
) ? void (0) : __assert_fail ("NumElts == 2 && \"unexpected type for BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4323, __extension__ __PRETTY_FUNCTION__));
4324 ReplaceNode(
4325 N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
4326 return;
4327 }
4328 assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR")(static_cast <bool> (EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR"
) ? void (0) : __assert_fail ("EltVT == MVT::f32 && \"unexpected type for BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4328, __extension__ __PRETTY_FUNCTION__));
4329 if (NumElts == 2) {
4330 ReplaceNode(
4331 N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
4332 return;
4333 }
4334 assert(NumElts == 4 && "unexpected type for BUILD_VECTOR")(static_cast <bool> (NumElts == 4 && "unexpected type for BUILD_VECTOR"
) ? void (0) : __assert_fail ("NumElts == 4 && \"unexpected type for BUILD_VECTOR\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 4334, __extension__ __PRETTY_FUNCTION__));
4335 ReplaceNode(N,
4336 createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
4337 N->getOperand(2), N->getOperand(3)));
4338 return;
4339 }
4340
4341 case ARMISD::VLD1DUP: {
4342 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
4343 ARM::VLD1DUPd32 };
4344 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
4345 ARM::VLD1DUPq32 };
4346 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes);
4347 return;
4348 }
4349
4350 case ARMISD::VLD2DUP: {
4351 static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4352 ARM::VLD2DUPd32 };
4353 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes);
4354 return;
4355 }
4356
4357 case ARMISD::VLD3DUP: {
4358 static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
4359 ARM::VLD3DUPd16Pseudo,
4360 ARM::VLD3DUPd32Pseudo };
4361 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes);
4362 return;
4363 }
4364
4365 case ARMISD::VLD4DUP: {
4366 static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
4367 ARM::VLD4DUPd16Pseudo,
4368 ARM::VLD4DUPd32Pseudo };
4369 SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes);
4370 return;
4371 }
4372
4373 case ARMISD::VLD1DUP_UPD: {
4374 static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
4375 ARM::VLD1DUPd16wb_fixed,
4376 ARM::VLD1DUPd32wb_fixed };
4377 static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
4378 ARM::VLD1DUPq16wb_fixed,
4379 ARM::VLD1DUPq32wb_fixed };
4380 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
4381 return;
4382 }
4383
4384 case ARMISD::VLD2DUP_UPD: {
4385 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8wb_fixed,
4386 ARM::VLD2DUPd16wb_fixed,
4387 ARM::VLD2DUPd32wb_fixed,
4388 ARM::VLD1q64wb_fixed };
4389 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4390 ARM::VLD2DUPq16EvenPseudo,
4391 ARM::VLD2DUPq32EvenPseudo };
4392 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudoWB_fixed,
4393 ARM::VLD2DUPq16OddPseudoWB_fixed,
4394 ARM::VLD2DUPq32OddPseudoWB_fixed };
4395 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, DOpcodes, QOpcodes0, QOpcodes1);
4396 return;
4397 }
4398
4399 case ARMISD::VLD3DUP_UPD: {
4400 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
4401 ARM::VLD3DUPd16Pseudo_UPD,
4402 ARM::VLD3DUPd32Pseudo_UPD,
4403 ARM::VLD1d64TPseudoWB_fixed };
4404 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4405 ARM::VLD3DUPq16EvenPseudo,
4406 ARM::VLD3DUPq32EvenPseudo };
4407 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo_UPD,
4408 ARM::VLD3DUPq16OddPseudo_UPD,
4409 ARM::VLD3DUPq32OddPseudo_UPD };
4410 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4411 return;
4412 }
4413
4414 case ARMISD::VLD4DUP_UPD: {
4415 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
4416 ARM::VLD4DUPd16Pseudo_UPD,
4417 ARM::VLD4DUPd32Pseudo_UPD,
4418 ARM::VLD1d64QPseudoWB_fixed };
4419 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4420 ARM::VLD4DUPq16EvenPseudo,
4421 ARM::VLD4DUPq32EvenPseudo };
4422 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo_UPD,
4423 ARM::VLD4DUPq16OddPseudo_UPD,
4424 ARM::VLD4DUPq32OddPseudo_UPD };
4425 SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4426 return;
4427 }
4428
4429 case ARMISD::VLD1_UPD: {
4430 static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
4431 ARM::VLD1d16wb_fixed,
4432 ARM::VLD1d32wb_fixed,
4433 ARM::VLD1d64wb_fixed };
4434 static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
4435 ARM::VLD1q16wb_fixed,
4436 ARM::VLD1q32wb_fixed,
4437 ARM::VLD1q64wb_fixed };
4438 SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
4439 return;
4440 }
4441
4442 case ARMISD::VLD2_UPD: {
4443 if (Subtarget->hasNEON()) {
4444 static const uint16_t DOpcodes[] = {
4445 ARM::VLD2d8wb_fixed, ARM::VLD2d16wb_fixed, ARM::VLD2d32wb_fixed,
4446 ARM::VLD1q64wb_fixed};
4447 static const uint16_t QOpcodes[] = {ARM::VLD2q8PseudoWB_fixed,
4448 ARM::VLD2q16PseudoWB_fixed,
4449 ARM::VLD2q32PseudoWB_fixed};
4450 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
4451 } else {
4452 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8,
4453 ARM::MVE_VLD21_8_wb};
4454 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
4455 ARM::MVE_VLD21_16_wb};
4456 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
4457 ARM::MVE_VLD21_32_wb};
4458 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4459 SelectMVE_VLD(N, 2, Opcodes, true);
4460 }
4461 return;
4462 }
4463
4464 case ARMISD::VLD3_UPD: {
4465 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
4466 ARM::VLD3d16Pseudo_UPD,
4467 ARM::VLD3d32Pseudo_UPD,
4468 ARM::VLD1d64TPseudoWB_fixed};
4469 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4470 ARM::VLD3q16Pseudo_UPD,
4471 ARM::VLD3q32Pseudo_UPD };
4472 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
4473 ARM::VLD3q16oddPseudo_UPD,
4474 ARM::VLD3q32oddPseudo_UPD };
4475 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4476 return;
4477 }
4478
4479 case ARMISD::VLD4_UPD: {
4480 if (Subtarget->hasNEON()) {
4481 static const uint16_t DOpcodes[] = {
4482 ARM::VLD4d8Pseudo_UPD, ARM::VLD4d16Pseudo_UPD, ARM::VLD4d32Pseudo_UPD,
4483 ARM::VLD1d64QPseudoWB_fixed};
4484 static const uint16_t QOpcodes0[] = {ARM::VLD4q8Pseudo_UPD,
4485 ARM::VLD4q16Pseudo_UPD,
4486 ARM::VLD4q32Pseudo_UPD};
4487 static const uint16_t QOpcodes1[] = {ARM::VLD4q8oddPseudo_UPD,
4488 ARM::VLD4q16oddPseudo_UPD,
4489 ARM::VLD4q32oddPseudo_UPD};
4490 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4491 } else {
4492 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
4493 ARM::MVE_VLD42_8,
4494 ARM::MVE_VLD43_8_wb};
4495 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
4496 ARM::MVE_VLD42_16,
4497 ARM::MVE_VLD43_16_wb};
4498 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
4499 ARM::MVE_VLD42_32,
4500 ARM::MVE_VLD43_32_wb};
4501 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
4502 SelectMVE_VLD(N, 4, Opcodes, true);
4503 }
4504 return;
4505 }
4506
4507 case ARMISD::VLD1x2_UPD: {
4508 if (Subtarget->hasNEON()) {
4509 static const uint16_t DOpcodes[] = {
4510 ARM::VLD1q8wb_fixed, ARM::VLD1q16wb_fixed, ARM::VLD1q32wb_fixed,
4511 ARM::VLD1q64wb_fixed};
4512 static const uint16_t QOpcodes[] = {
4513 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4514 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4515 SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
4516 return;
4517 }
4518 break;
4519 }
4520
4521 case ARMISD::VLD1x3_UPD: {
4522 if (Subtarget->hasNEON()) {
4523 static const uint16_t DOpcodes[] = {
4524 ARM::VLD1d8TPseudoWB_fixed, ARM::VLD1d16TPseudoWB_fixed,
4525 ARM::VLD1d32TPseudoWB_fixed, ARM::VLD1d64TPseudoWB_fixed};
4526 static const uint16_t QOpcodes0[] = {
4527 ARM::VLD1q8LowTPseudo_UPD, ARM::VLD1q16LowTPseudo_UPD,
4528 ARM::VLD1q32LowTPseudo_UPD, ARM::VLD1q64LowTPseudo_UPD};
4529 static const uint16_t QOpcodes1[] = {
4530 ARM::VLD1q8HighTPseudo_UPD, ARM::VLD1q16HighTPseudo_UPD,
4531 ARM::VLD1q32HighTPseudo_UPD, ARM::VLD1q64HighTPseudo_UPD};
4532 SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4533 return;
4534 }
4535 break;
4536 }
4537
4538 case ARMISD::VLD1x4_UPD: {
4539 if (Subtarget->hasNEON()) {
4540 static const uint16_t DOpcodes[] = {
4541 ARM::VLD1d8QPseudoWB_fixed, ARM::VLD1d16QPseudoWB_fixed,
4542 ARM::VLD1d32QPseudoWB_fixed, ARM::VLD1d64QPseudoWB_fixed};
4543 static const uint16_t QOpcodes0[] = {
4544 ARM::VLD1q8LowQPseudo_UPD, ARM::VLD1q16LowQPseudo_UPD,
4545 ARM::VLD1q32LowQPseudo_UPD, ARM::VLD1q64LowQPseudo_UPD};
4546 static const uint16_t QOpcodes1[] = {
4547 ARM::VLD1q8HighQPseudo_UPD, ARM::VLD1q16HighQPseudo_UPD,
4548 ARM::VLD1q32HighQPseudo_UPD, ARM::VLD1q64HighQPseudo_UPD};
4549 SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4550 return;
4551 }
4552 break;
4553 }
4554
4555 case ARMISD::VLD2LN_UPD: {
4556 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
4557 ARM::VLD2LNd16Pseudo_UPD,
4558 ARM::VLD2LNd32Pseudo_UPD };
4559 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
4560 ARM::VLD2LNq32Pseudo_UPD };
4561 SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
4562 return;
4563 }
4564
4565 case ARMISD::VLD3LN_UPD: {
4566 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
4567 ARM::VLD3LNd16Pseudo_UPD,
4568 ARM::VLD3LNd32Pseudo_UPD };
4569 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
4570 ARM::VLD3LNq32Pseudo_UPD };
4571 SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
4572 return;
4573 }
4574
4575 case ARMISD::VLD4LN_UPD: {
4576 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
4577 ARM::VLD4LNd16Pseudo_UPD,
4578 ARM::VLD4LNd32Pseudo_UPD };
4579 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
4580 ARM::VLD4LNq32Pseudo_UPD };
4581 SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
4582 return;
4583 }
4584
4585 case ARMISD::VST1_UPD: {
4586 static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
4587 ARM::VST1d16wb_fixed,
4588 ARM::VST1d32wb_fixed,
4589 ARM::VST1d64wb_fixed };
4590 static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
4591 ARM::VST1q16wb_fixed,
4592 ARM::VST1q32wb_fixed,
4593 ARM::VST1q64wb_fixed };
4594 SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
4595 return;
4596 }
4597
4598 case ARMISD::VST2_UPD: {
4599 if (Subtarget->hasNEON()) {
4600 static const uint16_t DOpcodes[] = {
4601 ARM::VST2d8wb_fixed, ARM::VST2d16wb_fixed, ARM::VST2d32wb_fixed,
4602 ARM::VST1q64wb_fixed};
4603 static const uint16_t QOpcodes[] = {ARM::VST2q8PseudoWB_fixed,
4604 ARM::VST2q16PseudoWB_fixed,
4605 ARM::VST2q32PseudoWB_fixed};
4606 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
4607 return;
4608 }
4609 break;
4610 }
4611
4612 case ARMISD::VST3_UPD: {
4613 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
4614 ARM::VST3d16Pseudo_UPD,
4615 ARM::VST3d32Pseudo_UPD,
4616 ARM::VST1d64TPseudoWB_fixed};
4617 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
4618 ARM::VST3q16Pseudo_UPD,
4619 ARM::VST3q32Pseudo_UPD };
4620 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
4621 ARM::VST3q16oddPseudo_UPD,
4622 ARM::VST3q32oddPseudo_UPD };
4623 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4624 return;
4625 }
4626
4627 case ARMISD::VST4_UPD: {
4628 if (Subtarget->hasNEON()) {
4629 static const uint16_t DOpcodes[] = {
4630 ARM::VST4d8Pseudo_UPD, ARM::VST4d16Pseudo_UPD, ARM::VST4d32Pseudo_UPD,
4631 ARM::VST1d64QPseudoWB_fixed};
4632 static const uint16_t QOpcodes0[] = {ARM::VST4q8Pseudo_UPD,
4633 ARM::VST4q16Pseudo_UPD,
4634 ARM::VST4q32Pseudo_UPD};
4635 static const uint16_t QOpcodes1[] = {ARM::VST4q8oddPseudo_UPD,
4636 ARM::VST4q16oddPseudo_UPD,
4637 ARM::VST4q32oddPseudo_UPD};
4638 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4639 return;
4640 }
4641 break;
4642 }
4643
4644 case ARMISD::VST1x2_UPD: {
4645 if (Subtarget->hasNEON()) {
4646 static const uint16_t DOpcodes[] = { ARM::VST1q8wb_fixed,
4647 ARM::VST1q16wb_fixed,
4648 ARM::VST1q32wb_fixed,
4649 ARM::VST1q64wb_fixed};
4650 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4651 ARM::VST1d16QPseudoWB_fixed,
4652 ARM::VST1d32QPseudoWB_fixed,
4653 ARM::VST1d64QPseudoWB_fixed };
4654 SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
4655 return;
4656 }
4657 break;
4658 }
4659
4660 case ARMISD::VST1x3_UPD: {
4661 if (Subtarget->hasNEON()) {
4662 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudoWB_fixed,
4663 ARM::VST1d16TPseudoWB_fixed,
4664 ARM::VST1d32TPseudoWB_fixed,
4665 ARM::VST1d64TPseudoWB_fixed };
4666 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
4667 ARM::VST1q16LowTPseudo_UPD,
4668 ARM::VST1q32LowTPseudo_UPD,
4669 ARM::VST1q64LowTPseudo_UPD };
4670 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo_UPD,
4671 ARM::VST1q16HighTPseudo_UPD,
4672 ARM::VST1q32HighTPseudo_UPD,
4673 ARM::VST1q64HighTPseudo_UPD };
4674 SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
4675 return;
4676 }
4677 break;
4678 }
4679
4680 case ARMISD::VST1x4_UPD: {
4681 if (Subtarget->hasNEON()) {
4682 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudoWB_fixed,
4683 ARM::VST1d16QPseudoWB_fixed,
4684 ARM::VST1d32QPseudoWB_fixed,
4685 ARM::VST1d64QPseudoWB_fixed };
4686 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
4687 ARM::VST1q16LowQPseudo_UPD,
4688 ARM::VST1q32LowQPseudo_UPD,
4689 ARM::VST1q64LowQPseudo_UPD };
4690 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo_UPD,
4691 ARM::VST1q16HighQPseudo_UPD,
4692 ARM::VST1q32HighQPseudo_UPD,
4693 ARM::VST1q64HighQPseudo_UPD };
4694 SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
4695 return;
4696 }
4697 break;
4698 }
4699 case ARMISD::VST2LN_UPD: {
4700 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
4701 ARM::VST2LNd16Pseudo_UPD,
4702 ARM::VST2LNd32Pseudo_UPD };
4703 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
4704 ARM::VST2LNq32Pseudo_UPD };
4705 SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
4706 return;
4707 }
4708
4709 case ARMISD::VST3LN_UPD: {
4710 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
4711 ARM::VST3LNd16Pseudo_UPD,
4712 ARM::VST3LNd32Pseudo_UPD };
4713 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
4714 ARM::VST3LNq32Pseudo_UPD };
4715 SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
4716 return;
4717 }
4718
4719 case ARMISD::VST4LN_UPD: {
4720 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
4721 ARM::VST4LNd16Pseudo_UPD,
4722 ARM::VST4LNd32Pseudo_UPD };
4723 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
4724 ARM::VST4LNq32Pseudo_UPD };
4725 SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
4726 return;
4727 }
4728
4729 case ISD::INTRINSIC_VOID:
4730 case ISD::INTRINSIC_W_CHAIN: {
4731 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
4732 switch (IntNo) {
4733 default:
4734 break;
4735
4736 case Intrinsic::arm_mrrc:
4737 case Intrinsic::arm_mrrc2: {
4738 SDLoc dl(N);
4739 SDValue Chain = N->getOperand(0);
4740 unsigned Opc;
4741
4742 if (Subtarget->isThumb())
4743 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
4744 else
4745 Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
4746
4747 SmallVector<SDValue, 5> Ops;
4748 Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
4749 Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
4750 Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
4751
4752 // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
4753 // instruction will always be '1111' but it is possible in assembly language to specify
4754 // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
4755 if (Opc != ARM::MRRC2) {
4756 Ops.push_back(getAL(CurDAG, dl));
4757 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4758 }
4759
4760 Ops.push_back(Chain);
4761
4762 // Writes to two registers.
4763 const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
4764
4765 ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
4766 return;
4767 }
4768 case Intrinsic::arm_ldaexd:
4769 case Intrinsic::arm_ldrexd: {
4770 SDLoc dl(N);
4771 SDValue Chain = N->getOperand(0);
4772 SDValue MemAddr = N->getOperand(2);
4773 bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
4774
4775 bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
4776 unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
4777 : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
4778
4779 // arm_ldrexd returns a i64 value in {i32, i32}
4780 std::vector<EVT> ResTys;
4781 if (isThumb) {
4782 ResTys.push_back(MVT::i32);
4783 ResTys.push_back(MVT::i32);
4784 } else
4785 ResTys.push_back(MVT::Untyped);
4786 ResTys.push_back(MVT::Other);
4787
4788 // Place arguments in the right order.
4789 SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
4790 CurDAG->getRegister(0, MVT::i32), Chain};
4791 SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4792 // Transfer memoperands.
4793 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
4794 CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
4795
4796 // Remap uses.
4797 SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
4798 if (!SDValue(N, 0).use_empty()) {
4799 SDValue Result;
4800 if (isThumb)
4801 Result = SDValue(Ld, 0);
4802 else {
4803 SDValue SubRegIdx =
4804 CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
4805 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4806 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
4807 Result = SDValue(ResNode,0);
4808 }
4809 ReplaceUses(SDValue(N, 0), Result);
4810 }
4811 if (!SDValue(N, 1).use_empty()) {
4812 SDValue Result;
4813 if (isThumb)
4814 Result = SDValue(Ld, 1);
4815 else {
4816 SDValue SubRegIdx =
4817 CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
4818 SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
4819 dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
4820 Result = SDValue(ResNode,0);
4821 }
4822 ReplaceUses(SDValue(N, 1), Result);
4823 }
4824 ReplaceUses(SDValue(N, 2), OutChain);
4825 CurDAG->RemoveDeadNode(N);
4826 return;
4827 }
4828 case Intrinsic::arm_stlexd:
4829 case Intrinsic::arm_strexd: {
4830 SDLoc dl(N);
4831 SDValue Chain = N->getOperand(0);
4832 SDValue Val0 = N->getOperand(2);
4833 SDValue Val1 = N->getOperand(3);
4834 SDValue MemAddr = N->getOperand(4);
4835
4836 // Store exclusive double return a i32 value which is the return status
4837 // of the issued store.
4838 const EVT ResTys[] = {MVT::i32, MVT::Other};
4839
4840 bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
4841 // Place arguments in the right order.
4842 SmallVector<SDValue, 7> Ops;
4843 if (isThumb) {
4844 Ops.push_back(Val0);
4845 Ops.push_back(Val1);
4846 } else
4847 // arm_strexd uses GPRPair.
4848 Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
4849 Ops.push_back(MemAddr);
4850 Ops.push_back(getAL(CurDAG, dl));
4851 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4852 Ops.push_back(Chain);
4853
4854 bool IsRelease = IntNo == Intrinsic::arm_stlexd;
4855 unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
4856 : (IsRelease ? ARM::STLEXD : ARM::STREXD);
4857
4858 SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
4859 // Transfer memoperands.
4860 MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
4861 CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
4862
4863 ReplaceNode(N, St);
4864 return;
4865 }
4866
4867 case Intrinsic::arm_neon_vld1: {
4868 static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
4869 ARM::VLD1d32, ARM::VLD1d64 };
4870 static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4871 ARM::VLD1q32, ARM::VLD1q64};
4872 SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
4873 return;
4874 }
4875
4876 case Intrinsic::arm_neon_vld1x2: {
4877 static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
4878 ARM::VLD1q32, ARM::VLD1q64 };
4879 static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
4880 ARM::VLD1d16QPseudo,
4881 ARM::VLD1d32QPseudo,
4882 ARM::VLD1d64QPseudo };
4883 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
4884 return;
4885 }
4886
4887 case Intrinsic::arm_neon_vld1x3: {
4888 static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
4889 ARM::VLD1d16TPseudo,
4890 ARM::VLD1d32TPseudo,
4891 ARM::VLD1d64TPseudo };
4892 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
4893 ARM::VLD1q16LowTPseudo_UPD,
4894 ARM::VLD1q32LowTPseudo_UPD,
4895 ARM::VLD1q64LowTPseudo_UPD };
4896 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
4897 ARM::VLD1q16HighTPseudo,
4898 ARM::VLD1q32HighTPseudo,
4899 ARM::VLD1q64HighTPseudo };
4900 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4901 return;
4902 }
4903
4904 case Intrinsic::arm_neon_vld1x4: {
4905 static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
4906 ARM::VLD1d16QPseudo,
4907 ARM::VLD1d32QPseudo,
4908 ARM::VLD1d64QPseudo };
4909 static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
4910 ARM::VLD1q16LowQPseudo_UPD,
4911 ARM::VLD1q32LowQPseudo_UPD,
4912 ARM::VLD1q64LowQPseudo_UPD };
4913 static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
4914 ARM::VLD1q16HighQPseudo,
4915 ARM::VLD1q32HighQPseudo,
4916 ARM::VLD1q64HighQPseudo };
4917 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4918 return;
4919 }
4920
4921 case Intrinsic::arm_neon_vld2: {
4922 static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
4923 ARM::VLD2d32, ARM::VLD1q64 };
4924 static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
4925 ARM::VLD2q32Pseudo };
4926 SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
4927 return;
4928 }
4929
4930 case Intrinsic::arm_neon_vld3: {
4931 static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
4932 ARM::VLD3d16Pseudo,
4933 ARM::VLD3d32Pseudo,
4934 ARM::VLD1d64TPseudo };
4935 static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
4936 ARM::VLD3q16Pseudo_UPD,
4937 ARM::VLD3q32Pseudo_UPD };
4938 static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
4939 ARM::VLD3q16oddPseudo,
4940 ARM::VLD3q32oddPseudo };
4941 SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
4942 return;
4943 }
4944
4945 case Intrinsic::arm_neon_vld4: {
4946 static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
4947 ARM::VLD4d16Pseudo,
4948 ARM::VLD4d32Pseudo,
4949 ARM::VLD1d64QPseudo };
4950 static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
4951 ARM::VLD4q16Pseudo_UPD,
4952 ARM::VLD4q32Pseudo_UPD };
4953 static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
4954 ARM::VLD4q16oddPseudo,
4955 ARM::VLD4q32oddPseudo };
4956 SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
4957 return;
4958 }
4959
4960 case Intrinsic::arm_neon_vld2dup: {
4961 static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
4962 ARM::VLD2DUPd32, ARM::VLD1q64 };
4963 static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
4964 ARM::VLD2DUPq16EvenPseudo,
4965 ARM::VLD2DUPq32EvenPseudo };
4966 static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
4967 ARM::VLD2DUPq16OddPseudo,
4968 ARM::VLD2DUPq32OddPseudo };
4969 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2,
4970 DOpcodes, QOpcodes0, QOpcodes1);
4971 return;
4972 }
4973
4974 case Intrinsic::arm_neon_vld3dup: {
4975 static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
4976 ARM::VLD3DUPd16Pseudo,
4977 ARM::VLD3DUPd32Pseudo,
4978 ARM::VLD1d64TPseudo };
4979 static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
4980 ARM::VLD3DUPq16EvenPseudo,
4981 ARM::VLD3DUPq32EvenPseudo };
4982 static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
4983 ARM::VLD3DUPq16OddPseudo,
4984 ARM::VLD3DUPq32OddPseudo };
4985 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3,
4986 DOpcodes, QOpcodes0, QOpcodes1);
4987 return;
4988 }
4989
4990 case Intrinsic::arm_neon_vld4dup: {
4991 static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
4992 ARM::VLD4DUPd16Pseudo,
4993 ARM::VLD4DUPd32Pseudo,
4994 ARM::VLD1d64QPseudo };
4995 static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
4996 ARM::VLD4DUPq16EvenPseudo,
4997 ARM::VLD4DUPq32EvenPseudo };
4998 static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
4999 ARM::VLD4DUPq16OddPseudo,
5000 ARM::VLD4DUPq32OddPseudo };
5001 SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4,
5002 DOpcodes, QOpcodes0, QOpcodes1);
5003 return;
5004 }
5005
5006 case Intrinsic::arm_neon_vld2lane: {
5007 static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
5008 ARM::VLD2LNd16Pseudo,
5009 ARM::VLD2LNd32Pseudo };
5010 static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
5011 ARM::VLD2LNq32Pseudo };
5012 SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
5013 return;
5014 }
5015
5016 case Intrinsic::arm_neon_vld3lane: {
5017 static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
5018 ARM::VLD3LNd16Pseudo,
5019 ARM::VLD3LNd32Pseudo };
5020 static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
5021 ARM::VLD3LNq32Pseudo };
5022 SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
5023 return;
5024 }
5025
5026 case Intrinsic::arm_neon_vld4lane: {
5027 static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
5028 ARM::VLD4LNd16Pseudo,
5029 ARM::VLD4LNd32Pseudo };
5030 static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
5031 ARM::VLD4LNq32Pseudo };
5032 SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
5033 return;
5034 }
5035
5036 case Intrinsic::arm_neon_vst1: {
5037 static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
5038 ARM::VST1d32, ARM::VST1d64 };
5039 static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5040 ARM::VST1q32, ARM::VST1q64 };
5041 SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
5042 return;
5043 }
5044
5045 case Intrinsic::arm_neon_vst1x2: {
5046 static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
5047 ARM::VST1q32, ARM::VST1q64 };
5048 static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
5049 ARM::VST1d16QPseudo,
5050 ARM::VST1d32QPseudo,
5051 ARM::VST1d64QPseudo };
5052 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
5053 return;
5054 }
5055
5056 case Intrinsic::arm_neon_vst1x3: {
5057 static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
5058 ARM::VST1d16TPseudo,
5059 ARM::VST1d32TPseudo,
5060 ARM::VST1d64TPseudo };
5061 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
5062 ARM::VST1q16LowTPseudo_UPD,
5063 ARM::VST1q32LowTPseudo_UPD,
5064 ARM::VST1q64LowTPseudo_UPD };
5065 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
5066 ARM::VST1q16HighTPseudo,
5067 ARM::VST1q32HighTPseudo,
5068 ARM::VST1q64HighTPseudo };
5069 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5070 return;
5071 }
5072
5073 case Intrinsic::arm_neon_vst1x4: {
5074 static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
5075 ARM::VST1d16QPseudo,
5076 ARM::VST1d32QPseudo,
5077 ARM::VST1d64QPseudo };
5078 static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
5079 ARM::VST1q16LowQPseudo_UPD,
5080 ARM::VST1q32LowQPseudo_UPD,
5081 ARM::VST1q64LowQPseudo_UPD };
5082 static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
5083 ARM::VST1q16HighQPseudo,
5084 ARM::VST1q32HighQPseudo,
5085 ARM::VST1q64HighQPseudo };
5086 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5087 return;
5088 }
5089
5090 case Intrinsic::arm_neon_vst2: {
5091 static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
5092 ARM::VST2d32, ARM::VST1q64 };
5093 static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
5094 ARM::VST2q32Pseudo };
5095 SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
5096 return;
5097 }
5098
5099 case Intrinsic::arm_neon_vst3: {
5100 static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
5101 ARM::VST3d16Pseudo,
5102 ARM::VST3d32Pseudo,
5103 ARM::VST1d64TPseudo };
5104 static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
5105 ARM::VST3q16Pseudo_UPD,
5106 ARM::VST3q32Pseudo_UPD };
5107 static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
5108 ARM::VST3q16oddPseudo,
5109 ARM::VST3q32oddPseudo };
5110 SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
5111 return;
5112 }
5113
5114 case Intrinsic::arm_neon_vst4: {
5115 static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
5116 ARM::VST4d16Pseudo,
5117 ARM::VST4d32Pseudo,
5118 ARM::VST1d64QPseudo };
5119 static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
5120 ARM::VST4q16Pseudo_UPD,
5121 ARM::VST4q32Pseudo_UPD };
5122 static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
5123 ARM::VST4q16oddPseudo,
5124 ARM::VST4q32oddPseudo };
5125 SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
5126 return;
5127 }
5128
5129 case Intrinsic::arm_neon_vst2lane: {
5130 static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
5131 ARM::VST2LNd16Pseudo,
5132 ARM::VST2LNd32Pseudo };
5133 static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
5134 ARM::VST2LNq32Pseudo };
5135 SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
5136 return;
5137 }
5138
5139 case Intrinsic::arm_neon_vst3lane: {
5140 static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
5141 ARM::VST3LNd16Pseudo,
5142 ARM::VST3LNd32Pseudo };
5143 static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
5144 ARM::VST3LNq32Pseudo };
5145 SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
5146 return;
5147 }
5148
5149 case Intrinsic::arm_neon_vst4lane: {
5150 static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
5151 ARM::VST4LNd16Pseudo,
5152 ARM::VST4LNd32Pseudo };
5153 static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
5154 ARM::VST4LNq32Pseudo };
5155 SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
5156 return;
5157 }
5158
5159 case Intrinsic::arm_mve_vldr_gather_base_wb:
5160 case Intrinsic::arm_mve_vldr_gather_base_wb_predicated: {
5161 static const uint16_t Opcodes[] = {ARM::MVE_VLDRWU32_qi_pre,
5162 ARM::MVE_VLDRDU64_qi_pre};
5163 SelectMVE_WB(N, Opcodes,
5164 IntNo == Intrinsic::arm_mve_vldr_gather_base_wb_predicated);
5165 return;
5166 }
5167
5168 case Intrinsic::arm_mve_vld2q: {
5169 static const uint16_t Opcodes8[] = {ARM::MVE_VLD20_8, ARM::MVE_VLD21_8};
5170 static const uint16_t Opcodes16[] = {ARM::MVE_VLD20_16,
5171 ARM::MVE_VLD21_16};
5172 static const uint16_t Opcodes32[] = {ARM::MVE_VLD20_32,
5173 ARM::MVE_VLD21_32};
5174 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5175 SelectMVE_VLD(N, 2, Opcodes, false);
5176 return;
5177 }
5178
5179 case Intrinsic::arm_mve_vld4q: {
5180 static const uint16_t Opcodes8[] = {ARM::MVE_VLD40_8, ARM::MVE_VLD41_8,
5181 ARM::MVE_VLD42_8, ARM::MVE_VLD43_8};
5182 static const uint16_t Opcodes16[] = {ARM::MVE_VLD40_16, ARM::MVE_VLD41_16,
5183 ARM::MVE_VLD42_16,
5184 ARM::MVE_VLD43_16};
5185 static const uint16_t Opcodes32[] = {ARM::MVE_VLD40_32, ARM::MVE_VLD41_32,
5186 ARM::MVE_VLD42_32,
5187 ARM::MVE_VLD43_32};
5188 static const uint16_t *const Opcodes[] = {Opcodes8, Opcodes16, Opcodes32};
5189 SelectMVE_VLD(N, 4, Opcodes, false);
5190 return;
5191 }
5192 }
5193 break;
5194 }
5195
5196 case ISD::INTRINSIC_WO_CHAIN: {
5197 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
5198 switch (IntNo) {
5199 default:
5200 break;
5201
5202 // Scalar f32 -> bf16
5203 case Intrinsic::arm_neon_vcvtbfp2bf: {
5204 SDLoc dl(N);
5205 const SDValue &Src = N->getOperand(1);
5206 llvm::EVT DestTy = N->getValueType(0);
5207 SDValue Pred = getAL(CurDAG, dl);
5208 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5209 SDValue Ops[] = { Src, Src, Pred, Reg0 };
5210 CurDAG->SelectNodeTo(N, ARM::BF16_VCVTB, DestTy, Ops);
5211 return;
5212 }
5213
5214 // Vector v4f32 -> v4bf16
5215 case Intrinsic::arm_neon_vcvtfp2bf: {
5216 SDLoc dl(N);
5217 const SDValue &Src = N->getOperand(1);
5218 SDValue Pred = getAL(CurDAG, dl);
5219 SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
5220 SDValue Ops[] = { Src, Pred, Reg0 };
5221 CurDAG->SelectNodeTo(N, ARM::BF16_VCVT, MVT::v4bf16, Ops);
5222 return;
5223 }
5224
5225 case Intrinsic::arm_mve_urshrl:
5226 SelectMVE_LongShift(N, ARM::MVE_URSHRL, true, false);
5227 return;
5228 case Intrinsic::arm_mve_uqshll:
5229 SelectMVE_LongShift(N, ARM::MVE_UQSHLL, true, false);
5230 return;
5231 case Intrinsic::arm_mve_srshrl:
5232 SelectMVE_LongShift(N, ARM::MVE_SRSHRL, true, false);
5233 return;
5234 case Intrinsic::arm_mve_sqshll:
5235 SelectMVE_LongShift(N, ARM::MVE_SQSHLL, true, false);
5236 return;
5237 case Intrinsic::arm_mve_uqrshll:
5238 SelectMVE_LongShift(N, ARM::MVE_UQRSHLL, false, true);
5239 return;
5240 case Intrinsic::arm_mve_sqrshrl:
5241 SelectMVE_LongShift(N, ARM::MVE_SQRSHRL, false, true);
5242 return;
5243
5244 case Intrinsic::arm_mve_vadc:
5245 case Intrinsic::arm_mve_vadc_predicated:
5246 SelectMVE_VADCSBC(N, ARM::MVE_VADC, ARM::MVE_VADCI, true,
5247 IntNo == Intrinsic::arm_mve_vadc_predicated);
5248 return;
5249 case Intrinsic::arm_mve_vsbc:
5250 case Intrinsic::arm_mve_vsbc_predicated:
5251 SelectMVE_VADCSBC(N, ARM::MVE_VSBC, ARM::MVE_VSBCI, true,
5252 IntNo == Intrinsic::arm_mve_vsbc_predicated);
5253 return;
5254 case Intrinsic::arm_mve_vshlc:
5255 case Intrinsic::arm_mve_vshlc_predicated:
5256 SelectMVE_VSHLC(N, IntNo == Intrinsic::arm_mve_vshlc_predicated);
5257 return;
5258
5259 case Intrinsic::arm_mve_vmlldava:
5260 case Intrinsic::arm_mve_vmlldava_predicated: {
5261 static const uint16_t OpcodesU[] = {
5262 ARM::MVE_VMLALDAVu16, ARM::MVE_VMLALDAVu32,
5263 ARM::MVE_VMLALDAVau16, ARM::MVE_VMLALDAVau32,
5264 };
5265 static const uint16_t OpcodesS[] = {
5266 ARM::MVE_VMLALDAVs16, ARM::MVE_VMLALDAVs32,
5267 ARM::MVE_VMLALDAVas16, ARM::MVE_VMLALDAVas32,
5268 ARM::MVE_VMLALDAVxs16, ARM::MVE_VMLALDAVxs32,
5269 ARM::MVE_VMLALDAVaxs16, ARM::MVE_VMLALDAVaxs32,
5270 ARM::MVE_VMLSLDAVs16, ARM::MVE_VMLSLDAVs32,
5271 ARM::MVE_VMLSLDAVas16, ARM::MVE_VMLSLDAVas32,
5272 ARM::MVE_VMLSLDAVxs16, ARM::MVE_VMLSLDAVxs32,
5273 ARM::MVE_VMLSLDAVaxs16, ARM::MVE_VMLSLDAVaxs32,
5274 };
5275 SelectMVE_VMLLDAV(N, IntNo == Intrinsic::arm_mve_vmlldava_predicated,
5276 OpcodesS, OpcodesU);
5277 return;
5278 }
5279
5280 case Intrinsic::arm_mve_vrmlldavha:
5281 case Intrinsic::arm_mve_vrmlldavha_predicated: {
5282 static const uint16_t OpcodesU[] = {
5283 ARM::MVE_VRMLALDAVHu32, ARM::MVE_VRMLALDAVHau32,
5284 };
5285 static const uint16_t OpcodesS[] = {
5286 ARM::MVE_VRMLALDAVHs32, ARM::MVE_VRMLALDAVHas32,
5287 ARM::MVE_VRMLALDAVHxs32, ARM::MVE_VRMLALDAVHaxs32,
5288 ARM::MVE_VRMLSLDAVHs32, ARM::MVE_VRMLSLDAVHas32,
5289 ARM::MVE_VRMLSLDAVHxs32, ARM::MVE_VRMLSLDAVHaxs32,
5290 };
5291 SelectMVE_VRMLLDAVH(N, IntNo == Intrinsic::arm_mve_vrmlldavha_predicated,
5292 OpcodesS, OpcodesU);
5293 return;
5294 }
5295
5296 case Intrinsic::arm_mve_vidup:
5297 case Intrinsic::arm_mve_vidup_predicated: {
5298 static const uint16_t Opcodes[] = {
5299 ARM::MVE_VIDUPu8, ARM::MVE_VIDUPu16, ARM::MVE_VIDUPu32,
5300 };
5301 SelectMVE_VxDUP(N, Opcodes, false,
5302 IntNo == Intrinsic::arm_mve_vidup_predicated);
5303 return;
5304 }
5305
5306 case Intrinsic::arm_mve_vddup:
5307 case Intrinsic::arm_mve_vddup_predicated: {
5308 static const uint16_t Opcodes[] = {
5309 ARM::MVE_VDDUPu8, ARM::MVE_VDDUPu16, ARM::MVE_VDDUPu32,
5310 };
5311 SelectMVE_VxDUP(N, Opcodes, false,
5312 IntNo == Intrinsic::arm_mve_vddup_predicated);
5313 return;
5314 }
5315
5316 case Intrinsic::arm_mve_viwdup:
5317 case Intrinsic::arm_mve_viwdup_predicated: {
5318 static const uint16_t Opcodes[] = {
5319 ARM::MVE_VIWDUPu8, ARM::MVE_VIWDUPu16, ARM::MVE_VIWDUPu32,
5320 };
5321 SelectMVE_VxDUP(N, Opcodes, true,
5322 IntNo == Intrinsic::arm_mve_viwdup_predicated);
5323 return;
5324 }
5325
5326 case Intrinsic::arm_mve_vdwdup:
5327 case Intrinsic::arm_mve_vdwdup_predicated: {
5328 static const uint16_t Opcodes[] = {
5329 ARM::MVE_VDWDUPu8, ARM::MVE_VDWDUPu16, ARM::MVE_VDWDUPu32,
5330 };
5331 SelectMVE_VxDUP(N, Opcodes, true,
5332 IntNo == Intrinsic::arm_mve_vdwdup_predicated);
5333 return;
5334 }
5335
5336 case Intrinsic::arm_cde_cx1d:
5337 case Intrinsic::arm_cde_cx1da:
5338 case Intrinsic::arm_cde_cx2d:
5339 case Intrinsic::arm_cde_cx2da:
5340 case Intrinsic::arm_cde_cx3d:
5341 case Intrinsic::arm_cde_cx3da: {
5342 bool HasAccum = IntNo == Intrinsic::arm_cde_cx1da ||
5343 IntNo == Intrinsic::arm_cde_cx2da ||
5344 IntNo == Intrinsic::arm_cde_cx3da;
5345 size_t NumExtraOps;
5346 uint16_t Opcode;
5347 switch (IntNo) {
5348 case Intrinsic::arm_cde_cx1d:
5349 case Intrinsic::arm_cde_cx1da:
5350 NumExtraOps = 0;
5351 Opcode = HasAccum ? ARM::CDE_CX1DA : ARM::CDE_CX1D;
5352 break;
5353 case Intrinsic::arm_cde_cx2d:
5354 case Intrinsic::arm_cde_cx2da:
5355 NumExtraOps = 1;
5356 Opcode = HasAccum ? ARM::CDE_CX2DA : ARM::CDE_CX2D;
5357 break;
5358 case Intrinsic::arm_cde_cx3d:
5359 case Intrinsic::arm_cde_cx3da:
5360 NumExtraOps = 2;
5361 Opcode = HasAccum ? ARM::CDE_CX3DA : ARM::CDE_CX3D;
5362 break;
5363 default:
5364 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5364);
5365 }
5366 SelectCDE_CXxD(N, Opcode, NumExtraOps, HasAccum);
5367 return;
5368 }
5369 }
5370 break;
5371 }
5372
5373 case ISD::ATOMIC_CMP_SWAP:
5374 SelectCMP_SWAP(N);
5375 return;
5376 }
5377
5378 SelectCode(N);
5379}
5380
5381// Inspect a register string of the form
5382// cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
5383// cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
5384// and obtain the integer operands from them, adding these operands to the
5385// provided vector.
5386static void getIntOperandsFromRegisterString(StringRef RegString,
5387 SelectionDAG *CurDAG,
5388 const SDLoc &DL,
5389 std::vector<SDValue> &Ops) {
5390 SmallVector<StringRef, 5> Fields;
5391 RegString.split(Fields, ':');
5392
5393 if (Fields.size() > 1) {
5394 bool AllIntFields = true;
5395
5396 for (StringRef Field : Fields) {
5397 // Need to trim out leading 'cp' characters and get the integer field.
5398 unsigned IntField;
5399 AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
5400 Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
5401 }
5402
5403 assert(AllIntFields &&(static_cast <bool> (AllIntFields && "Unexpected non-integer value in special register string."
) ? void (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5404, __extension__ __PRETTY_FUNCTION__))
5404 "Unexpected non-integer value in special register string.")(static_cast <bool> (AllIntFields && "Unexpected non-integer value in special register string."
) ? void (0) : __assert_fail ("AllIntFields && \"Unexpected non-integer value in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5404, __extension__ __PRETTY_FUNCTION__));
5405 (void)AllIntFields;
5406 }
5407}
5408
5409// Maps a Banked Register string to its mask value. The mask value returned is
5410// for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
5411// mask operand, which expresses which register is to be used, e.g. r8, and in
5412// which mode it is to be used, e.g. usr. Returns -1 to signify that the string
5413// was invalid.
5414static inline int getBankedRegisterMask(StringRef RegString) {
5415 auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
5416 if (!TheReg)
5417 return -1;
5418 return TheReg->Encoding;
5419}
5420
5421// The flags here are common to those allowed for apsr in the A class cores and
5422// those allowed for the special registers in the M class cores. Returns a
5423// value representing which flags were present, -1 if invalid.
5424static inline int getMClassFlagsMask(StringRef Flags) {
5425 return StringSwitch<int>(Flags)
5426 .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
5427 // correct when flags are not permitted
5428 .Case("g", 0x1)
5429 .Case("nzcvq", 0x2)
5430 .Case("nzcvqg", 0x3)
5431 .Default(-1);
5432}
5433
5434// Maps MClass special registers string to its value for use in the
5435// t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
5436// Returns -1 to signify that the string was invalid.
5437static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
5438 auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
5439 const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
5440 if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
5441 return -1;
5442 return (int)(TheReg->Encoding & 0xFFF); // SYSm value
5443}
5444
5445static int getARClassRegisterMask(StringRef Reg, StringRef Flags) {
5446 // The mask operand contains the special register (R Bit) in bit 4, whether
5447 // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
5448 // bits 3-0 contains the fields to be accessed in the special register, set by
5449 // the flags provided with the register.
5450 int Mask = 0;
5451 if (Reg == "apsr") {
5452 // The flags permitted for apsr are the same flags that are allowed in
5453 // M class registers. We get the flag value and then shift the flags into
5454 // the correct place to combine with the mask.
5455 Mask = getMClassFlagsMask(Flags);
5456 if (Mask == -1)
5457 return -1;
5458 return Mask << 2;
5459 }
5460
5461 if (Reg != "cpsr" && Reg != "spsr") {
5462 return -1;
5463 }
5464
5465 // This is the same as if the flags were "fc"
5466 if (Flags.empty() || Flags == "all")
5467 return Mask | 0x9;
5468
5469 // Inspect the supplied flags string and set the bits in the mask for
5470 // the relevant and valid flags allowed for cpsr and spsr.
5471 for (char Flag : Flags) {
5472 int FlagVal;
5473 switch (Flag) {
5474 case 'c':
5475 FlagVal = 0x1;
5476 break;
5477 case 'x':
5478 FlagVal = 0x2;
5479 break;
5480 case 's':
5481 FlagVal = 0x4;
5482 break;
5483 case 'f':
5484 FlagVal = 0x8;
5485 break;
5486 default:
5487 FlagVal = 0;
5488 }
5489
5490 // This avoids allowing strings where the same flag bit appears twice.
5491 if (!FlagVal || (Mask & FlagVal))
5492 return -1;
5493 Mask |= FlagVal;
5494 }
5495
5496 // If the register is spsr then we need to set the R bit.
5497 if (Reg == "spsr")
5498 Mask |= 0x10;
5499
5500 return Mask;
5501}
5502
5503// Lower the read_register intrinsic to ARM specific DAG nodes
5504// using the supplied metadata string to select the instruction node to use
5505// and the registers/masks to construct as operands for the node.
5506bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
5507 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
5508 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5509 bool IsThumb2 = Subtarget->isThumb2();
5510 SDLoc DL(N);
5511
5512 std::vector<SDValue> Ops;
5513 getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
5514
5515 if (!Ops.empty()) {
5516 // If the special register string was constructed of fields (as defined
5517 // in the ACLE) then need to lower to MRC node (32 bit) or
5518 // MRRC node(64 bit), we can make the distinction based on the number of
5519 // operands we have.
5520 unsigned Opcode;
5521 SmallVector<EVT, 3> ResTypes;
5522 if (Ops.size() == 5){
5523 Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
5524 ResTypes.append({ MVT::i32, MVT::Other });
5525 } else {
5526 assert(Ops.size() == 3 &&(static_cast <bool> (Ops.size() == 3 && "Invalid number of fields in special register string."
) ? void (0) : __assert_fail ("Ops.size() == 3 && \"Invalid number of fields in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5527, __extension__ __PRETTY_FUNCTION__))
5527 "Invalid number of fields in special register string.")(static_cast <bool> (Ops.size() == 3 && "Invalid number of fields in special register string."
) ? void (0) : __assert_fail ("Ops.size() == 3 && \"Invalid number of fields in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5527, __extension__ __PRETTY_FUNCTION__));
5528 Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
5529 ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
5530 }
5531
5532 Ops.push_back(getAL(CurDAG, DL));
5533 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5534 Ops.push_back(N->getOperand(0));
5535 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
5536 return true;
5537 }
5538
5539 std::string SpecialReg = RegString->getString().lower();
5540
5541 int BankedReg = getBankedRegisterMask(SpecialReg);
5542 if (BankedReg != -1) {
5543 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
5544 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5545 N->getOperand(0) };
5546 ReplaceNode(
5547 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
5548 DL, MVT::i32, MVT::Other, Ops));
5549 return true;
5550 }
5551
5552 // The VFP registers are read by creating SelectionDAG nodes with opcodes
5553 // corresponding to the register that is being read from. So we switch on the
5554 // string to find which opcode we need to use.
5555 unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
5556 .Case("fpscr", ARM::VMRS)
5557 .Case("fpexc", ARM::VMRS_FPEXC)
5558 .Case("fpsid", ARM::VMRS_FPSID)
5559 .Case("mvfr0", ARM::VMRS_MVFR0)
5560 .Case("mvfr1", ARM::VMRS_MVFR1)
5561 .Case("mvfr2", ARM::VMRS_MVFR2)
5562 .Case("fpinst", ARM::VMRS_FPINST)
5563 .Case("fpinst2", ARM::VMRS_FPINST2)
5564 .Default(0);
5565
5566 // If an opcode was found then we can lower the read to a VFP instruction.
5567 if (Opcode) {
5568 if (!Subtarget->hasVFP2Base())
5569 return false;
5570 if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8Base())
5571 return false;
5572
5573 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5574 N->getOperand(0) };
5575 ReplaceNode(N,
5576 CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
5577 return true;
5578 }
5579
5580 // If the target is M Class then need to validate that the register string
5581 // is an acceptable value, so check that a mask can be constructed from the
5582 // string.
5583 if (Subtarget->isMClass()) {
5584 int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
5585 if (SYSmValue == -1)
5586 return false;
5587
5588 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
5589 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5590 N->getOperand(0) };
5591 ReplaceNode(
5592 N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
5593 return true;
5594 }
5595
5596 // Here we know the target is not M Class so we need to check if it is one
5597 // of the remaining possible values which are apsr, cpsr or spsr.
5598 if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
5599 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5600 N->getOperand(0) };
5601 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
5602 DL, MVT::i32, MVT::Other, Ops));
5603 return true;
5604 }
5605
5606 if (SpecialReg == "spsr") {
5607 Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5608 N->getOperand(0) };
5609 ReplaceNode(
5610 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
5611 MVT::i32, MVT::Other, Ops));
5612 return true;
5613 }
5614
5615 return false;
5616}
5617
5618// Lower the write_register intrinsic to ARM specific DAG nodes
5619// using the supplied metadata string to select the instruction node to use
5620// and the registers/masks to use in the nodes
5621bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
5622 const auto *MD = cast<MDNodeSDNode>(N->getOperand(1));
5623 const auto *RegString = cast<MDString>(MD->getMD()->getOperand(0));
5624 bool IsThumb2 = Subtarget->isThumb2();
5625 SDLoc DL(N);
5626
5627 std::vector<SDValue> Ops;
5628 getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
5629
5630 if (!Ops.empty()) {
5631 // If the special register string was constructed of fields (as defined
5632 // in the ACLE) then need to lower to MCR node (32 bit) or
5633 // MCRR node(64 bit), we can make the distinction based on the number of
5634 // operands we have.
5635 unsigned Opcode;
5636 if (Ops.size() == 5) {
5637 Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
5638 Ops.insert(Ops.begin()+2, N->getOperand(2));
5639 } else {
5640 assert(Ops.size() == 3 &&(static_cast <bool> (Ops.size() == 3 && "Invalid number of fields in special register string."
) ? void (0) : __assert_fail ("Ops.size() == 3 && \"Invalid number of fields in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5641, __extension__ __PRETTY_FUNCTION__))
5641 "Invalid number of fields in special register string.")(static_cast <bool> (Ops.size() == 3 && "Invalid number of fields in special register string."
) ? void (0) : __assert_fail ("Ops.size() == 3 && \"Invalid number of fields in special register string.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5641, __extension__ __PRETTY_FUNCTION__));
5642 Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
5643 SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
5644 Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
5645 }
5646
5647 Ops.push_back(getAL(CurDAG, DL));
5648 Ops.push_back(CurDAG->getRegister(0, MVT::i32));
5649 Ops.push_back(N->getOperand(0));
5650
5651 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
5652 return true;
5653 }
5654
5655 std::string SpecialReg = RegString->getString().lower();
5656 int BankedReg = getBankedRegisterMask(SpecialReg);
5657 if (BankedReg != -1) {
5658 Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
5659 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5660 N->getOperand(0) };
5661 ReplaceNode(
5662 N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
5663 DL, MVT::Other, Ops));
5664 return true;
5665 }
5666
5667 // The VFP registers are written to by creating SelectionDAG nodes with
5668 // opcodes corresponding to the register that is being written. So we switch
5669 // on the string to find which opcode we need to use.
5670 unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
5671 .Case("fpscr", ARM::VMSR)
5672 .Case("fpexc", ARM::VMSR_FPEXC)
5673 .Case("fpsid", ARM::VMSR_FPSID)
5674 .Case("fpinst", ARM::VMSR_FPINST)
5675 .Case("fpinst2", ARM::VMSR_FPINST2)
5676 .Default(0);
5677
5678 if (Opcode) {
5679 if (!Subtarget->hasVFP2Base())
5680 return false;
5681 Ops = { N->getOperand(2), getAL(CurDAG, DL),
5682 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
5683 ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
5684 return true;
5685 }
5686
5687 std::pair<StringRef, StringRef> Fields;
5688 Fields = StringRef(SpecialReg).rsplit('_');
5689 std::string Reg = Fields.first.str();
5690 StringRef Flags = Fields.second;
5691
5692 // If the target was M Class then need to validate the special register value
5693 // and retrieve the mask for use in the instruction node.
5694 if (Subtarget->isMClass()) {
5695 int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
5696 if (SYSmValue == -1)
5697 return false;
5698
5699 SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
5700 N->getOperand(2), getAL(CurDAG, DL),
5701 CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
5702 ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
5703 return true;
5704 }
5705
5706 // We then check to see if a valid mask can be constructed for one of the
5707 // register string values permitted for the A and R class cores. These values
5708 // are apsr, spsr and cpsr; these are also valid on older cores.
5709 int Mask = getARClassRegisterMask(Reg, Flags);
5710 if (Mask != -1) {
5711 Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
5712 getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
5713 N->getOperand(0) };
5714 ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
5715 DL, MVT::Other, Ops));
5716 return true;
5717 }
5718
5719 return false;
5720}
5721
5722bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
5723 std::vector<SDValue> AsmNodeOperands;
5724 unsigned Flag, Kind;
5725 bool Changed = false;
5726 unsigned NumOps = N->getNumOperands();
5727
5728 // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
5729 // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
5730 // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
5731 // respectively. Since there is no constraint to explicitly specify a
5732 // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
5733 // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
5734 // them into a GPRPair.
5735
5736 SDLoc dl(N);
5737 SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
5738 : SDValue(nullptr,0);
5739
5740 SmallVector<bool, 8> OpChanged;
5741 // Glue node will be appended late.
5742 for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
5743 SDValue op = N->getOperand(i);
5744 AsmNodeOperands.push_back(op);
5745
5746 if (i < InlineAsm::Op_FirstOperand)
5747 continue;
5748
5749 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
5750 Flag = C->getZExtValue();
5751 Kind = InlineAsm::getKind(Flag);
5752 }
5753 else
5754 continue;
5755
5756 // Immediate operands to inline asm in the SelectionDAG are modeled with
5757 // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
5758 // the second is a constant with the value of the immediate. If we get here
5759 // and we have a Kind_Imm, skip the next operand, and continue.
5760 if (Kind == InlineAsm::Kind_Imm) {
5761 SDValue op = N->getOperand(++i);
5762 AsmNodeOperands.push_back(op);
5763 continue;
5764 }
5765
5766 unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
5767 if (NumRegs)
5768 OpChanged.push_back(false);
5769
5770 unsigned DefIdx = 0;
5771 bool IsTiedToChangedOp = false;
5772 // If it's a use that is tied with a previous def, it has no
5773 // reg class constraint.
5774 if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
5775 IsTiedToChangedOp = OpChanged[DefIdx];
5776
5777 // Memory operands to inline asm in the SelectionDAG are modeled with two
5778 // operands: a constant of value InlineAsm::Kind_Mem followed by the input
5779 // operand. If we get here and we have a Kind_Mem, skip the next operand (so
5780 // it doesn't get misinterpreted), and continue. We do this here because
5781 // it's important to update the OpChanged array correctly before moving on.
5782 if (Kind == InlineAsm::Kind_Mem) {
5783 SDValue op = N->getOperand(++i);
5784 AsmNodeOperands.push_back(op);
5785 continue;
5786 }
5787
5788 if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
5789 && Kind != InlineAsm::Kind_RegDefEarlyClobber)
5790 continue;
5791
5792 unsigned RC;
5793 bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
5794 if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
5795 || NumRegs != 2)
5796 continue;
5797
5798 assert((i+2 < NumOps) && "Invalid number of operands in inline asm")(static_cast <bool> ((i+2 < NumOps) && "Invalid number of operands in inline asm"
) ? void (0) : __assert_fail ("(i+2 < NumOps) && \"Invalid number of operands in inline asm\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5798, __extension__ __PRETTY_FUNCTION__));
5799 SDValue V0 = N->getOperand(i+1);
5800 SDValue V1 = N->getOperand(i+2);
5801 unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
5802 unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
5803 SDValue PairedReg;
5804 MachineRegisterInfo &MRI = MF->getRegInfo();
5805
5806 if (Kind == InlineAsm::Kind_RegDef ||
5807 Kind == InlineAsm::Kind_RegDefEarlyClobber) {
5808 // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
5809 // the original GPRs.
5810
5811 Register GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5812 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5813 SDValue Chain = SDValue(N,0);
5814
5815 SDNode *GU = N->getGluedUser();
5816 SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
5817 Chain.getValue(1));
5818
5819 // Extract values from a GPRPair reg and copy to the original GPR reg.
5820 SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
5821 RegCopy);
5822 SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
5823 RegCopy);
5824 SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
5825 RegCopy.getValue(1));
5826 SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
5827
5828 // Update the original glue user.
5829 std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
5830 Ops.push_back(T1.getValue(1));
5831 CurDAG->UpdateNodeOperands(GU, Ops);
5832 }
5833 else {
5834 // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a
5835 // GPRPair and then pass the GPRPair to the inline asm.
5836 SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
5837
5838 // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
5839 SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
5840 Chain.getValue(1));
5841 SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
5842 T0.getValue(1));
5843 SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
5844
5845 // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
5846 // i32 VRs of inline asm with it.
5847 Register GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
5848 PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
5849 Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
5850
5851 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
5852 Glue = Chain.getValue(1);
5853 }
5854
5855 Changed = true;
5856
5857 if(PairedReg.getNode()) {
5858 OpChanged[OpChanged.size() -1 ] = true;
5859 Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
5860 if (IsTiedToChangedOp)
5861 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
5862 else
5863 Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
5864 // Replace the current flag.
5865 AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
5866 Flag, dl, MVT::i32);
5867 // Add the new register node and skip the original two GPRs.
5868 AsmNodeOperands.push_back(PairedReg);
5869 // Skip the next two GPRs.
5870 i += 2;
5871 }
5872 }
5873
5874 if (Glue.getNode())
5875 AsmNodeOperands.push_back(Glue);
5876 if (!Changed)
5877 return false;
5878
5879 SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
5880 CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
5881 New->setNodeId(-1);
5882 ReplaceNode(N, New.getNode());
5883 return true;
5884}
5885
5886
5887bool ARMDAGToDAGISel::
5888SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
5889 std::vector<SDValue> &OutOps) {
5890 switch(ConstraintID) {
5891 default:
5892 llvm_unreachable("Unexpected asm memory constraint")::llvm::llvm_unreachable_internal("Unexpected asm memory constraint"
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp"
, 5892);
5893 case InlineAsm::Constraint_m:
5894 case InlineAsm::Constraint_o:
5895 case InlineAsm::Constraint_Q:
5896 case InlineAsm::Constraint_Um:
5897 case InlineAsm::Constraint_Un:
5898 case InlineAsm::Constraint_Uq:
5899 case InlineAsm::Constraint_Us:
5900 case InlineAsm::Constraint_Ut:
5901 case InlineAsm::Constraint_Uv:
5902 case InlineAsm::Constraint_Uy:
5903 // Require the address to be in a register. That is safe for all ARM
5904 // variants and it is hard to do anything much smarter without knowing
5905 // how the operand is used.
5906 OutOps.push_back(Op);
5907 return false;
5908 }
5909 return true;
5910}
5911
5912/// createARMISelDag - This pass converts a legalized DAG into a
5913/// ARM-specific DAG, ready for instruction scheduling.
5914///
5915FunctionPass *llvm::createARMISelDag(ARMBaseTargetMachine &TM,
5916 CodeGenOpt::Level OptLevel) {
5917 return new ARMDAGToDAGISel(TM, OptLevel);
5918}
←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h

→
1//===- CodeGen/ValueTypes.h - Low-Level Target independ. types --*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the set of low-level target independent types which various
10// values in the code generator are. This allows the target specific behavior
11// of instructions to be described to target independent passes.
12//
13//===----------------------------------------------------------------------===//
14
15#ifndef LLVM_CODEGEN_VALUETYPES_H
16#define LLVM_CODEGEN_VALUETYPES_H
17
18#include "llvm/Support/Compiler.h"
19#include "llvm/Support/MachineValueType.h"
20#include "llvm/Support/MathExtras.h"
21#include "llvm/Support/TypeSize.h"
22#include "llvm/Support/WithColor.h"
23#include <cassert>
24#include <cstdint>
25#include <string>
26
27namespace llvm {
28
29 class LLVMContext;
30 class Type;
31
32 /// Extended Value Type. Capable of holding value types which are not native
33 /// for any processor (such as the i12345 type), as well as the types an MVT
34 /// can represent.
35 struct EVT {
36 private:
37 MVT V = MVT::INVALID_SIMPLE_VALUE_TYPE;
38 Type *LLVMTy = nullptr;
39
40 public:
41 constexpr EVT() = default;
42 constexpr EVT(MVT::SimpleValueType SVT) : V(SVT) {}
43 constexpr EVT(MVT S) : V(S) {}
44
45 bool operator==(EVT VT) const {
46 return !(*this != VT);
47 }
48 bool operator!=(EVT VT) const {
49 if (V.SimpleTy != VT.V.SimpleTy)
50 return true;
51 if (V.SimpleTy == MVT::INVALID_SIMPLE_VALUE_TYPE)
52 return LLVMTy != VT.LLVMTy;
53 return false;
54 }
55
56 /// Returns the EVT that represents a floating-point type with the given
57 /// number of bits. There are two floating-point types with 128 bits - this
58 /// returns f128 rather than ppcf128.
59 static EVT getFloatingPointVT(unsigned BitWidth) {
60 return MVT::getFloatingPointVT(BitWidth);
61 }
62
63 /// Returns the EVT that represents an integer with the given number of
64 /// bits.
65 static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth) {
66 MVT M = MVT::getIntegerVT(BitWidth);
67 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
68 return M;
69 return getExtendedIntegerVT(Context, BitWidth);
70 }
71
72 /// Returns the EVT that represents a vector NumElements in length, where
73 /// each element is of type VT.
74 static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements,
75 bool IsScalable = false) {
76 MVT M = MVT::getVectorVT(VT.V, NumElements, IsScalable);
77 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
78 return M;
79 return getExtendedVectorVT(Context, VT, NumElements, IsScalable);
80 }
81
82 /// Returns the EVT that represents a vector EC.Min elements in length,
83 /// where each element is of type VT.
84 static EVT getVectorVT(LLVMContext &Context, EVT VT, ElementCount EC) {
85 MVT M = MVT::getVectorVT(VT.V, EC);
86 if (M.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE)
87 return M;
88 return getExtendedVectorVT(Context, VT, EC);
89 }
90
91 /// Return a vector with the same number of elements as this vector, but
92 /// with the element type converted to an integer type with the same
93 /// bitwidth.
94 EVT changeVectorElementTypeToInteger() const {
95 if (isSimple())
96 return getSimpleVT().changeVectorElementTypeToInteger();
97 return changeExtendedVectorElementTypeToInteger();
98 }
99
100 /// Return a VT for a vector type whose attributes match ourselves
101 /// with the exception of the element type that is chosen by the caller.
102 EVT changeVectorElementType(EVT EltVT) const {
103 if (isSimple()) {
104 assert(EltVT.isSimple() &&(static_cast <bool> (EltVT.isSimple() && "Can't change simple vector VT to have extended element VT"
) ? void (0) : __assert_fail ("EltVT.isSimple() && \"Can't change simple vector VT to have extended element VT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 105, __extension__ __PRETTY_FUNCTION__))
105 "Can't change simple vector VT to have extended element VT")(static_cast <bool> (EltVT.isSimple() && "Can't change simple vector VT to have extended element VT"
) ? void (0) : __assert_fail ("EltVT.isSimple() && \"Can't change simple vector VT to have extended element VT\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 105, __extension__ __PRETTY_FUNCTION__));
106 return getSimpleVT().changeVectorElementType(EltVT.getSimpleVT());
107 }
108 return changeExtendedVectorElementType(EltVT);
109 }
110
111 /// Return the type converted to an equivalently sized integer or vector
112 /// with integer element type. Similar to changeVectorElementTypeToInteger,
113 /// but also handles scalars.
114 EVT changeTypeToInteger() {
115 if (isVector())
116 return changeVectorElementTypeToInteger();
117
118 if (isSimple())
119 return getSimpleVT().changeTypeToInteger();
120 return changeExtendedTypeToInteger();
121 }
122
123 /// Test if the given EVT has zero size, this will fail if called on a
124 /// scalable type
125 bool isZeroSized() const {
126 return !isScalableVector() && getSizeInBits() == 0;
127 }
128
129 /// Test if the given EVT is simple (as opposed to being extended).
130 bool isSimple() const {
131 return V.SimpleTy != MVT::INVALID_SIMPLE_VALUE_TYPE;
132 }
133
134 /// Test if the given EVT is extended (as opposed to being simple).
135 bool isExtended() const {
136 return !isSimple();
137 }
138
139 /// Return true if this is a FP or a vector FP type.
140 bool isFloatingPoint() const {
141 return isSimple() ? V.isFloatingPoint() : isExtendedFloatingPoint();
142 }
143
144 /// Return true if this is an integer or a vector integer type.
145 bool isInteger() const {
146 return isSimple() ? V.isInteger() : isExtendedInteger();
147 }
148
149 /// Return true if this is an integer, but not a vector.
150 bool isScalarInteger() const {
151 return isSimple() ? V.isScalarInteger() : isExtendedScalarInteger();
152 }
153
154 /// Return true if this is a vector value type.
155 bool isVector() const {
156 return isSimple() ? V.isVector() : isExtendedVector();
4
←
'?' condition is false
→
5
←
Returning value, which participates in a condition later
→
157 }
158
159 /// Return true if this is a vector type where the runtime
160 /// length is machine dependent
161 bool isScalableVector() const {
162 return isSimple() ? V.isScalableVector() : isExtendedScalableVector();
163 }
164
165 bool isFixedLengthVector() const {
166 return isSimple() ? V.isFixedLengthVector()
167 : isExtendedFixedLengthVector();
168 }
169
170 /// Return true if this is a 16-bit vector type.
171 bool is16BitVector() const {
172 return isSimple() ? V.is16BitVector() : isExtended16BitVector();
173 }
174
175 /// Return true if this is a 32-bit vector type.
176 bool is32BitVector() const {
177 return isSimple() ? V.is32BitVector() : isExtended32BitVector();
178 }
179
180 /// Return true if this is a 64-bit vector type.
181 bool is64BitVector() const {
182 return isSimple() ? V.is64BitVector() : isExtended64BitVector();
183 }
184
185 /// Return true if this is a 128-bit vector type.
186 bool is128BitVector() const {
187 return isSimple() ? V.is128BitVector() : isExtended128BitVector();
188 }
189
190 /// Return true if this is a 256-bit vector type.
191 bool is256BitVector() const {
192 return isSimple() ? V.is256BitVector() : isExtended256BitVector();
193 }
194
195 /// Return true if this is a 512-bit vector type.
196 bool is512BitVector() const {
197 return isSimple() ? V.is512BitVector() : isExtended512BitVector();
198 }
199
200 /// Return true if this is a 1024-bit vector type.
201 bool is1024BitVector() const {
202 return isSimple() ? V.is1024BitVector() : isExtended1024BitVector();
203 }
204
205 /// Return true if this is a 2048-bit vector type.
206 bool is2048BitVector() const {
207 return isSimple() ? V.is2048BitVector() : isExtended2048BitVector();
208 }
209
210 /// Return true if this is an overloaded type for TableGen.
211 bool isOverloaded() const {
212 return (V==MVT::iAny || V==MVT::fAny || V==MVT::vAny || V==MVT::iPTRAny);
213 }
214
215 /// Return true if the bit size is a multiple of 8.
216 bool isByteSized() const {
217 return !isZeroSized() && getSizeInBits().isKnownMultipleOf(8);
218 }
219
220 /// Return true if the size is a power-of-two number of bytes.
221 bool isRound() const {
222 if (isScalableVector())
223 return false;
224 unsigned BitSize = getSizeInBits();
225 return BitSize >= 8 && !(BitSize & (BitSize - 1));
226 }
227
228 /// Return true if this has the same number of bits as VT.
229 bool bitsEq(EVT VT) const {
230 if (EVT::operator==(VT)) return true;
231 return getSizeInBits() == VT.getSizeInBits();
232 }
233
234 /// Return true if we know at compile time this has more bits than VT.
235 bool knownBitsGT(EVT VT) const {
236 return TypeSize::isKnownGT(getSizeInBits(), VT.getSizeInBits());
237 }
238
239 /// Return true if we know at compile time this has more than or the same
240 /// bits as VT.
241 bool knownBitsGE(EVT VT) const {
242 return TypeSize::isKnownGE(getSizeInBits(), VT.getSizeInBits());
243 }
244
245 /// Return true if we know at compile time this has fewer bits than VT.
246 bool knownBitsLT(EVT VT) const {
247 return TypeSize::isKnownLT(getSizeInBits(), VT.getSizeInBits());
248 }
249
250 /// Return true if we know at compile time this has fewer than or the same
251 /// bits as VT.
252 bool knownBitsLE(EVT VT) const {
253 return TypeSize::isKnownLE(getSizeInBits(), VT.getSizeInBits());
254 }
255
256 /// Return true if this has more bits than VT.
257 bool bitsGT(EVT VT) const {
258 if (EVT::operator==(VT)) return false;
259 assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 260, __extension__ __PRETTY_FUNCTION__))
260 "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 260, __extension__ __PRETTY_FUNCTION__));
261 return knownBitsGT(VT);
262 }
263
264 /// Return true if this has no less bits than VT.
265 bool bitsGE(EVT VT) const {
266 if (EVT::operator==(VT)) return true;
267 assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 268, __extension__ __PRETTY_FUNCTION__))
268 "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 268, __extension__ __PRETTY_FUNCTION__));
269 return knownBitsGE(VT);
270 }
271
272 /// Return true if this has less bits than VT.
273 bool bitsLT(EVT VT) const {
274 if (EVT::operator==(VT)) return false;
275 assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 276, __extension__ __PRETTY_FUNCTION__))
276 "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 276, __extension__ __PRETTY_FUNCTION__));
277 return knownBitsLT(VT);
278 }
279
280 /// Return true if this has no more bits than VT.
281 bool bitsLE(EVT VT) const {
282 if (EVT::operator==(VT)) return true;
283 assert(isScalableVector() == VT.isScalableVector() &&(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 284, __extension__ __PRETTY_FUNCTION__))
284 "Comparison between scalable and fixed types")(static_cast <bool> (isScalableVector() == VT.isScalableVector
() && "Comparison between scalable and fixed types") ?
void (0) : __assert_fail ("isScalableVector() == VT.isScalableVector() && \"Comparison between scalable and fixed types\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 284, __extension__ __PRETTY_FUNCTION__));
285 return knownBitsLE(VT);
286 }
287
288 /// Return the SimpleValueType held in the specified simple EVT.
289 MVT getSimpleVT() const {
290 assert(isSimple() && "Expected a SimpleValueType!")(static_cast <bool> (isSimple() && "Expected a SimpleValueType!"
) ? void (0) : __assert_fail ("isSimple() && \"Expected a SimpleValueType!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 290, __extension__ __PRETTY_FUNCTION__));
291 return V;
292 }
293
294 /// If this is a vector type, return the element type, otherwise return
295 /// this.
296 EVT getScalarType() const {
297 return isVector() ? getVectorElementType() : *this;
298 }
299
300 /// Given a vector type, return the type of each element.
301 EVT getVectorElementType() const {
302 assert(isVector() && "Invalid vector type!")(static_cast <bool> (isVector() && "Invalid vector type!"
) ? void (0) : __assert_fail ("isVector() && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 302, __extension__ __PRETTY_FUNCTION__));
303 if (isSimple())
304 return V.getVectorElementType();
305 return getExtendedVectorElementType();
306 }
307
308 /// Given a vector type, return the number of elements it contains.
309 unsigned getVectorNumElements() const {
310 assert(isVector() && "Invalid vector type!")(static_cast <bool> (isVector() && "Invalid vector type!"
) ? void (0) : __assert_fail ("isVector() && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 310, __extension__ __PRETTY_FUNCTION__));
311
312 if (isScalableVector())
313 llvm::reportInvalidSizeRequest(
314 "Possible incorrect use of EVT::getVectorNumElements() for "
315 "scalable vector. Scalable flag may be dropped, use "
316 "EVT::getVectorElementCount() instead");
317
318 return isSimple() ? V.getVectorNumElements()
319 : getExtendedVectorNumElements();
320 }
321
322 // Given a (possibly scalable) vector type, return the ElementCount
323 ElementCount getVectorElementCount() const {
324 assert((isVector()) && "Invalid vector type!")(static_cast <bool> ((isVector()) && "Invalid vector type!"
) ? void (0) : __assert_fail ("(isVector()) && \"Invalid vector type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 324, __extension__ __PRETTY_FUNCTION__));
325 if (isSimple())
326 return V.getVectorElementCount();
327
328 return getExtendedVectorElementCount();
329 }
330
331 /// Given a vector type, return the minimum number of elements it contains.
332 unsigned getVectorMinNumElements() const {
333 return getVectorElementCount().getKnownMinValue();
334 }
335
336 /// Return the size of the specified value type in bits.
337 ///
338 /// If the value type is a scalable vector type, the scalable property will
339 /// be set and the runtime size will be a positive integer multiple of the
340 /// base size.
341 TypeSize getSizeInBits() const {
342 if (isSimple())
343 return V.getSizeInBits();
344 return getExtendedSizeInBits();
345 }
346
347 /// Return the size of the specified fixed width value type in bits. The
348 /// function will assert if the type is scalable.
349 uint64_t getFixedSizeInBits() const {
350 return getSizeInBits().getFixedSize();
351 }
352
353 uint64_t getScalarSizeInBits() const {
354 return getScalarType().getSizeInBits().getFixedSize();
355 }
356
357 /// Return the number of bytes overwritten by a store of the specified value
358 /// type.
359 ///
360 /// If the value type is a scalable vector type, the scalable property will
361 /// be set and the runtime size will be a positive integer multiple of the
362 /// base size.
363 TypeSize getStoreSize() const {
364 TypeSize BaseSize = getSizeInBits();
365 return {(BaseSize.getKnownMinSize() + 7) / 8, BaseSize.isScalable()};
366 }
367
368 /// Return the number of bits overwritten by a store of the specified value
369 /// type.
370 ///
371 /// If the value type is a scalable vector type, the scalable property will
372 /// be set and the runtime size will be a positive integer multiple of the
373 /// base size.
374 TypeSize getStoreSizeInBits() const {
375 return getStoreSize() * 8;
376 }
377
378 /// Rounds the bit-width of the given integer EVT up to the nearest power of
379 /// two (and at least to eight), and returns the integer EVT with that
380 /// number of bits.
381 EVT getRoundIntegerType(LLVMContext &Context) const {
382 assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast <bool> (isInteger() && !isVector()
&& "Invalid integer type!") ? void (0) : __assert_fail
("isInteger() && !isVector() && \"Invalid integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 382, __extension__ __PRETTY_FUNCTION__));
383 unsigned BitWidth = getSizeInBits();
384 if (BitWidth <= 8)
385 return EVT(MVT::i8);
386 return getIntegerVT(Context, 1 << Log2_32_Ceil(BitWidth));
387 }
388
389 /// Finds the smallest simple value type that is greater than or equal to
390 /// half the width of this EVT. If no simple value type can be found, an
391 /// extended integer value type of half the size (rounded up) is returned.
392 EVT getHalfSizedIntegerVT(LLVMContext &Context) const {
393 assert(isInteger() && !isVector() && "Invalid integer type!")(static_cast <bool> (isInteger() && !isVector()
&& "Invalid integer type!") ? void (0) : __assert_fail
("isInteger() && !isVector() && \"Invalid integer type!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 393, __extension__ __PRETTY_FUNCTION__));
394 unsigned EVTSize = getSizeInBits();
395 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
396 IntVT <= MVT::LAST_INTEGER_VALUETYPE; ++IntVT) {
397 EVT HalfVT = EVT((MVT::SimpleValueType)IntVT);
398 if (HalfVT.getSizeInBits() * 2 >= EVTSize)
399 return HalfVT;
400 }
401 return getIntegerVT(Context, (EVTSize + 1) / 2);
402 }
403
404 /// Return a VT for an integer vector type with the size of the
405 /// elements doubled. The typed returned may be an extended type.
406 EVT widenIntegerVectorElementType(LLVMContext &Context) const {
407 EVT EltVT = getVectorElementType();
408 EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits());
409 return EVT::getVectorVT(Context, EltVT, getVectorElementCount());
410 }
411
412 // Return a VT for a vector type with the same element type but
413 // half the number of elements. The type returned may be an
414 // extended type.
415 EVT getHalfNumVectorElementsVT(LLVMContext &Context) const {
416 EVT EltVT = getVectorElementType();
417 auto EltCnt = getVectorElementCount();
418 assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!")(static_cast <bool> (EltCnt.isKnownEven() && "Splitting vector, but not in half!"
) ? void (0) : __assert_fail ("EltCnt.isKnownEven() && \"Splitting vector, but not in half!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/ValueTypes.h"
, 418, __extension__ __PRETTY_FUNCTION__));
419 return EVT::getVectorVT(Context, EltVT, EltCnt.divideCoefficientBy(2));
420 }
421
422 // Return a VT for a vector type with the same element type but
423 // double the number of elements. The type returned may be an
424 // extended type.
425 EVT getDoubleNumVectorElementsVT(LLVMContext &Context) const {
426 EVT EltVT = getVectorElementType();
427 auto EltCnt = getVectorElementCount();
428 return EVT::getVectorVT(Context, EltVT, EltCnt * 2);
429 }
430
431 /// Returns true if the given vector is a power of 2.
432 bool isPow2VectorType() const {
433 unsigned NElts = getVectorMinNumElements();
434 return !(NElts & (NElts - 1));
435 }
436
437 /// Widens the length of the given vector EVT up to the nearest power of 2
438 /// and returns that type.
439 EVT getPow2VectorType(LLVMContext &Context) const {
440 if (!isPow2VectorType()) {
441 ElementCount NElts = getVectorElementCount();
442 unsigned NewMinCount = 1 << Log2_32_Ceil(NElts.getKnownMinValue());
443 NElts = ElementCount::get(NewMinCount, NElts.isScalable());
444 return EVT::getVectorVT(Context, getVectorElementType(), NElts);
445 }
446 else {
447 return *this;
448 }
449 }
450
451 /// This function returns value type as a string, e.g. "i32".
452 std::string getEVTString() const;
453
454 /// This method returns an LLVM type corresponding to the specified EVT.
455 /// For integer types, this returns an unsigned type. Note that this will
456 /// abort for types that cannot be represented.
457 Type *getTypeForEVT(LLVMContext &Context) const;
458
459 /// Return the value type corresponding to the specified type.
460 /// This returns all pointers as iPTR. If HandleUnknown is true, unknown
461 /// types are returned as Other, otherwise they are invalid.
462 static EVT getEVT(Type *Ty, bool HandleUnknown = false);
463
464 intptr_t getRawBits() const {
465 if (isSimple())
466 return V.SimpleTy;
467 else
468 return (intptr_t)(LLVMTy);
469 }
470
471 /// A meaningless but well-behaved order, useful for constructing
472 /// containers.
473 struct compareRawBits {
474 bool operator()(EVT L, EVT R) const {
475 if (L.V.SimpleTy == R.V.SimpleTy)
476 return L.LLVMTy < R.LLVMTy;
477 else
478 return L.V.SimpleTy < R.V.SimpleTy;
479 }
480 };
481
482 private:
483 // Methods for handling the Extended-type case in functions above.
484 // These are all out-of-line to prevent users of this header file
485 // from having a dependency on Type.h.
486 EVT changeExtendedTypeToInteger() const;
487 EVT changeExtendedVectorElementType(EVT EltVT) const;
488 EVT changeExtendedVectorElementTypeToInteger() const;
489 static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth);
490 static EVT getExtendedVectorVT(LLVMContext &C, EVT VT, unsigned NumElements,
491 bool IsScalable);
492 static EVT getExtendedVectorVT(LLVMContext &Context, EVT VT,
493 ElementCount EC);
494 bool isExtendedFloatingPoint() const LLVM_READONLY__attribute__((__pure__));
495 bool isExtendedInteger() const LLVM_READONLY__attribute__((__pure__));
496 bool isExtendedScalarInteger() const LLVM_READONLY__attribute__((__pure__));
497 bool isExtendedVector() const LLVM_READONLY__attribute__((__pure__));
498 bool isExtended16BitVector() const LLVM_READONLY__attribute__((__pure__));
499 bool isExtended32BitVector() const LLVM_READONLY__attribute__((__pure__));
500 bool isExtended64BitVector() const LLVM_READONLY__attribute__((__pure__));
501 bool isExtended128BitVector() const LLVM_READONLY__attribute__((__pure__));
502 bool isExtended256BitVector() const LLVM_READONLY__attribute__((__pure__));
503 bool isExtended512BitVector() const LLVM_READONLY__attribute__((__pure__));
504 bool isExtended1024BitVector() const LLVM_READONLY__attribute__((__pure__));
505 bool isExtended2048BitVector() const LLVM_READONLY__attribute__((__pure__));
506 bool isExtendedFixedLengthVector() const LLVM_READONLY__attribute__((__pure__));
507 bool isExtendedScalableVector() const LLVM_READONLY__attribute__((__pure__));
508 EVT getExtendedVectorElementType() const;
509 unsigned getExtendedVectorNumElements() const LLVM_READONLY__attribute__((__pure__));
510 ElementCount getExtendedVectorElementCount() const LLVM_READONLY__attribute__((__pure__));
511 TypeSize getExtendedSizeInBits() const LLVM_READONLY__attribute__((__pure__));
512 };
513
514} // end namespace llvm
515
516#endif // LLVM_CODEGEN_VALUETYPES_H
←

/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
14
←
Assuming 'Node' is equal to 'O.Node'
→
15
←
Assuming 'ResNo' is equal to 'O.ResNo'
→
16
←
Returning the value 1, which participates in a condition later
→
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
13
←
Calling 'SDValue::operator=='
→
17
←
Returning from 'SDValue::operator=='
→
18
←
Returning zero, which participates in a condition later
→
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class VPLoadStoreSDNode;
513 friend class MaskedLoadStoreSDNode;
514 friend class MaskedGatherScatterSDNode;
515 friend class VPGatherScatterSDNode;
516
517 uint16_t : NumMemSDNodeBits;
518
519 // This storage is shared between disparate class hierarchies to hold an
520 // enumeration specific to the class hierarchy in use.
521 // LSBaseSDNode => enum ISD::MemIndexedMode
522 // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
523 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
524 // VPGatherScatterSDNode => enum ISD::MemIndexType
525 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
526 uint16_t AddressingMode : 3;
527 };
528 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
529
530 class LoadSDNodeBitfields {
531 friend class LoadSDNode;
532 friend class VPLoadSDNode;
533 friend class MaskedLoadSDNode;
534 friend class MaskedGatherSDNode;
535 friend class VPGatherSDNode;
536
537 uint16_t : NumLSBaseSDNodeBits;
538
539 uint16_t ExtTy : 2; // enum ISD::LoadExtType
540 uint16_t IsExpanding : 1;
541 };
542
543 class StoreSDNodeBitfields {
544 friend class StoreSDNode;
545 friend class VPStoreSDNode;
546 friend class MaskedStoreSDNode;
547 friend class MaskedScatterSDNode;
548 friend class VPScatterSDNode;
549
550 uint16_t : NumLSBaseSDNodeBits;
551
552 uint16_t IsTruncating : 1;
553 uint16_t IsCompressing : 1;
554 };
555
556 union {
557 char RawSDNodeBits[sizeof(uint16_t)];
558 SDNodeBitfields SDNodeBits;
559 ConstantSDNodeBitfields ConstantSDNodeBits;
560 MemSDNodeBitfields MemSDNodeBits;
561 LSBaseSDNodeBitfields LSBaseSDNodeBits;
562 LoadSDNodeBitfields LoadSDNodeBits;
563 StoreSDNodeBitfields StoreSDNodeBits;
564 };
565END_TWO_BYTE_PACK()
566#undef BEGIN_TWO_BYTE_PACK
567#undef END_TWO_BYTE_PACK
568
569 // RawSDNodeBits must cover the entirety of the union. This means that all of
570 // the union's members must have size <= RawSDNodeBits. We write the RHS as
571 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
572 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
573 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
574 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
575 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
576 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
577 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
578
579private:
580 friend class SelectionDAG;
581 // TODO: unfriend HandleSDNode once we fix its operand handling.
582 friend class HandleSDNode;
583
584 /// Unique id per SDNode in the DAG.
585 int NodeId = -1;
586
587 /// The values that are used by this operation.
588 SDUse *OperandList = nullptr;
589
590 /// The types of the values this node defines. SDNode's may
591 /// define multiple values simultaneously.
592 const EVT *ValueList;
593
594 /// List of uses for this SDNode.
595 SDUse *UseList = nullptr;
596
597 /// The number of entries in the Operand/Value list.
598 unsigned short NumOperands = 0;
599 unsigned short NumValues;
600
601 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
602 // original LLVM instructions.
603 // This is used for turning off scheduling, because we'll forgo
604 // the normal scheduling algorithms and output the instructions according to
605 // this ordering.
606 unsigned IROrder;
607
608 /// Source line information.
609 DebugLoc debugLoc;
610
611 /// Return a pointer to the specified value type.
612 static const EVT *getValueTypeList(EVT VT);
613
614 SDNodeFlags Flags;
615
616public:
617 /// Unique and persistent id per SDNode in the DAG.
618 /// Used for debug printing.
619 uint16_t PersistentId;
620
621 //===--------------------------------------------------------------------===//
622 // Accessors
623 //
624
625 /// Return the SelectionDAG opcode value for this node. For
626 /// pre-isel nodes (those for which isMachineOpcode returns false), these
627 /// are the opcode values in the ISD and <target>ISD namespaces. For
628 /// post-isel opcodes, see getMachineOpcode.
629 unsigned getOpcode() const { return (unsigned short)NodeType; }
630
631 /// Test if this node has a target-specific opcode (in the
632 /// \<target\>ISD namespace).
633 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
634
635 /// Test if this node has a target-specific opcode that may raise
636 /// FP exceptions (in the \<target\>ISD namespace and greater than
637 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
638 /// opcode are currently automatically considered to possibly raise
639 /// FP exceptions as well.
640 bool isTargetStrictFPOpcode() const {
641 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
642 }
643
644 /// Test if this node has a target-specific
645 /// memory-referencing opcode (in the \<target\>ISD namespace and
646 /// greater than FIRST_TARGET_MEMORY_OPCODE).
647 bool isTargetMemoryOpcode() const {
648 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
649 }
650
651 /// Return true if the type of the node type undefined.
652 bool isUndef() const { return NodeType == ISD::UNDEF; }
653
654 /// Test if this node is a memory intrinsic (with valid pointer information).
655 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
656 /// non-memory intrinsics (with chains) that are not really instances of
657 /// MemSDNode. For such nodes, we need some extra state to determine the
658 /// proper classof relationship.
659 bool isMemIntrinsic() const {
660 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
661 NodeType == ISD::INTRINSIC_VOID) &&
662 SDNodeBits.IsMemIntrinsic;
663 }
664
665 /// Test if this node is a strict floating point pseudo-op.
666 bool isStrictFPOpcode() {
667 switch (NodeType) {
668 default:
669 return false;
670 case ISD::STRICT_FP16_TO_FP:
671 case ISD::STRICT_FP_TO_FP16:
672#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
673 case ISD::STRICT_##DAGN:
674#include "llvm/IR/ConstrainedOps.def"
675 return true;
676 }
677 }
678
679 /// Test if this node has a post-isel opcode, directly
680 /// corresponding to a MachineInstr opcode.
681 bool isMachineOpcode() const { return NodeType < 0; }
682
683 /// This may only be called if isMachineOpcode returns
684 /// true. It returns the MachineInstr opcode value that the node's opcode
685 /// corresponds to.
686 unsigned getMachineOpcode() const {
687 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 687, __extension__ __PRETTY_FUNCTION__));
688 return ~NodeType;
689 }
690
691 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
692 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
693
694 bool isDivergent() const { return SDNodeBits.IsDivergent; }
695
696 /// Return true if there are no uses of this node.
697 bool use_empty() const { return UseList == nullptr; }
698
699 /// Return true if there is exactly one use of this node.
700 bool hasOneUse() const { return hasSingleElement(uses()); }
701
702 /// Return the number of uses of this node. This method takes
703 /// time proportional to the number of uses.
704 size_t use_size() const { return std::distance(use_begin(), use_end()); }
705
706 /// Return the unique node id.
707 int getNodeId() const { return NodeId; }
708
709 /// Set unique node id.
710 void setNodeId(int Id) { NodeId = Id; }
711
712 /// Return the node ordering.
713 unsigned getIROrder() const { return IROrder; }
714
715 /// Set the node ordering.
716 void setIROrder(unsigned Order) { IROrder = Order; }
717
718 /// Return the source location info.
719 const DebugLoc &getDebugLoc() const { return debugLoc; }
720
721 /// Set source location info. Try to avoid this, putting
722 /// it in the constructor is preferable.
723 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
724
725 /// This class provides iterator support for SDUse
726 /// operands that use a specific SDNode.
727 class use_iterator {
728 friend class SDNode;
729
730 SDUse *Op = nullptr;
731
732 explicit use_iterator(SDUse *op) : Op(op) {}
733
734 public:
735 using iterator_category = std::forward_iterator_tag;
736 using value_type = SDUse;
737 using difference_type = std::ptrdiff_t;
738 using pointer = value_type *;
739 using reference = value_type &;
740
741 use_iterator() = default;
742 use_iterator(const use_iterator &I) : Op(I.Op) {}
743
744 bool operator==(const use_iterator &x) const {
745 return Op == x.Op;
746 }
747 bool operator!=(const use_iterator &x) const {
748 return !operator==(x);
749 }
750
751 /// Return true if this iterator is at the end of uses list.
752 bool atEnd() const { return Op == nullptr; }
753
754 // Iterator traversal: forward iteration only.
755 use_iterator &operator++() { // Preincrement
756 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 756, __extension__ __PRETTY_FUNCTION__));
757 Op = Op->getNext();
758 return *this;
759 }
760
761 use_iterator operator++(int) { // Postincrement
762 use_iterator tmp = *this; ++*this; return tmp;
763 }
764
765 /// Retrieve a pointer to the current user node.
766 SDNode *operator*() const {
767 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 767, __extension__ __PRETTY_FUNCTION__));
768 return Op->getUser();
769 }
770
771 SDNode *operator->() const { return operator*(); }
772
773 SDUse &getUse() const { return *Op; }
774
775 /// Retrieve the operand # of this use in its user.
776 unsigned getOperandNo() const {
777 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 777, __extension__ __PRETTY_FUNCTION__));
778 return (unsigned)(Op - Op->getUser()->OperandList);
779 }
780 };
781
782 /// Provide iteration support to walk over all uses of an SDNode.
783 use_iterator use_begin() const {
784 return use_iterator(UseList);
785 }
786
787 static use_iterator use_end() { return use_iterator(nullptr); }
788
789 inline iterator_range<use_iterator> uses() {
790 return make_range(use_begin(), use_end());
791 }
792 inline iterator_range<use_iterator> uses() const {
793 return make_range(use_begin(), use_end());
794 }
795
796 /// Return true if there are exactly NUSES uses of the indicated value.
797 /// This method ignores uses of other values defined by this operation.
798 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
799
800 /// Return true if there are any use of the indicated value.
801 /// This method ignores uses of other values defined by this operation.
802 bool hasAnyUseOfValue(unsigned Value) const;
803
804 /// Return true if this node is the only use of N.
805 bool isOnlyUserOf(const SDNode *N) const;
806
807 /// Return true if this node is an operand of N.
808 bool isOperandOf(const SDNode *N) const;
809
810 /// Return true if this node is a predecessor of N.
811 /// NOTE: Implemented on top of hasPredecessor and every bit as
812 /// expensive. Use carefully.
813 bool isPredecessorOf(const SDNode *N) const {
814 return N->hasPredecessor(this);
815 }
816
817 /// Return true if N is a predecessor of this node.
818 /// N is either an operand of this node, or can be reached by recursively
819 /// traversing up the operands.
820 /// NOTE: This is an expensive method. Use it carefully.
821 bool hasPredecessor(const SDNode *N) const;
822
823 /// Returns true if N is a predecessor of any node in Worklist. This
824 /// helper keeps Visited and Worklist sets externally to allow unions
825 /// searches to be performed in parallel, caching of results across
826 /// queries and incremental addition to Worklist. Stops early if N is
827 /// found but will resume. Remember to clear Visited and Worklists
828 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
829 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
830 /// topologically ordered (Operands have strictly smaller node id) and search
831 /// can be pruned leveraging this.
832 static bool hasPredecessorHelper(const SDNode *N,
833 SmallPtrSetImpl<const SDNode *> &Visited,
834 SmallVectorImpl<const SDNode *> &Worklist,
835 unsigned int MaxSteps = 0,
836 bool TopologicalPrune = false) {
837 SmallVector<const SDNode *, 8> DeferredNodes;
838 if (Visited.count(N))
839 return true;
840
841 // Node Id's are assigned in three places: As a topological
842 // ordering (> 0), during legalization (results in values set to
843 // 0), new nodes (set to -1). If N has a topolgical id then we
844 // know that all nodes with ids smaller than it cannot be
845 // successors and we need not check them. Filter out all node
846 // that can't be matches. We add them to the worklist before exit
847 // in case of multiple calls. Note that during selection the topological id
848 // may be violated if a node's predecessor is selected before it. We mark
849 // this at selection negating the id of unselected successors and
850 // restricting topological pruning to positive ids.
851
852 int NId = N->getNodeId();
853 // If we Invalidated the Id, reconstruct original NId.
854 if (NId < -1)
855 NId = -(NId + 1);
856
857 bool Found = false;
858 while (!Worklist.empty()) {
859 const SDNode *M = Worklist.pop_back_val();
860 int MId = M->getNodeId();
861 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
862 (MId > 0) && (MId < NId)) {
863 DeferredNodes.push_back(M);
864 continue;
865 }
866 for (const SDValue &OpV : M->op_values()) {
867 SDNode *Op = OpV.getNode();
868 if (Visited.insert(Op).second)
869 Worklist.push_back(Op);
870 if (Op == N)
871 Found = true;
872 }
873 if (Found)
874 break;
875 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
876 break;
877 }
878 // Push deferred nodes back on worklist.
879 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
880 // If we bailed early, conservatively return found.
881 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
882 return true;
883 return Found;
884 }
885
886 /// Return true if all the users of N are contained in Nodes.
887 /// NOTE: Requires at least one match, but doesn't require them all.
888 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
889
890 /// Return the number of values used by this operation.
891 unsigned getNumOperands() const { return NumOperands; }
892
893 /// Return the maximum number of operands that a SDNode can hold.
894 static constexpr size_t getMaxNumOperands() {
895 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
896 }
897
898 /// Helper method returns the integer value of a ConstantSDNode operand.
899 inline uint64_t getConstantOperandVal(unsigned Num) const;
900
901 /// Helper method returns the APInt of a ConstantSDNode operand.
902 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
903
904 const SDValue &getOperand(unsigned Num) const {
905 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 905, __extension__ __PRETTY_FUNCTION__));
906 return OperandList[Num];
907 }
908
909 using op_iterator = SDUse *;
910
911 op_iterator op_begin() const { return OperandList; }
912 op_iterator op_end() const { return OperandList+NumOperands; }
913 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
914
915 /// Iterator for directly iterating over the operand SDValue's.
916 struct value_op_iterator
917 : iterator_adaptor_base<value_op_iterator, op_iterator,
918 std::random_access_iterator_tag, SDValue,
919 ptrdiff_t, value_op_iterator *,
920 value_op_iterator *> {
921 explicit value_op_iterator(SDUse *U = nullptr)
922 : iterator_adaptor_base(U) {}
923
924 const SDValue &operator*() const { return I->get(); }
925 };
926
927 iterator_range<value_op_iterator> op_values() const {
928 return make_range(value_op_iterator(op_begin()),
929 value_op_iterator(op_end()));
930 }
931
932 SDVTList getVTList() const {
933 SDVTList X = { ValueList, NumValues };
934 return X;
935 }
936
937 /// If this node has a glue operand, return the node
938 /// to which the glue operand points. Otherwise return NULL.
939 SDNode *getGluedNode() const {
940 if (getNumOperands() != 0 &&
941 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
942 return getOperand(getNumOperands()-1).getNode();
943 return nullptr;
944 }
945
946 /// If this node has a glue value with a user, return
947 /// the user (there is at most one). Otherwise return NULL.
948 SDNode *getGluedUser() const {
949 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
950 if (UI.getUse().get().getValueType() == MVT::Glue)
951 return *UI;
952 return nullptr;
953 }
954
955 SDNodeFlags getFlags() const { return Flags; }
956 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
957
958 /// Clear any flags in this node that aren't also set in Flags.
959 /// If Flags is not in a defined state then this has no effect.
960 void intersectFlagsWith(const SDNodeFlags Flags);
961
962 /// Return the number of values defined/returned by this operator.
963 unsigned getNumValues() const { return NumValues; }
964
965 /// Return the type of a specified result.
966 EVT getValueType(unsigned ResNo) const {
967 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 967, __extension__ __PRETTY_FUNCTION__));
968 return ValueList[ResNo];
969 }
970
971 /// Return the type of a specified result as a simple type.
972 MVT getSimpleValueType(unsigned ResNo) const {
973 return getValueType(ResNo).getSimpleVT();
974 }
975
976 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
977 ///
978 /// If the value type is a scalable vector type, the scalable property will
979 /// be set and the runtime size will be a positive integer multiple of the
980 /// base size.
981 TypeSize getValueSizeInBits(unsigned ResNo) const {
982 return getValueType(ResNo).getSizeInBits();
983 }
984
985 using value_iterator = const EVT *;
986
987 value_iterator value_begin() const { return ValueList; }
988 value_iterator value_end() const { return ValueList+NumValues; }
989 iterator_range<value_iterator> values() const {
990 return llvm::make_range(value_begin(), value_end());
991 }
992
993 /// Return the opcode of this operation for printing.
994 std::string getOperationName(const SelectionDAG *G = nullptr) const;
995 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
996 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
997 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
998 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
999 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and all children down to
1002 /// the leaves. The given SelectionDAG allows target-specific nodes
1003 /// to be printed in human-readable form. Unlike printr, this will
1004 /// print the whole DAG, including children that appear multiple
1005 /// times.
1006 ///
1007 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1008
1009 /// Print a SelectionDAG node and children up to
1010 /// depth "depth." The given SelectionDAG allows target-specific
1011 /// nodes to be printed in human-readable form. Unlike printr, this
1012 /// will print children that appear multiple times wherever they are
1013 /// used.
1014 ///
1015 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1016 unsigned depth = 100) const;
1017
1018 /// Dump this node, for debugging.
1019 void dump() const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 void dumpr() const;
1023
1024 /// Dump this node, for debugging.
1025 /// The given SelectionDAG allows target-specific nodes to be printed
1026 /// in human-readable form.
1027 void dump(const SelectionDAG *G) const;
1028
1029 /// Dump (recursively) this node and its use-def subgraph.
1030 /// The given SelectionDAG allows target-specific nodes to be printed
1031 /// in human-readable form.
1032 void dumpr(const SelectionDAG *G) const;
1033
1034 /// printrFull to dbgs(). The given SelectionDAG allows
1035 /// target-specific nodes to be printed in human-readable form.
1036 /// Unlike dumpr, this will print the whole DAG, including children
1037 /// that appear multiple times.
1038 void dumprFull(const SelectionDAG *G = nullptr) const;
1039
1040 /// printrWithDepth to dbgs(). The given
1041 /// SelectionDAG allows target-specific nodes to be printed in
1042 /// human-readable form. Unlike dumpr, this will print children
1043 /// that appear multiple times wherever they are used.
1044 ///
1045 void dumprWithDepth(const SelectionDAG *G = nullptr,
1046 unsigned depth = 100) const;
1047
1048 /// Gather unique data for the node.
1049 void Profile(FoldingSetNodeID &ID) const;
1050
1051 /// This method should only be used by the SDUse class.
1052 void addUse(SDUse &U) { U.addToList(&UseList); }
1053
1054protected:
1055 static SDVTList getSDVTList(EVT VT) {
1056 SDVTList Ret = { getValueTypeList(VT), 1 };
1057 return Ret;
1058 }
1059
1060 /// Create an SDNode.
1061 ///
1062 /// SDNodes are created without any operands, and never own the operand
1063 /// storage. To add operands, see SelectionDAG::createOperands.
1064 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1065 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1066 IROrder(Order), debugLoc(std::move(dl)) {
1067 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1068 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1068, __extension__ __PRETTY_FUNCTION__));
1069 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__))
1070 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__));
1071 }
1072
1073 /// Release the operands and set this node to have zero operands.
1074 void DropOperands();
1075};
1076
1077/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1078/// into SDNode creation functions.
1079/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1080/// from the original Instruction, and IROrder is the ordinal position of
1081/// the instruction.
1082/// When an SDNode is created after the DAG is being built, both DebugLoc and
1083/// the IROrder are propagated from the original SDNode.
1084/// So SDLoc class provides two constructors besides the default one, one to
1085/// be used by the DAGBuilder, the other to be used by others.
1086class SDLoc {
1087private:
1088 DebugLoc DL;
1089 int IROrder = 0;
1090
1091public:
1092 SDLoc() = default;
1093 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1094 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1095 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1096 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1096, __extension__ __PRETTY_FUNCTION__));
1097 if (I)
1098 DL = I->getDebugLoc();
1099 }
1100
1101 unsigned getIROrder() const { return IROrder; }
1102 const DebugLoc &getDebugLoc() const { return DL; }
1103};
1104
1105// Define inline functions from the SDValue class.
1106
1107inline SDValue::SDValue(SDNode *node, unsigned resno)
1108 : Node(node), ResNo(resno) {
1109 // Explicitly check for !ResNo to avoid use-after-free, because there are
1110 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1111 // combines.
1112 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
1113 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__));
1114 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1114, __extension__ __PRETTY_FUNCTION__));
1115}
1116
1117inline unsigned SDValue::getOpcode() const {
1118 return Node->getOpcode();
1119}
1120
1121inline EVT SDValue::getValueType() const {
1122 return Node->getValueType(ResNo);
1123}
1124
1125inline unsigned SDValue::getNumOperands() const {
1126 return Node->getNumOperands();
1127}
1128
1129inline const SDValue &SDValue::getOperand(unsigned i) const {
1130 return Node->getOperand(i);
1131}
1132
1133inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1134 return Node->getConstantOperandVal(i);
1135}
1136
1137inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1138 return Node->getConstantOperandAPInt(i);
1139}
1140
1141inline bool SDValue::isTargetOpcode() const {
1142 return Node->isTargetOpcode();
1143}
1144
1145inline bool SDValue::isTargetMemoryOpcode() const {
1146 return Node->isTargetMemoryOpcode();
1147}
1148
1149inline bool SDValue::isMachineOpcode() const {
1150 return Node->isMachineOpcode();
1151}
1152
1153inline unsigned SDValue::getMachineOpcode() const {
1154 return Node->getMachineOpcode();
1155}
1156
1157inline bool SDValue::isUndef() const {
1158 return Node->isUndef();
1159}
1160
1161inline bool SDValue::use_empty() const {
1162 return !Node->hasAnyUseOfValue(ResNo);
1163}
1164
1165inline bool SDValue::hasOneUse() const {
1166 return Node->hasNUsesOfValue(1, ResNo);
1167}
1168
1169inline const DebugLoc &SDValue::getDebugLoc() const {
1170 return Node->getDebugLoc();
1171}
1172
1173inline void SDValue::dump() const {
1174 return Node->dump();
1175}
1176
1177inline void SDValue::dump(const SelectionDAG *G) const {
1178 return Node->dump(G);
1179}
1180
1181inline void SDValue::dumpr() const {
1182 return Node->dumpr();
1183}
1184
1185inline void SDValue::dumpr(const SelectionDAG *G) const {
1186 return Node->dumpr(G);
1187}
1188
1189// Define inline functions from the SDUse class.
1190
1191inline void SDUse::set(const SDValue &V) {
1192 if (Val.getNode()) removeFromList();
1193 Val = V;
1194 if (V.getNode()) V.getNode()->addUse(*this);
1195}
1196
1197inline void SDUse::setInitial(const SDValue &V) {
1198 Val = V;
1199 V.getNode()->addUse(*this);
1200}
1201
1202inline void SDUse::setNode(SDNode *N) {
1203 if (Val.getNode()) removeFromList();
1204 Val.setNode(N);
1205 if (N) N->addUse(*this);
1206}
1207
1208/// This class is used to form a handle around another node that
1209/// is persistent and is updated across invocations of replaceAllUsesWith on its
1210/// operand. This node should be directly created by end-users and not added to
1211/// the AllNodes list.
1212class HandleSDNode : public SDNode {
1213 SDUse Op;
1214
1215public:
1216 explicit HandleSDNode(SDValue X)
1217 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1218 // HandleSDNodes are never inserted into the DAG, so they won't be
1219 // auto-numbered. Use ID 65535 as a sentinel.
1220 PersistentId = 0xffff;
1221
1222 // Manually set up the operand list. This node type is special in that it's
1223 // always stack allocated and SelectionDAG does not manage its operands.
1224 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1225 // be so special.
1226 Op.setUser(this);
1227 Op.setInitial(X);
1228 NumOperands = 1;
1229 OperandList = &Op;
1230 }
1231 ~HandleSDNode();
1232
1233 const SDValue &getValue() const { return Op; }
1234};
1235
1236class AddrSpaceCastSDNode : public SDNode {
1237private:
1238 unsigned SrcAddrSpace;
1239 unsigned DestAddrSpace;
1240
1241public:
1242 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1243 unsigned SrcAS, unsigned DestAS);
1244
1245 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1246 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1247
1248 static bool classof(const SDNode *N) {
1249 return N->getOpcode() == ISD::ADDRSPACECAST;
1250 }
1251};
1252
1253/// This is an abstract virtual class for memory operations.
1254class MemSDNode : public SDNode {
1255private:
1256 // VT of in-memory value.
1257 EVT MemoryVT;
1258
1259protected:
1260 /// Memory reference information.
1261 MachineMemOperand *MMO;
1262
1263public:
1264 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1265 EVT memvt, MachineMemOperand *MMO);
1266
1267 bool readMem() const { return MMO->isLoad(); }
1268 bool writeMem() const { return MMO->isStore(); }
1269
1270 /// Returns alignment and volatility of the memory access
1271 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1272 Align getAlign() const { return MMO->getAlign(); }
1273 // FIXME: Remove once transition to getAlign is over.
1274 unsigned getAlignment() const { return MMO->getAlign().value(); }
1275
1276 /// Return the SubclassData value, without HasDebugValue. This contains an
1277 /// encoding of the volatile flag, as well as bits used by subclasses. This
1278 /// function should only be used to compute a FoldingSetNodeID value.
1279 /// The HasDebugValue bit is masked out because CSE map needs to match
1280 /// nodes with debug info with nodes without debug info. Same is about
1281 /// isDivergent bit.
1282 unsigned getRawSubclassData() const {
1283 uint16_t Data;
1284 union {
1285 char RawSDNodeBits[sizeof(uint16_t)];
1286 SDNodeBitfields SDNodeBits;
1287 };
1288 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1289 SDNodeBits.HasDebugValue = 0;
1290 SDNodeBits.IsDivergent = false;
1291 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1292 return Data;
1293 }
1294
1295 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1296 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1297 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1298 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1299
1300 // Returns the offset from the location of the access.
1301 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1302
1303 /// Returns the AA info that describes the dereference.
1304 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1305
1306 /// Returns the Ranges that describes the dereference.
1307 const MDNode *getRanges() const { return MMO->getRanges(); }
1308
1309 /// Returns the synchronization scope ID for this memory operation.
1310 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1311
1312 /// Return the atomic ordering requirements for this memory operation. For
1313 /// cmpxchg atomic operations, return the atomic ordering requirements when
1314 /// store occurs.
1315 AtomicOrdering getSuccessOrdering() const {
1316 return MMO->getSuccessOrdering();
1317 }
1318
1319 /// Return a single atomic ordering that is at least as strong as both the
1320 /// success and failure orderings for an atomic operation. (For operations
1321 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1322 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1323
1324 /// Return true if the memory operation ordering is Unordered or higher.
1325 bool isAtomic() const { return MMO->isAtomic(); }
1326
1327 /// Returns true if the memory operation doesn't imply any ordering
1328 /// constraints on surrounding memory operations beyond the normal memory
1329 /// aliasing rules.
1330 bool isUnordered() const { return MMO->isUnordered(); }
1331
1332 /// Returns true if the memory operation is neither atomic or volatile.
1333 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1334
1335 /// Return the type of the in-memory value.
1336 EVT getMemoryVT() const { return MemoryVT; }
1337
1338 /// Return a MachineMemOperand object describing the memory
1339 /// reference performed by operation.
1340 MachineMemOperand *getMemOperand() const { return MMO; }
1341
1342 const MachinePointerInfo &getPointerInfo() const {
1343 return MMO->getPointerInfo();
1344 }
1345
1346 /// Return the address space for the associated pointer
1347 unsigned getAddressSpace() const {
1348 return getPointerInfo().getAddrSpace();
1349 }
1350
1351 /// Update this MemSDNode's MachineMemOperand information
1352 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1353 /// This must only be used when the new alignment applies to all users of
1354 /// this MachineMemOperand.
1355 void refineAlignment(const MachineMemOperand *NewMMO) {
1356 MMO->refineAlignment(NewMMO);
1357 }
1358
1359 const SDValue &getChain() const { return getOperand(0); }
1360
1361 const SDValue &getBasePtr() const {
1362 switch (getOpcode()) {
1363 case ISD::STORE:
1364 case ISD::VP_STORE:
1365 case ISD::MSTORE:
1366 case ISD::VP_SCATTER:
1367 return getOperand(2);
1368 case ISD::MGATHER:
1369 case ISD::MSCATTER:
1370 return getOperand(3);
1371 default:
1372 return getOperand(1);
1373 }
1374 }
1375
1376 // Methods to support isa and dyn_cast
1377 static bool classof(const SDNode *N) {
1378 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1379 // with either an intrinsic or a target opcode.
1380 switch (N->getOpcode()) {
1381 case ISD::LOAD:
1382 case ISD::STORE:
1383 case ISD::PREFETCH:
1384 case ISD::ATOMIC_CMP_SWAP:
1385 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1386 case ISD::ATOMIC_SWAP:
1387 case ISD::ATOMIC_LOAD_ADD:
1388 case ISD::ATOMIC_LOAD_SUB:
1389 case ISD::ATOMIC_LOAD_AND:
1390 case ISD::ATOMIC_LOAD_CLR:
1391 case ISD::ATOMIC_LOAD_OR:
1392 case ISD::ATOMIC_LOAD_XOR:
1393 case ISD::ATOMIC_LOAD_NAND:
1394 case ISD::ATOMIC_LOAD_MIN:
1395 case ISD::ATOMIC_LOAD_MAX:
1396 case ISD::ATOMIC_LOAD_UMIN:
1397 case ISD::ATOMIC_LOAD_UMAX:
1398 case ISD::ATOMIC_LOAD_FADD:
1399 case ISD::ATOMIC_LOAD_FSUB:
1400 case ISD::ATOMIC_LOAD:
1401 case ISD::ATOMIC_STORE:
1402 case ISD::MLOAD:
1403 case ISD::MSTORE:
1404 case ISD::MGATHER:
1405 case ISD::MSCATTER:
1406 case ISD::VP_LOAD:
1407 case ISD::VP_STORE:
1408 case ISD::VP_GATHER:
1409 case ISD::VP_SCATTER:
1410 return true;
1411 default:
1412 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1413 }
1414 }
1415};
1416
1417/// This is an SDNode representing atomic operations.
1418class AtomicSDNode : public MemSDNode {
1419public:
1420 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1421 EVT MemVT, MachineMemOperand *MMO)
1422 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1423 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__))
1424 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__));
1425 }
1426
1427 const SDValue &getBasePtr() const { return getOperand(1); }
1428 const SDValue &getVal() const { return getOperand(2); }
1429
1430 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1431 /// otherwise.
1432 bool isCompareAndSwap() const {
1433 unsigned Op = getOpcode();
1434 return Op == ISD::ATOMIC_CMP_SWAP ||
1435 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1436 }
1437
1438 /// For cmpxchg atomic operations, return the atomic ordering requirements
1439 /// when store does not occur.
1440 AtomicOrdering getFailureOrdering() const {
1441 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __extension__ __PRETTY_FUNCTION__));
1442 return MMO->getFailureOrdering();
1443 }
1444
1445 // Methods to support isa and dyn_cast
1446 static bool classof(const SDNode *N) {
1447 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1448 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1449 N->getOpcode() == ISD::ATOMIC_SWAP ||
1450 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1451 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1452 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1453 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1454 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1455 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1456 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1457 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1458 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1459 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1460 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1461 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1462 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1463 N->getOpcode() == ISD::ATOMIC_LOAD ||
1464 N->getOpcode() == ISD::ATOMIC_STORE;
1465 }
1466};
1467
1468/// This SDNode is used for target intrinsics that touch
1469/// memory and need an associated MachineMemOperand. Its opcode may be
1470/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1471/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1472class MemIntrinsicSDNode : public MemSDNode {
1473public:
1474 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1475 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1476 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1477 SDNodeBits.IsMemIntrinsic = true;
1478 }
1479
1480 // Methods to support isa and dyn_cast
1481 static bool classof(const SDNode *N) {
1482 // We lower some target intrinsics to their target opcode
1483 // early a node with a target opcode can be of this class
1484 return N->isMemIntrinsic() ||
1485 N->getOpcode() == ISD::PREFETCH ||
1486 N->isTargetMemoryOpcode();
1487 }
1488};
1489
1490/// This SDNode is used to implement the code generator
1491/// support for the llvm IR shufflevector instruction. It combines elements
1492/// from two input vectors into a new input vector, with the selection and
1493/// ordering of elements determined by an array of integers, referred to as
1494/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1495/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1496/// An index of -1 is treated as undef, such that the code generator may put
1497/// any value in the corresponding element of the result.
1498class ShuffleVectorSDNode : public SDNode {
1499 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1500 // is freed when the SelectionDAG object is destroyed.
1501 const int *Mask;
1502
1503protected:
1504 friend class SelectionDAG;
1505
1506 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1507 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1508
1509public:
1510 ArrayRef<int> getMask() const {
1511 EVT VT = getValueType(0);
1512 return makeArrayRef(Mask, VT.getVectorNumElements());
1513 }
1514
1515 int getMaskElt(unsigned Idx) const {
1516 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1516, __extension__ __PRETTY_FUNCTION__));
1517 return Mask[Idx];
1518 }
1519
1520 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1521
1522 int getSplatIndex() const {
1523 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1523, __extension__ __PRETTY_FUNCTION__));
1524 EVT VT = getValueType(0);
1525 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1526 if (Mask[i] >= 0)
1527 return Mask[i];
1528
1529 // We can choose any index value here and be correct because all elements
1530 // are undefined. Return 0 for better potential for callers to simplify.
1531 return 0;
1532 }
1533
1534 static bool isSplatMask(const int *Mask, EVT VT);
1535
1536 /// Change values in a shuffle permute mask assuming
1537 /// the two vector operands have swapped position.
1538 static void commuteMask(MutableArrayRef<int> Mask) {
1539 unsigned NumElems = Mask.size();
1540 for (unsigned i = 0; i != NumElems; ++i) {
1541 int idx = Mask[i];
1542 if (idx < 0)
1543 continue;
1544 else if (idx < (int)NumElems)
1545 Mask[i] = idx + NumElems;
1546 else
1547 Mask[i] = idx - NumElems;
1548 }
1549 }
1550
1551 static bool classof(const SDNode *N) {
1552 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1553 }
1554};
1555
1556class ConstantSDNode : public SDNode {
1557 friend class SelectionDAG;
1558
1559 const ConstantInt *Value;
1560
1561 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1562 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1563 getSDVTList(VT)),
1564 Value(val) {
1565 ConstantSDNodeBits.IsOpaque = isOpaque;
1566 }
1567
1568public:
1569 const ConstantInt *getConstantIntValue() const { return Value; }
1570 const APInt &getAPIntValue() const { return Value->getValue(); }
1571 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1572 int64_t getSExtValue() const { return Value->getSExtValue(); }
1573 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1574 return Value->getLimitedValue(Limit);
1575 }
1576 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1577 Align getAlignValue() const { return Value->getAlignValue(); }
1578
1579 bool isOne() const { return Value->isOne(); }
1580 bool isZero() const { return Value->isZero(); }
1581 // NOTE: This is soft-deprecated. Please use `isZero()` instead.
1582 bool isNullValue() const { return isZero(); }
1583 bool isAllOnes() const { return Value->isMinusOne(); }
1584 // NOTE: This is soft-deprecated. Please use `isAllOnes()` instead.
1585 bool isAllOnesValue() const { return isAllOnes(); }
1586 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1587 bool isMinSignedValue() const { return Value->isMinValue(true); }
1588
1589 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1590
1591 static bool classof(const SDNode *N) {
1592 return N->getOpcode() == ISD::Constant ||
1593 N->getOpcode() == ISD::TargetConstant;
1594 }
1595};
1596
1597uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1598 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1599}
1600
1601const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1602 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1603}
1604
1605class ConstantFPSDNode : public SDNode {
1606 friend class SelectionDAG;
1607
1608 const ConstantFP *Value;
1609
1610 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1611 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1612 DebugLoc(), getSDVTList(VT)),
1613 Value(val) {}
1614
1615public:
1616 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1617 const ConstantFP *getConstantFPValue() const { return Value; }
1618
1619 /// Return true if the value is positive or negative zero.
1620 bool isZero() const { return Value->isZero(); }
1621
1622 /// Return true if the value is a NaN.
1623 bool isNaN() const { return Value->isNaN(); }
1624
1625 /// Return true if the value is an infinity
1626 bool isInfinity() const { return Value->isInfinity(); }
1627
1628 /// Return true if the value is negative.
1629 bool isNegative() const { return Value->isNegative(); }
1630
1631 /// We don't rely on operator== working on double values, as
1632 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1633 /// As such, this method can be used to do an exact bit-for-bit comparison of
1634 /// two floating point values.
1635
1636 /// We leave the version with the double argument here because it's just so
1637 /// convenient to write "2.0" and the like. Without this function we'd
1638 /// have to duplicate its logic everywhere it's called.
1639 bool isExactlyValue(double V) const {
1640 return Value->getValueAPF().isExactlyValue(V);
1641 }
1642 bool isExactlyValue(const APFloat& V) const;
1643
1644 static bool isValueValidForType(EVT VT, const APFloat& Val);
1645
1646 static bool classof(const SDNode *N) {
1647 return N->getOpcode() == ISD::ConstantFP ||
1648 N->getOpcode() == ISD::TargetConstantFP;
1649 }
1650};
1651
1652/// Returns true if \p V is a constant integer zero.
1653bool isNullConstant(SDValue V);
1654
1655/// Returns true if \p V is an FP constant with a value of positive zero.
1656bool isNullFPConstant(SDValue V);
1657
1658/// Returns true if \p V is an integer constant with all bits set.
1659bool isAllOnesConstant(SDValue V);
1660
1661/// Returns true if \p V is a constant integer one.
1662bool isOneConstant(SDValue V);
1663
1664/// Return the non-bitcasted source operand of \p V if it exists.
1665/// If \p V is not a bitcasted value, it is returned as-is.
1666SDValue peekThroughBitcasts(SDValue V);
1667
1668/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1669/// If \p V is not a bitcasted one-use value, it is returned as-is.
1670SDValue peekThroughOneUseBitcasts(SDValue V);
1671
1672/// Return the non-extracted vector source operand of \p V if it exists.
1673/// If \p V is not an extracted subvector, it is returned as-is.
1674SDValue peekThroughExtractSubvectors(SDValue V);
1675
1676/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1677/// constant is canonicalized to be operand 1.
1678bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1679
1680/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1681ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1682 bool AllowTruncation = false);
1683
1684/// Returns the SDNode if it is a demanded constant splat BuildVector or
1685/// constant int.
1686ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1687 bool AllowUndefs = false,
1688 bool AllowTruncation = false);
1689
1690/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1691ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1692
1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant float.
1695ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1696 bool AllowUndefs = false);
1697
1698/// Return true if the value is a constant 0 integer or a splatted vector of
1699/// a constant 0 integer (with no undefs by default).
1700/// Build vector implicit truncation is not an issue for null values.
1701bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1702
1703/// Return true if the value is a constant 1 integer or a splatted vector of a
1704/// constant 1 integer (with no undefs).
1705/// Does not permit build vector implicit truncation.
1706bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1707
1708/// Return true if the value is a constant -1 integer or a splatted vector of a
1709/// constant -1 integer (with no undefs).
1710/// Does not permit build vector implicit truncation.
1711bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1712
1713/// Return true if \p V is either a integer or FP constant.
1714inline bool isIntOrFPConstant(SDValue V) {
1715 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1716}
1717
1718class GlobalAddressSDNode : public SDNode {
1719 friend class SelectionDAG;
1720
1721 const GlobalValue *TheGlobal;
1722 int64_t Offset;
1723 unsigned TargetFlags;
1724
1725 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1726 const GlobalValue *GA, EVT VT, int64_t o,
1727 unsigned TF);
1728
1729public:
1730 const GlobalValue *getGlobal() const { return TheGlobal; }
1731 int64_t getOffset() const { return Offset; }
1732 unsigned getTargetFlags() const { return TargetFlags; }
1733 // Return the address space this GlobalAddress belongs to.
1734 unsigned getAddressSpace() const;
1735
1736 static bool classof(const SDNode *N) {
1737 return N->getOpcode() == ISD::GlobalAddress ||
1738 N->getOpcode() == ISD::TargetGlobalAddress ||
1739 N->getOpcode() == ISD::GlobalTLSAddress ||
1740 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1741 }
1742};
1743
1744class FrameIndexSDNode : public SDNode {
1745 friend class SelectionDAG;
1746
1747 int FI;
1748
1749 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1750 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1751 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1752 }
1753
1754public:
1755 int getIndex() const { return FI; }
1756
1757 static bool classof(const SDNode *N) {
1758 return N->getOpcode() == ISD::FrameIndex ||
1759 N->getOpcode() == ISD::TargetFrameIndex;
1760 }
1761};
1762
1763/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1764/// the offet and size that are started/ended in the underlying FrameIndex.
1765class LifetimeSDNode : public SDNode {
1766 friend class SelectionDAG;
1767 int64_t Size;
1768 int64_t Offset; // -1 if offset is unknown.
1769
1770 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1771 SDVTList VTs, int64_t Size, int64_t Offset)
1772 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1773public:
1774 int64_t getFrameIndex() const {
1775 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1776 }
1777
1778 bool hasOffset() const { return Offset >= 0; }
1779 int64_t getOffset() const {
1780 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1780, __extension__ __PRETTY_FUNCTION__));
1781 return Offset;
1782 }
1783 int64_t getSize() const {
1784 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1784, __extension__ __PRETTY_FUNCTION__));
1785 return Size;
1786 }
1787
1788 // Methods to support isa and dyn_cast
1789 static bool classof(const SDNode *N) {
1790 return N->getOpcode() == ISD::LIFETIME_START ||
1791 N->getOpcode() == ISD::LIFETIME_END;
1792 }
1793};
1794
1795/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1796/// the index of the basic block being probed. A pseudo probe serves as a place
1797/// holder and will be removed at the end of compilation. It does not have any
1798/// operand because we do not want the instruction selection to deal with any.
1799class PseudoProbeSDNode : public SDNode {
1800 friend class SelectionDAG;
1801 uint64_t Guid;
1802 uint64_t Index;
1803 uint32_t Attributes;
1804
1805 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1806 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1807 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1808 Attributes(Attr) {}
1809
1810public:
1811 uint64_t getGuid() const { return Guid; }
1812 uint64_t getIndex() const { return Index; }
1813 uint32_t getAttributes() const { return Attributes; }
1814
1815 // Methods to support isa and dyn_cast
1816 static bool classof(const SDNode *N) {
1817 return N->getOpcode() == ISD::PSEUDO_PROBE;
1818 }
1819};
1820
1821class JumpTableSDNode : public SDNode {
1822 friend class SelectionDAG;
1823
1824 int JTI;
1825 unsigned TargetFlags;
1826
1827 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1828 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1829 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1830 }
1831
1832public:
1833 int getIndex() const { return JTI; }
1834 unsigned getTargetFlags() const { return TargetFlags; }
1835
1836 static bool classof(const SDNode *N) {
1837 return N->getOpcode() == ISD::JumpTable ||
1838 N->getOpcode() == ISD::TargetJumpTable;
1839 }
1840};
1841
1842class ConstantPoolSDNode : public SDNode {
1843 friend class SelectionDAG;
1844
1845 union {
1846 const Constant *ConstVal;
1847 MachineConstantPoolValue *MachineCPVal;
1848 } Val;
1849 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1850 Align Alignment; // Minimum alignment requirement of CP.
1851 unsigned TargetFlags;
1852
1853 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1854 Align Alignment, unsigned TF)
1855 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1856 DebugLoc(), getSDVTList(VT)),
1857 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1858 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__));
1859 Val.ConstVal = c;
1860 }
1861
1862 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1863 Align Alignment, unsigned TF)
1864 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1865 DebugLoc(), getSDVTList(VT)),
1866 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1867 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1867, __extension__ __PRETTY_FUNCTION__));
1868 Val.MachineCPVal = v;
1869 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1870 }
1871
1872public:
1873 bool isMachineConstantPoolEntry() const {
1874 return Offset < 0;
1875 }
1876
1877 const Constant *getConstVal() const {
1878 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1878, __extension__ __PRETTY_FUNCTION__));
1879 return Val.ConstVal;
1880 }
1881
1882 MachineConstantPoolValue *getMachineCPVal() const {
1883 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1883, __extension__ __PRETTY_FUNCTION__));
1884 return Val.MachineCPVal;
1885 }
1886
1887 int getOffset() const {
1888 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1889 }
1890
1891 // Return the alignment of this constant pool object, which is either 0 (for
1892 // default alignment) or the desired value.
1893 Align getAlign() const { return Alignment; }
1894 unsigned getTargetFlags() const { return TargetFlags; }
1895
1896 Type *getType() const;
1897
1898 static bool classof(const SDNode *N) {
1899 return N->getOpcode() == ISD::ConstantPool ||
1900 N->getOpcode() == ISD::TargetConstantPool;
1901 }
1902};
1903
1904/// Completely target-dependent object reference.
1905class TargetIndexSDNode : public SDNode {
1906 friend class SelectionDAG;
1907
1908 unsigned TargetFlags;
1909 int Index;
1910 int64_t Offset;
1911
1912public:
1913 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1914 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1915 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1916
1917 unsigned getTargetFlags() const { return TargetFlags; }
1918 int getIndex() const { return Index; }
1919 int64_t getOffset() const { return Offset; }
1920
1921 static bool classof(const SDNode *N) {
1922 return N->getOpcode() == ISD::TargetIndex;
1923 }
1924};
1925
1926class BasicBlockSDNode : public SDNode {
1927 friend class SelectionDAG;
1928
1929 MachineBasicBlock *MBB;
1930
1931 /// Debug info is meaningful and potentially useful here, but we create
1932 /// blocks out of order when they're jumped to, which makes it a bit
1933 /// harder. Let's see if we need it first.
1934 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1935 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1936 {}
1937
1938public:
1939 MachineBasicBlock *getBasicBlock() const { return MBB; }
1940
1941 static bool classof(const SDNode *N) {
1942 return N->getOpcode() == ISD::BasicBlock;
1943 }
1944};
1945
1946/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1947class BuildVectorSDNode : public SDNode {
1948public:
1949 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1950 explicit BuildVectorSDNode() = delete;
1951
1952 /// Check if this is a constant splat, and if so, find the
1953 /// smallest element size that splats the vector. If MinSplatBits is
1954 /// nonzero, the element size must be at least that large. Note that the
1955 /// splat element may be the entire vector (i.e., a one element vector).
1956 /// Returns the splat element value in SplatValue. Any undefined bits in
1957 /// that value are zero, and the corresponding bits in the SplatUndef mask
1958 /// are set. The SplatBitSize value is set to the splat element size in
1959 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1960 /// undefined. isBigEndian describes the endianness of the target.
1961 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1962 unsigned &SplatBitSize, bool &HasAnyUndefs,
1963 unsigned MinSplatBits = 0,
1964 bool isBigEndian = false) const;
1965
1966 /// Returns the demanded splatted value or a null value if this is not a
1967 /// splat.
1968 ///
1969 /// The DemandedElts mask indicates the elements that must be in the splat.
1970 /// If passed a non-null UndefElements bitvector, it will resize it to match
1971 /// the vector width and set the bits where elements are undef.
1972 SDValue getSplatValue(const APInt &DemandedElts,
1973 BitVector *UndefElements = nullptr) const;
1974
1975 /// Returns the splatted value or a null value if this is not a splat.
1976 ///
1977 /// If passed a non-null UndefElements bitvector, it will resize it to match
1978 /// the vector width and set the bits where elements are undef.
1979 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1980
1981 /// Find the shortest repeating sequence of values in the build vector.
1982 ///
1983 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1984 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1985 ///
1986 /// Currently this must be a power-of-2 build vector.
1987 /// The DemandedElts mask indicates the elements that must be present,
1988 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1989 /// non-null UndefElements bitvector, it will resize it to match the original
1990 /// vector width and set the bits where elements are undef. If result is
1991 /// false, Sequence will be empty.
1992 bool getRepeatedSequence(const APInt &DemandedElts,
1993 SmallVectorImpl<SDValue> &Sequence,
1994 BitVector *UndefElements = nullptr) const;
1995
1996 /// Find the shortest repeating sequence of values in the build vector.
1997 ///
1998 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1999 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2000 ///
2001 /// Currently this must be a power-of-2 build vector.
2002 /// If passed a non-null UndefElements bitvector, it will resize it to match
2003 /// the original vector width and set the bits where elements are undef.
2004 /// If result is false, Sequence will be empty.
2005 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
2006 BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant or null if this is not a constant
2009 /// splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantSDNode *
2015 getConstantSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant or null if this is not a constant
2019 /// splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantSDNode *
2024 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// Returns the demanded splatted constant FP or null if this is not a
2027 /// constant FP splat.
2028 ///
2029 /// The DemandedElts mask indicates the elements that must be in the splat.
2030 /// If passed a non-null UndefElements bitvector, it will resize it to match
2031 /// the vector width and set the bits where elements are undef.
2032 ConstantFPSDNode *
2033 getConstantFPSplatNode(const APInt &DemandedElts,
2034 BitVector *UndefElements = nullptr) const;
2035
2036 /// Returns the splatted constant FP or null if this is not a constant
2037 /// FP splat.
2038 ///
2039 /// If passed a non-null UndefElements bitvector, it will resize it to match
2040 /// the vector width and set the bits where elements are undef.
2041 ConstantFPSDNode *
2042 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2043
2044 /// If this is a constant FP splat and the splatted constant FP is an
2045 /// exact power or 2, return the log base 2 integer value. Otherwise,
2046 /// return -1.
2047 ///
2048 /// The BitWidth specifies the necessary bit precision.
2049 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2050 uint32_t BitWidth) const;
2051
2052 /// Extract the raw bit data from a build vector of Undef, Constant or
2053 /// ConstantFP node elements. Each raw bit element will be \p
2054 /// DstEltSizeInBits wide, undef elements are treated as zero, and entirely
2055 /// undefined elements are flagged in \p UndefElements.
2056 bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits,
2057 SmallVectorImpl<APInt> &RawBitElements,
2058 BitVector &UndefElements) const;
2059
2060 bool isConstant() const;
2061
2062 static bool classof(const SDNode *N) {
2063 return N->getOpcode() == ISD::BUILD_VECTOR;
2064 }
2065};
2066
2067/// An SDNode that holds an arbitrary LLVM IR Value. This is
2068/// used when the SelectionDAG needs to make a simple reference to something
2069/// in the LLVM IR representation.
2070///
2071class SrcValueSDNode : public SDNode {
2072 friend class SelectionDAG;
2073
2074 const Value *V;
2075
2076 /// Create a SrcValue for a general value.
2077 explicit SrcValueSDNode(const Value *v)
2078 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2079
2080public:
2081 /// Return the contained Value.
2082 const Value *getValue() const { return V; }
2083
2084 static bool classof(const SDNode *N) {
2085 return N->getOpcode() == ISD::SRCVALUE;
2086 }
2087};
2088
2089class MDNodeSDNode : public SDNode {
2090 friend class SelectionDAG;
2091
2092 const MDNode *MD;
2093
2094 explicit MDNodeSDNode(const MDNode *md)
2095 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2096 {}
2097
2098public:
2099 const MDNode *getMD() const { return MD; }
2100
2101 static bool classof(const SDNode *N) {
2102 return N->getOpcode() == ISD::MDNODE_SDNODE;
2103 }
2104};
2105
2106class RegisterSDNode : public SDNode {
2107 friend class SelectionDAG;
2108
2109 Register Reg;
2110
2111 RegisterSDNode(Register reg, EVT VT)
2112 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2113
2114public:
2115 Register getReg() const { return Reg; }
2116
2117 static bool classof(const SDNode *N) {
2118 return N->getOpcode() == ISD::Register;
2119 }
2120};
2121
2122class RegisterMaskSDNode : public SDNode {
2123 friend class SelectionDAG;
2124
2125 // The memory for RegMask is not owned by the node.
2126 const uint32_t *RegMask;
2127
2128 RegisterMaskSDNode(const uint32_t *mask)
2129 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2130 RegMask(mask) {}
2131
2132public:
2133 const uint32_t *getRegMask() const { return RegMask; }
2134
2135 static bool classof(const SDNode *N) {
2136 return N->getOpcode() == ISD::RegisterMask;
2137 }
2138};
2139
2140class BlockAddressSDNode : public SDNode {
2141 friend class SelectionDAG;
2142
2143 const BlockAddress *BA;
2144 int64_t Offset;
2145 unsigned TargetFlags;
2146
2147 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2148 int64_t o, unsigned Flags)
2149 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2150 BA(ba), Offset(o), TargetFlags(Flags) {}
2151
2152public:
2153 const BlockAddress *getBlockAddress() const { return BA; }
2154 int64_t getOffset() const { return Offset; }
2155 unsigned getTargetFlags() const { return TargetFlags; }
2156
2157 static bool classof(const SDNode *N) {
2158 return N->getOpcode() == ISD::BlockAddress ||
2159 N->getOpcode() == ISD::TargetBlockAddress;
2160 }
2161};
2162
2163class LabelSDNode : public SDNode {
2164 friend class SelectionDAG;
2165
2166 MCSymbol *Label;
2167
2168 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2169 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2170 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2170, __extension__ __PRETTY_FUNCTION__));
2171 }
2172
2173public:
2174 MCSymbol *getLabel() const { return Label; }
2175
2176 static bool classof(const SDNode *N) {
2177 return N->getOpcode() == ISD::EH_LABEL ||
2178 N->getOpcode() == ISD::ANNOTATION_LABEL;
2179 }
2180};
2181
2182class ExternalSymbolSDNode : public SDNode {
2183 friend class SelectionDAG;
2184
2185 const char *Symbol;
2186 unsigned TargetFlags;
2187
2188 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2189 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2190 DebugLoc(), getSDVTList(VT)),
2191 Symbol(Sym), TargetFlags(TF) {}
2192
2193public:
2194 const char *getSymbol() const { return Symbol; }
2195 unsigned getTargetFlags() const { return TargetFlags; }
2196
2197 static bool classof(const SDNode *N) {
2198 return N->getOpcode() == ISD::ExternalSymbol ||
2199 N->getOpcode() == ISD::TargetExternalSymbol;
2200 }
2201};
2202
2203class MCSymbolSDNode : public SDNode {
2204 friend class SelectionDAG;
2205
2206 MCSymbol *Symbol;
2207
2208 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2209 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2210
2211public:
2212 MCSymbol *getMCSymbol() const { return Symbol; }
2213
2214 static bool classof(const SDNode *N) {
2215 return N->getOpcode() == ISD::MCSymbol;
2216 }
2217};
2218
2219class CondCodeSDNode : public SDNode {
2220 friend class SelectionDAG;
2221
2222 ISD::CondCode Condition;
2223
2224 explicit CondCodeSDNode(ISD::CondCode Cond)
2225 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2226 Condition(Cond) {}
2227
2228public:
2229 ISD::CondCode get() const { return Condition; }
2230
2231 static bool classof(const SDNode *N) {
2232 return N->getOpcode() == ISD::CONDCODE;
2233 }
2234};
2235
2236/// This class is used to represent EVT's, which are used
2237/// to parameterize some operations.
2238class VTSDNode : public SDNode {
2239 friend class SelectionDAG;
2240
2241 EVT ValueType;
2242
2243 explicit VTSDNode(EVT VT)
2244 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2245 ValueType(VT) {}
2246
2247public:
2248 EVT getVT() const { return ValueType; }
2249
2250 static bool classof(const SDNode *N) {
2251 return N->getOpcode() == ISD::VALUETYPE;
2252 }
2253};
2254
2255/// Base class for LoadSDNode and StoreSDNode
2256class LSBaseSDNode : public MemSDNode {
2257public:
2258 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2259 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2260 MachineMemOperand *MMO)
2261 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2262 LSBaseSDNodeBits.AddressingMode = AM;
2263 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2263, __extension__ __PRETTY_FUNCTION__));
2264 }
2265
2266 const SDValue &getOffset() const {
2267 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2268 }
2269
2270 /// Return the addressing mode for this load or store:
2271 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2272 ISD::MemIndexedMode getAddressingMode() const {
2273 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2274 }
2275
2276 /// Return true if this is a pre/post inc/dec load/store.
2277 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2278
2279 /// Return true if this is NOT a pre/post inc/dec load/store.
2280 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2281
2282 static bool classof(const SDNode *N) {
2283 return N->getOpcode() == ISD::LOAD ||
2284 N->getOpcode() == ISD::STORE;
2285 }
2286};
2287
2288/// This class is used to represent ISD::LOAD nodes.
2289class LoadSDNode : public LSBaseSDNode {
2290 friend class SelectionDAG;
2291
2292 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2293 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2294 MachineMemOperand *MMO)
2295 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2296 LoadSDNodeBits.ExtTy = ETy;
2297 assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2297, __extension__ __PRETTY_FUNCTION__));
2298 assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2298, __extension__ __PRETTY_FUNCTION__));
2299 }
2300
2301public:
2302 /// Return whether this is a plain node,
2303 /// or one of the varieties of value-extending loads.
2304 ISD::LoadExtType getExtensionType() const {
2305 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2306 }
2307
2308 const SDValue &getBasePtr() const { return getOperand(1); }
2309 const SDValue &getOffset() const { return getOperand(2); }
2310
2311 static bool classof(const SDNode *N) {
2312 return N->getOpcode() == ISD::LOAD;
2313 }
2314};
2315
2316/// This class is used to represent ISD::STORE nodes.
2317class StoreSDNode : public LSBaseSDNode {
2318 friend class SelectionDAG;
2319
2320 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2321 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2322 MachineMemOperand *MMO)
2323 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2324 StoreSDNodeBits.IsTruncating = isTrunc;
2325 assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2325, __extension__ __PRETTY_FUNCTION__));
2326 assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2326, __extension__ __PRETTY_FUNCTION__));
2327 }
2328
2329public:
2330 /// Return true if the op does a truncation before store.
2331 /// For integers this is the same as doing a TRUNCATE and storing the result.
2332 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2333 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2334 void setTruncatingStore(bool Truncating) {
2335 StoreSDNodeBits.IsTruncating = Truncating;
2336 }
2337
2338 const SDValue &getValue() const { return getOperand(1); }
2339 const SDValue &getBasePtr() const { return getOperand(2); }
2340 const SDValue &getOffset() const { return getOperand(3); }
2341
2342 static bool classof(const SDNode *N) {
2343 return N->getOpcode() == ISD::STORE;
2344 }
2345};
2346
2347/// This base class is used to represent VP_LOAD and VP_STORE nodes
2348class VPLoadStoreSDNode : public MemSDNode {
2349public:
2350 friend class SelectionDAG;
2351
2352 VPLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2353 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2354 MachineMemOperand *MMO)
2355 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2356 LSBaseSDNodeBits.AddressingMode = AM;
2357 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2357, __extension__ __PRETTY_FUNCTION__));
2358 }
2359
2360 // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
2361 // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
2362 // Mask is a vector of i1 elements;
2363 // the type of EVL is TLI.getVPExplicitVectorLengthTy().
2364 const SDValue &getOffset() const {
2365 return getOperand(getOpcode() == ISD::VP_LOAD ? 2 : 3);
2366 }
2367 const SDValue &getBasePtr() const {
2368 return getOperand(getOpcode() == ISD::VP_LOAD ? 1 : 2);
2369 }
2370 const SDValue &getMask() const {
2371 return getOperand(getOpcode() == ISD::VP_LOAD ? 3 : 4);
2372 }
2373 const SDValue &getVectorLength() const {
2374 return getOperand(getOpcode() == ISD::VP_LOAD ? 4 : 5);
2375 }
2376
2377 /// Return the addressing mode for this load or store:
2378 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2379 ISD::MemIndexedMode getAddressingMode() const {
2380 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2381 }
2382
2383 /// Return true if this is a pre/post inc/dec load/store.
2384 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2385
2386 /// Return true if this is NOT a pre/post inc/dec load/store.
2387 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2388
2389 static bool classof(const SDNode *N) {
2390 return N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
2391 }
2392};
2393
2394/// This class is used to represent a VP_LOAD node
2395class VPLoadSDNode : public VPLoadStoreSDNode {
2396public:
2397 friend class SelectionDAG;
2398
2399 VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2400 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
2401 EVT MemVT, MachineMemOperand *MMO)
2402 : VPLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2403 LoadSDNodeBits.ExtTy = ETy;
2404 LoadSDNodeBits.IsExpanding = isExpanding;
2405 }
2406
2407 ISD::LoadExtType getExtensionType() const {
2408 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2409 }
2410
2411 const SDValue &getBasePtr() const { return getOperand(1); }
2412 const SDValue &getOffset() const { return getOperand(2); }
2413 const SDValue &getMask() const { return getOperand(3); }
2414 const SDValue &getVectorLength() const { return getOperand(4); }
2415
2416 static bool classof(const SDNode *N) {
2417 return N->getOpcode() == ISD::VP_LOAD;
2418 }
2419 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2420};
2421
2422/// This class is used to represent a VP_STORE node
2423class VPStoreSDNode : public VPLoadStoreSDNode {
2424public:
2425 friend class SelectionDAG;
2426
2427 VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2428 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2429 EVT MemVT, MachineMemOperand *MMO)
2430 : VPLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
2431 StoreSDNodeBits.IsTruncating = isTrunc;
2432 StoreSDNodeBits.IsCompressing = isCompressing;
2433 }
2434
2435 /// Return true if this is a truncating store.
2436 /// For integers this is the same as doing a TRUNCATE and storing the result.
2437 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2438 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2439
2440 /// Returns true if the op does a compression to the vector before storing.
2441 /// The node contiguously stores the active elements (integers or floats)
2442 /// in src (those with their respective bit set in writemask k) to unaligned
2443 /// memory at base_addr.
2444 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2445
2446 const SDValue &getValue() const { return getOperand(1); }
2447 const SDValue &getBasePtr() const { return getOperand(2); }
2448 const SDValue &getOffset() const { return getOperand(3); }
2449 const SDValue &getMask() const { return getOperand(4); }
2450 const SDValue &getVectorLength() const { return getOperand(5); }
2451
2452 static bool classof(const SDNode *N) {
2453 return N->getOpcode() == ISD::VP_STORE;
2454 }
2455};
2456
2457/// This base class is used to represent MLOAD and MSTORE nodes
2458class MaskedLoadStoreSDNode : public MemSDNode {
2459public:
2460 friend class SelectionDAG;
2461
2462 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2463 const DebugLoc &dl, SDVTList VTs,
2464 ISD::MemIndexedMode AM, EVT MemVT,
2465 MachineMemOperand *MMO)
2466 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2467 LSBaseSDNodeBits.AddressingMode = AM;
2468 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2468, __extension__ __PRETTY_FUNCTION__));
2469 }
2470
2471 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2472 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2473 // Mask is a vector of i1 elements
2474 const SDValue &getOffset() const {
2475 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2476 }
2477 const SDValue &getMask() const {
2478 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2479 }
2480
2481 /// Return the addressing mode for this load or store:
2482 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2483 ISD::MemIndexedMode getAddressingMode() const {
2484 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2485 }
2486
2487 /// Return true if this is a pre/post inc/dec load/store.
2488 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2489
2490 /// Return true if this is NOT a pre/post inc/dec load/store.
2491 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2492
2493 static bool classof(const SDNode *N) {
2494 return N->getOpcode() == ISD::MLOAD ||
2495 N->getOpcode() == ISD::MSTORE;
2496 }
2497};
2498
2499/// This class is used to represent an MLOAD node
2500class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2501public:
2502 friend class SelectionDAG;
2503
2504 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2505 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2506 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2507 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2508 LoadSDNodeBits.ExtTy = ETy;
2509 LoadSDNodeBits.IsExpanding = IsExpanding;
2510 }
2511
2512 ISD::LoadExtType getExtensionType() const {
2513 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2514 }
2515
2516 const SDValue &getBasePtr() const { return getOperand(1); }
2517 const SDValue &getOffset() const { return getOperand(2); }
2518 const SDValue &getMask() const { return getOperand(3); }
2519 const SDValue &getPassThru() const { return getOperand(4); }
2520
2521 static bool classof(const SDNode *N) {
2522 return N->getOpcode() == ISD::MLOAD;
2523 }
2524
2525 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2526};
2527
2528/// This class is used to represent an MSTORE node
2529class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2530public:
2531 friend class SelectionDAG;
2532
2533 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2534 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2535 EVT MemVT, MachineMemOperand *MMO)
2536 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2537 StoreSDNodeBits.IsTruncating = isTrunc;
2538 StoreSDNodeBits.IsCompressing = isCompressing;
2539 }
2540
2541 /// Return true if the op does a truncation before store.
2542 /// For integers this is the same as doing a TRUNCATE and storing the result.
2543 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2544 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2545
2546 /// Returns true if the op does a compression to the vector before storing.
2547 /// The node contiguously stores the active elements (integers or floats)
2548 /// in src (those with their respective bit set in writemask k) to unaligned
2549 /// memory at base_addr.
2550 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2551
2552 const SDValue &getValue() const { return getOperand(1); }
2553 const SDValue &getBasePtr() const { return getOperand(2); }
2554 const SDValue &getOffset() const { return getOperand(3); }
2555 const SDValue &getMask() const { return getOperand(4); }
2556
2557 static bool classof(const SDNode *N) {
2558 return N->getOpcode() == ISD::MSTORE;
2559 }
2560};
2561
2562/// This is a base class used to represent
2563/// VP_GATHER and VP_SCATTER nodes
2564///
2565class VPGatherScatterSDNode : public MemSDNode {
2566public:
2567 friend class SelectionDAG;
2568
2569 VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2570 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2571 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2572 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2573 LSBaseSDNodeBits.AddressingMode = IndexType;
2574 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2574, __extension__ __PRETTY_FUNCTION__));
2575 }
2576
2577 /// How is Index applied to BasePtr when computing addresses.
2578 ISD::MemIndexType getIndexType() const {
2579 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2580 }
2581 bool isIndexScaled() const {
2582 return (getIndexType() == ISD::SIGNED_SCALED) ||
2583 (getIndexType() == ISD::UNSIGNED_SCALED);
2584 }
2585 bool isIndexSigned() const {
2586 return (getIndexType() == ISD::SIGNED_SCALED) ||
2587 (getIndexType() == ISD::SIGNED_UNSCALED);
2588 }
2589
2590 // In the both nodes address is Op1, mask is Op2:
2591 // VPGatherSDNode (Chain, base, index, scale, mask, vlen)
2592 // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
2593 // Mask is a vector of i1 elements
2594 const SDValue &getBasePtr() const {
2595 return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
2596 }
2597 const SDValue &getIndex() const {
2598 return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
2599 }
2600 const SDValue &getScale() const {
2601 return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
2602 }
2603 const SDValue &getMask() const {
2604 return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
2605 }
2606 const SDValue &getVectorLength() const {
2607 return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
2608 }
2609
2610 static bool classof(const SDNode *N) {
2611 return N->getOpcode() == ISD::VP_GATHER ||
2612 N->getOpcode() == ISD::VP_SCATTER;
2613 }
2614};
2615
2616/// This class is used to represent an VP_GATHER node
2617///
2618class VPGatherSDNode : public VPGatherScatterSDNode {
2619public:
2620 friend class SelectionDAG;
2621
2622 VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2623 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2624 : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
2625 IndexType) {}
2626
2627 static bool classof(const SDNode *N) {
2628 return N->getOpcode() == ISD::VP_GATHER;
2629 }
2630};
2631
2632/// This class is used to represent an VP_SCATTER node
2633///
2634class VPScatterSDNode : public VPGatherScatterSDNode {
2635public:
2636 friend class SelectionDAG;
2637
2638 VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2639 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2640 : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
2641 IndexType) {}
2642
2643 const SDValue &getValue() const { return getOperand(1); }
2644
2645 static bool classof(const SDNode *N) {
2646 return N->getOpcode() == ISD::VP_SCATTER;
2647 }
2648};
2649
2650/// This is a base class used to represent
2651/// MGATHER and MSCATTER nodes
2652///
2653class MaskedGatherScatterSDNode : public MemSDNode {
2654public:
2655 friend class SelectionDAG;
2656
2657 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2658 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2659 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2660 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2661 LSBaseSDNodeBits.AddressingMode = IndexType;
2662 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2662, __extension__ __PRETTY_FUNCTION__));
2663 }
2664
2665 /// How is Index applied to BasePtr when computing addresses.
2666 ISD::MemIndexType getIndexType() const {
2667 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2668 }
2669 void setIndexType(ISD::MemIndexType IndexType) {
2670 LSBaseSDNodeBits.AddressingMode = IndexType;
2671 }
2672 bool isIndexScaled() const {
2673 return (getIndexType() == ISD::SIGNED_SCALED) ||
2674 (getIndexType() == ISD::UNSIGNED_SCALED);
2675 }
2676 bool isIndexSigned() const {
2677 return (getIndexType() == ISD::SIGNED_SCALED) ||
2678 (getIndexType() == ISD::SIGNED_UNSCALED);
2679 }
2680
2681 // In the both nodes address is Op1, mask is Op2:
2682 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2683 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2684 // Mask is a vector of i1 elements
2685 const SDValue &getBasePtr() const { return getOperand(3); }
2686 const SDValue &getIndex() const { return getOperand(4); }
2687 const SDValue &getMask() const { return getOperand(2); }
2688 const SDValue &getScale() const { return getOperand(5); }
2689
2690 static bool classof(const SDNode *N) {
2691 return N->getOpcode() == ISD::MGATHER ||
2692 N->getOpcode() == ISD::MSCATTER;
2693 }
2694};
2695
2696/// This class is used to represent an MGATHER node
2697///
2698class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2699public:
2700 friend class SelectionDAG;
2701
2702 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2703 EVT MemVT, MachineMemOperand *MMO,
2704 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2705 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2706 IndexType) {
2707 LoadSDNodeBits.ExtTy = ETy;
2708 }
2709
2710 const SDValue &getPassThru() const { return getOperand(1); }
2711
2712 ISD::LoadExtType getExtensionType() const {
2713 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2714 }
2715
2716 static bool classof(const SDNode *N) {
2717 return N->getOpcode() == ISD::MGATHER;
2718 }
2719};
2720
2721/// This class is used to represent an MSCATTER node
2722///
2723class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2724public:
2725 friend class SelectionDAG;
2726
2727 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2728 EVT MemVT, MachineMemOperand *MMO,
2729 ISD::MemIndexType IndexType, bool IsTrunc)
2730 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2731 IndexType) {
2732 StoreSDNodeBits.IsTruncating = IsTrunc;
2733 }
2734
2735 /// Return true if the op does a truncation before store.
2736 /// For integers this is the same as doing a TRUNCATE and storing the result.
2737 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2738 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2739
2740 const SDValue &getValue() const { return getOperand(1); }
2741
2742 static bool classof(const SDNode *N) {
2743 return N->getOpcode() == ISD::MSCATTER;
2744 }
2745};
2746
2747/// An SDNode that represents everything that will be needed
2748/// to construct a MachineInstr. These nodes are created during the
2749/// instruction selection proper phase.
2750///
2751/// Note that the only supported way to set the `memoperands` is by calling the
2752/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2753/// inside the DAG rather than in the node.
2754class MachineSDNode : public SDNode {
2755private:
2756 friend class SelectionDAG;
2757
2758 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2759 : SDNode(Opc, Order, DL, VTs) {}
2760
2761 // We use a pointer union between a single `MachineMemOperand` pointer and
2762 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2763 // the number of these is zero, the single pointer variant used when the
2764 // number is one, and the array is used for larger numbers.
2765 //
2766 // The array is allocated via the `SelectionDAG`'s allocator and so will
2767 // always live until the DAG is cleaned up and doesn't require ownership here.
2768 //
2769 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2770 // subclasses aren't managed in a conforming C++ manner. See the comments on
2771 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2772 // constraint here is that these don't manage memory with their constructor or
2773 // destructor and can be initialized to a good state even if they start off
2774 // uninitialized.
2775 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2776
2777 // Note that this could be folded into the above `MemRefs` member if doing so
2778 // is advantageous at some point. We don't need to store this in most cases.
2779 // However, at the moment this doesn't appear to make the allocation any
2780 // smaller and makes the code somewhat simpler to read.
2781 int NumMemRefs = 0;
2782
2783public:
2784 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2785
2786 ArrayRef<MachineMemOperand *> memoperands() const {
2787 // Special case the common cases.
2788 if (NumMemRefs == 0)
2789 return {};
2790 if (NumMemRefs == 1)
2791 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2792
2793 // Otherwise we have an actual array.
2794 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2795 }
2796 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2797 mmo_iterator memoperands_end() const { return memoperands().end(); }
2798 bool memoperands_empty() const { return memoperands().empty(); }
2799
2800 /// Clear out the memory reference descriptor list.
2801 void clearMemRefs() {
2802 MemRefs = nullptr;
2803 NumMemRefs = 0;
2804 }
2805
2806 static bool classof(const SDNode *N) {
2807 return N->isMachineOpcode();
2808 }
2809};
2810
2811/// An SDNode that records if a register contains a value that is guaranteed to
2812/// be aligned accordingly.
2813class AssertAlignSDNode : public SDNode {
2814 Align Alignment;
2815
2816public:
2817 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2818 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2819
2820 Align getAlign() const { return Alignment; }
2821
2822 static bool classof(const SDNode *N) {
2823 return N->getOpcode() == ISD::AssertAlign;
2824 }
2825};
2826
2827class SDNodeIterator {
2828 const SDNode *Node;
2829 unsigned Operand;
2830
2831 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2832
2833public:
2834 using iterator_category = std::forward_iterator_tag;
2835 using value_type = SDNode;
2836 using difference_type = std::ptrdiff_t;
2837 using pointer = value_type *;
2838 using reference = value_type &;
2839
2840 bool operator==(const SDNodeIterator& x) const {
2841 return Operand == x.Operand;
2842 }
2843 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2844
2845 pointer operator*() const {
2846 return Node->getOperand(Operand).getNode();
2847 }
2848 pointer operator->() const { return operator*(); }
2849
2850 SDNodeIterator& operator++() { // Preincrement
2851 ++Operand;
2852 return *this;
2853 }
2854 SDNodeIterator operator++(int) { // Postincrement
2855 SDNodeIterator tmp = *this; ++*this; return tmp;
2856 }
2857 size_t operator-(SDNodeIterator Other) const {
2858 assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2859, __extension__ __PRETTY_FUNCTION__))
2859 "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211110111138+cffbfd01e37b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2859, __extension__ __PRETTY_FUNCTION__));
2860 return Operand - Other.Operand;
2861 }
2862
2863 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2864 static SDNodeIterator end (const SDNode *N) {
2865 return SDNodeIterator(N, N->getNumOperands());
2866 }
2867
2868 unsigned getOperand() const { return Operand; }
2869 const SDNode *getNode() const { return Node; }
2870};
2871
2872template <> struct GraphTraits<SDNode*> {
2873 using NodeRef = SDNode *;
2874 using ChildIteratorType = SDNodeIterator;
2875
2876 static NodeRef getEntryNode(SDNode *N) { return N; }
2877
2878 static ChildIteratorType child_begin(NodeRef N) {
2879 return SDNodeIterator::begin(N);
2880 }
2881
2882 static ChildIteratorType child_end(NodeRef N) {
2883 return SDNodeIterator::end(N);
2884 }
2885};
2886
2887/// A representation of the largest SDNode, for use in sizeof().
2888///
2889/// This needs to be a union because the largest node differs on 32 bit systems
2890/// with 4 and 8 byte pointer alignment, respectively.
2891using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2892 BlockAddressSDNode,
2893 GlobalAddressSDNode,
2894 PseudoProbeSDNode>;
2895
2896/// The SDNode class with the greatest alignment requirement.
2897using MostAlignedSDNode = GlobalAddressSDNode;
2898
2899namespace ISD {
2900
2901 /// Returns true if the specified node is a non-extending and unindexed load.
2902 inline bool isNormalLoad(const SDNode *N) {
2903 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2904 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2905 Ld->getAddressingMode() == ISD::UNINDEXED;
2906 }
2907
2908 /// Returns true if the specified node is a non-extending load.
2909 inline bool isNON_EXTLoad(const SDNode *N) {
2910 return isa<LoadSDNode>(N) &&
2911 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2912 }
2913
2914 /// Returns true if the specified node is a EXTLOAD.
2915 inline bool isEXTLoad(const SDNode *N) {
2916 return isa<LoadSDNode>(N) &&
2917 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2918 }
2919
2920 /// Returns true if the specified node is a SEXTLOAD.
2921 inline bool isSEXTLoad(const SDNode *N) {
2922 return isa<LoadSDNode>(N) &&
2923 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2924 }
2925
2926 /// Returns true if the specified node is a ZEXTLOAD.
2927 inline bool isZEXTLoad(const SDNode *N) {
2928 return isa<LoadSDNode>(N) &&
2929 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2930 }
2931
2932 /// Returns true if the specified node is an unindexed load.
2933 inline bool isUNINDEXEDLoad(const SDNode *N) {
2934 return isa<LoadSDNode>(N) &&
2935 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2936 }
2937
2938 /// Returns true if the specified node is a non-truncating
2939 /// and unindexed store.
2940 inline bool isNormalStore(const SDNode *N) {
2941 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2942 return St && !St->isTruncatingStore() &&
2943 St->getAddressingMode() == ISD::UNINDEXED;
2944 }
2945
2946 /// Returns true if the specified node is an unindexed store.
2947 inline bool isUNINDEXEDStore(const SDNode *N) {
2948 return isa<StoreSDNode>(N) &&
2949 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2950 }
2951
2952 /// Attempt to match a unary predicate against a scalar/splat constant or
2953 /// every element of a constant BUILD_VECTOR.
2954 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2955 bool matchUnaryPredicate(SDValue Op,
2956 std::function<bool(ConstantSDNode *)> Match,
2957 bool AllowUndefs = false);
2958
2959 /// Attempt to match a binary predicate against a pair of scalar/splat
2960 /// constants or every element of a pair of constant BUILD_VECTORs.
2961 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2962 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2963 bool matchBinaryPredicate(
2964 SDValue LHS, SDValue RHS,
2965 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2966 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2967
2968 /// Returns true if the specified value is the overflow result from one
2969 /// of the overflow intrinsic nodes.
2970 inline bool isOverflowIntrOpRes(SDValue Op) {
2971 unsigned Opc = Op.getOpcode();
2972 return (Op.getResNo() == 1 &&
2973 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2974 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2975 }
2976
2977} // end namespace ISD
2978
2979} // end namespace llvm
2980
2981#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H