LLVM  9.0.0svn
ARMISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the ARM target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARM.h"
14 #include "ARMBaseInstrInfo.h"
15 #include "ARMTargetMachine.h"
17 #include "Utils/ARMBaseInfo.h"
18 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/IR/CallingConv.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DerivedTypes.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/Intrinsics.h"
31 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "arm-isel"
40 
41 static cl::opt<bool>
42 DisableShifterOp("disable-shifter-op", cl::Hidden,
43  cl::desc("Disable isel of shifter-op"),
44  cl::init(false));
45 
46 //===--------------------------------------------------------------------===//
47 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
48 /// instructions for SelectionDAG operations.
49 ///
50 namespace {
51 
52 class ARMDAGToDAGISel : public SelectionDAGISel {
53  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
54  /// make the right decision when generating code for different targets.
55  const ARMSubtarget *Subtarget;
56 
57 public:
58  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
59  : SelectionDAGISel(tm, OptLevel) {}
60 
61  bool runOnMachineFunction(MachineFunction &MF) override {
62  // Reset the subtarget each time through.
63  Subtarget = &MF.getSubtarget<ARMSubtarget>();
65  return true;
66  }
67 
68  StringRef getPassName() const override { return "ARM Instruction Selection"; }
69 
70  void PreprocessISelDAG() override;
71 
72  /// getI32Imm - Return a target constant of type i32 with the specified
73  /// value.
74  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
75  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
76  }
77 
78  void Select(SDNode *N) override;
79 
80  bool hasNoVMLxHazardUse(SDNode *N) const;
81  bool isShifterOpProfitable(const SDValue &Shift,
82  ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
83  bool SelectRegShifterOperand(SDValue N, SDValue &A,
84  SDValue &B, SDValue &C,
85  bool CheckProfitability = true);
86  bool SelectImmShifterOperand(SDValue N, SDValue &A,
87  SDValue &B, bool CheckProfitability = true);
88  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
89  SDValue &B, SDValue &C) {
90  // Don't apply the profitability check
91  return SelectRegShifterOperand(N, A, B, C, false);
92  }
93  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
94  SDValue &B) {
95  // Don't apply the profitability check
96  return SelectImmShifterOperand(N, A, B, false);
97  }
98 
99  bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
100 
101  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
102  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
103 
104  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
105  const ConstantSDNode *CN = cast<ConstantSDNode>(N);
106  Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
107  Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
108  return true;
109  }
110 
111  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
112  SDValue &Offset, SDValue &Opc);
113  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
114  SDValue &Offset, SDValue &Opc);
115  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
116  SDValue &Offset, SDValue &Opc);
117  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
118  bool SelectAddrMode3(SDValue N, SDValue &Base,
119  SDValue &Offset, SDValue &Opc);
120  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
121  SDValue &Offset, SDValue &Opc);
122  bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
123  bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
124  bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
125  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
126  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
127 
128  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
129 
130  // Thumb Addressing Modes:
131  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
132  bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
133  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
134  SDValue &OffImm);
135  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
136  SDValue &OffImm);
137  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
138  SDValue &OffImm);
139  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
140  SDValue &OffImm);
141  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
142 
143  // Thumb 2 Addressing Modes:
144  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
145  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
146  SDValue &OffImm);
147  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
148  SDValue &OffImm);
149  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
150  SDValue &OffReg, SDValue &ShImm);
151  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
152 
153  inline bool is_so_imm(unsigned Imm) const {
154  return ARM_AM::getSOImmVal(Imm) != -1;
155  }
156 
157  inline bool is_so_imm_not(unsigned Imm) const {
158  return ARM_AM::getSOImmVal(~Imm) != -1;
159  }
160 
161  inline bool is_t2_so_imm(unsigned Imm) const {
162  return ARM_AM::getT2SOImmVal(Imm) != -1;
163  }
164 
165  inline bool is_t2_so_imm_not(unsigned Imm) const {
166  return ARM_AM::getT2SOImmVal(~Imm) != -1;
167  }
168 
169  // Include the pieces autogenerated from the target description.
170 #include "ARMGenDAGISel.inc"
171 
172 private:
173  void transferMemOperands(SDNode *Src, SDNode *Dst);
174 
175  /// Indexed (pre/post inc/dec) load matching code for ARM.
176  bool tryARMIndexedLoad(SDNode *N);
177  bool tryT1IndexedLoad(SDNode *N);
178  bool tryT2IndexedLoad(SDNode *N);
179 
180  /// SelectVLD - Select NEON load intrinsics. NumVecs should be
181  /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
182  /// loads of D registers and even subregs and odd subregs of Q registers.
183  /// For NumVecs <= 2, QOpcodes1 is not used.
184  void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
185  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
186  const uint16_t *QOpcodes1);
187 
188  /// SelectVST - Select NEON store intrinsics. NumVecs should
189  /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
190  /// stores of D registers and even subregs and odd subregs of Q registers.
191  /// For NumVecs <= 2, QOpcodes1 is not used.
192  void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
193  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
194  const uint16_t *QOpcodes1);
195 
196  /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
197  /// be 2, 3 or 4. The opcode arrays specify the instructions used for
198  /// load/store of D registers and Q registers.
199  void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
200  unsigned NumVecs, const uint16_t *DOpcodes,
201  const uint16_t *QOpcodes);
202 
203  /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
204  /// should be 1, 2, 3 or 4. The opcode array specifies the instructions used
205  /// for loading D registers.
206  void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
207  unsigned NumVecs, const uint16_t *DOpcodes,
208  const uint16_t *QOpcodes0 = nullptr,
209  const uint16_t *QOpcodes1 = nullptr);
210 
211  /// Try to select SBFX/UBFX instructions for ARM.
212  bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
213 
214  // Select special operations if node forms integer ABS pattern
215  bool tryABSOp(SDNode *N);
216 
217  bool tryReadRegister(SDNode *N);
218  bool tryWriteRegister(SDNode *N);
219 
220  bool tryInlineAsm(SDNode *N);
221 
222  void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
223 
224  void SelectCMP_SWAP(SDNode *N);
225 
226  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
227  /// inline asm expressions.
228  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
229  std::vector<SDValue> &OutOps) override;
230 
231  // Form pairs of consecutive R, S, D, or Q registers.
233  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
234  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
235  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
236 
237  // Form sequences of 4 consecutive S, D, or Q registers.
238  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
239  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
240  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
241 
242  // Get the alignment operand for a NEON VLD or VST instruction.
243  SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
244  bool is64BitVector);
245 
246  /// Returns the number of instructions required to materialize the given
247  /// constant in a register, or 3 if a literal pool load is needed.
248  unsigned ConstantMaterializationCost(unsigned Val) const;
249 
250  /// Checks if N is a multiplication by a constant where we can extract out a
251  /// power of two from the constant so that it can be used in a shift, but only
252  /// if it simplifies the materialization of the constant. Returns true if it
253  /// is, and assigns to PowerOfTwo the power of two that should be extracted
254  /// out and to NewMulConst the new constant to be multiplied by.
255  bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
256  unsigned &PowerOfTwo, SDValue &NewMulConst) const;
257 
258  /// Replace N with M in CurDAG, in a way that also ensures that M gets
259  /// selected when N would have been selected.
260  void replaceDAGValue(const SDValue &N, SDValue M);
261 };
262 }
263 
264 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
265 /// operand. If so Imm will receive the 32-bit value.
266 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
267  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
268  Imm = cast<ConstantSDNode>(N)->getZExtValue();
269  return true;
270  }
271  return false;
272 }
273 
274 // isInt32Immediate - This method tests to see if a constant operand.
275 // If so Imm will receive the 32 bit value.
276 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
277  return isInt32Immediate(N.getNode(), Imm);
278 }
279 
280 // isOpcWithIntImmediate - This method tests to see if the node is a specific
281 // opcode and that it has a immediate integer right operand.
282 // If so Imm will receive the 32 bit value.
283 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
284  return N->getOpcode() == Opc &&
285  isInt32Immediate(N->getOperand(1).getNode(), Imm);
286 }
287 
288 /// Check whether a particular node is a constant value representable as
289 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
290 ///
291 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
292 static bool isScaledConstantInRange(SDValue Node, int Scale,
293  int RangeMin, int RangeMax,
294  int &ScaledConstant) {
295  assert(Scale > 0 && "Invalid scale!");
296 
297  // Check that this is a constant.
298  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
299  if (!C)
300  return false;
301 
302  ScaledConstant = (int) C->getZExtValue();
303  if ((ScaledConstant % Scale) != 0)
304  return false;
305 
306  ScaledConstant /= Scale;
307  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
308 }
309 
310 void ARMDAGToDAGISel::PreprocessISelDAG() {
311  if (!Subtarget->hasV6T2Ops())
312  return;
313 
314  bool isThumb2 = Subtarget->isThumb();
315  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
316  E = CurDAG->allnodes_end(); I != E; ) {
317  SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
318 
319  if (N->getOpcode() != ISD::ADD)
320  continue;
321 
322  // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
323  // leading zeros, followed by consecutive set bits, followed by 1 or 2
324  // trailing zeros, e.g. 1020.
325  // Transform the expression to
326  // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
327  // of trailing zeros of c2. The left shift would be folded as an shifter
328  // operand of 'add' and the 'and' and 'srl' would become a bits extraction
329  // node (UBFX).
330 
331  SDValue N0 = N->getOperand(0);
332  SDValue N1 = N->getOperand(1);
333  unsigned And_imm = 0;
334  if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
335  if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
336  std::swap(N0, N1);
337  }
338  if (!And_imm)
339  continue;
340 
341  // Check if the AND mask is an immediate of the form: 000.....1111111100
342  unsigned TZ = countTrailingZeros(And_imm);
343  if (TZ != 1 && TZ != 2)
344  // Be conservative here. Shifter operands aren't always free. e.g. On
345  // Swift, left shifter operand of 1 / 2 for free but others are not.
346  // e.g.
347  // ubfx r3, r1, #16, #8
348  // ldr.w r3, [r0, r3, lsl #2]
349  // vs.
350  // mov.w r9, #1020
351  // and.w r2, r9, r1, lsr #14
352  // ldr r2, [r0, r2]
353  continue;
354  And_imm >>= TZ;
355  if (And_imm & (And_imm + 1))
356  continue;
357 
358  // Look for (and (srl X, c1), c2).
359  SDValue Srl = N1.getOperand(0);
360  unsigned Srl_imm = 0;
361  if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
362  (Srl_imm <= 2))
363  continue;
364 
365  // Make sure first operand is not a shifter operand which would prevent
366  // folding of the left shift.
367  SDValue CPTmp0;
368  SDValue CPTmp1;
369  SDValue CPTmp2;
370  if (isThumb2) {
371  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
372  continue;
373  } else {
374  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
375  SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
376  continue;
377  }
378 
379  // Now make the transformation.
380  Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
381  Srl.getOperand(0),
382  CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
383  MVT::i32));
384  N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
385  Srl,
386  CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
387  N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
388  N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
389  CurDAG->UpdateNodeOperands(N, N0, N1);
390  }
391 }
392 
393 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
394 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
395 /// least on current ARM implementations) which should be avoidded.
396 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
397  if (OptLevel == CodeGenOpt::None)
398  return true;
399 
400  if (!Subtarget->hasVMLxHazards())
401  return true;
402 
403  if (!N->hasOneUse())
404  return false;
405 
406  SDNode *Use = *N->use_begin();
407  if (Use->getOpcode() == ISD::CopyToReg)
408  return true;
409  if (Use->isMachineOpcode()) {
410  const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
411  CurDAG->getSubtarget().getInstrInfo());
412 
413  const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
414  if (MCID.mayStore())
415  return true;
416  unsigned Opcode = MCID.getOpcode();
417  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
418  return true;
419  // vmlx feeding into another vmlx. We actually want to unfold
420  // the use later in the MLxExpansion pass. e.g.
421  // vmla
422  // vmla (stall 8 cycles)
423  //
424  // vmul (5 cycles)
425  // vadd (5 cycles)
426  // vmla
427  // This adds up to about 18 - 19 cycles.
428  //
429  // vmla
430  // vmul (stall 4 cycles)
431  // vadd adds up to about 14 cycles.
432  return TII->isFpMLxInstruction(Opcode);
433  }
434 
435  return false;
436 }
437 
438 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
439  ARM_AM::ShiftOpc ShOpcVal,
440  unsigned ShAmt) {
441  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
442  return true;
443  if (Shift.hasOneUse())
444  return true;
445  // R << 2 is free.
446  return ShOpcVal == ARM_AM::lsl &&
447  (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
448 }
449 
450 unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
451  if (Subtarget->isThumb()) {
452  if (Val <= 255) return 1; // MOV
453  if (Subtarget->hasV6T2Ops() &&
454  (Val <= 0xffff || // MOV
455  ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
456  ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
457  return 1;
458  if (Val <= 510) return 2; // MOV + ADDi8
459  if (~Val <= 255) return 2; // MOV + MVN
460  if (ARM_AM::isThumbImmShiftedVal(Val)) return 2; // MOV + LSL
461  } else {
462  if (ARM_AM::getSOImmVal(Val) != -1) return 1; // MOV
463  if (ARM_AM::getSOImmVal(~Val) != -1) return 1; // MVN
464  if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
465  if (ARM_AM::isSOImmTwoPartVal(Val)) return 2; // two instrs
466  }
467  if (Subtarget->useMovt()) return 2; // MOVW + MOVT
468  return 3; // Literal pool load
469 }
470 
471 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
472  unsigned MaxShift,
473  unsigned &PowerOfTwo,
474  SDValue &NewMulConst) const {
475  assert(N.getOpcode() == ISD::MUL);
476  assert(MaxShift > 0);
477 
478  // If the multiply is used in more than one place then changing the constant
479  // will make other uses incorrect, so don't.
480  if (!N.hasOneUse()) return false;
481  // Check if the multiply is by a constant
483  if (!MulConst) return false;
484  // If the constant is used in more than one place then modifying it will mean
485  // we need to materialize two constants instead of one, which is a bad idea.
486  if (!MulConst->hasOneUse()) return false;
487  unsigned MulConstVal = MulConst->getZExtValue();
488  if (MulConstVal == 0) return false;
489 
490  // Find the largest power of 2 that MulConstVal is a multiple of
491  PowerOfTwo = MaxShift;
492  while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
493  --PowerOfTwo;
494  if (PowerOfTwo == 0) return false;
495  }
496 
497  // Only optimise if the new cost is better
498  unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
499  NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
500  unsigned OldCost = ConstantMaterializationCost(MulConstVal);
501  unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
502  return NewCost < OldCost;
503 }
504 
505 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
506  CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
507  ReplaceUses(N, M);
508 }
509 
510 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
511  SDValue &BaseReg,
512  SDValue &Opc,
513  bool CheckProfitability) {
514  if (DisableShifterOp)
515  return false;
516 
517  // If N is a multiply-by-constant and it's profitable to extract a shift and
518  // use it in a shifted operand do so.
519  if (N.getOpcode() == ISD::MUL) {
520  unsigned PowerOfTwo = 0;
521  SDValue NewMulConst;
522  if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
523  HandleSDNode Handle(N);
524  SDLoc Loc(N);
525  replaceDAGValue(N.getOperand(1), NewMulConst);
526  BaseReg = Handle.getValue();
527  Opc = CurDAG->getTargetConstant(
528  ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
529  return true;
530  }
531  }
532 
534 
535  // Don't match base register only case. That is matched to a separate
536  // lower complexity pattern with explicit register operand.
537  if (ShOpcVal == ARM_AM::no_shift) return false;
538 
539  BaseReg = N.getOperand(0);
540  unsigned ShImmVal = 0;
542  if (!RHS) return false;
543  ShImmVal = RHS->getZExtValue() & 31;
544  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
545  SDLoc(N), MVT::i32);
546  return true;
547 }
548 
549 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
550  SDValue &BaseReg,
551  SDValue &ShReg,
552  SDValue &Opc,
553  bool CheckProfitability) {
554  if (DisableShifterOp)
555  return false;
556 
558 
559  // Don't match base register only case. That is matched to a separate
560  // lower complexity pattern with explicit register operand.
561  if (ShOpcVal == ARM_AM::no_shift) return false;
562 
563  BaseReg = N.getOperand(0);
564  unsigned ShImmVal = 0;
566  if (RHS) return false;
567 
568  ShReg = N.getOperand(1);
569  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
570  return false;
571  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
572  SDLoc(N), MVT::i32);
573  return true;
574 }
575 
576 // Determine whether an ISD::OR's operands are suitable to turn the operation
577 // into an addition, which often has more compact encodings.
578 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
579  assert(Parent->getOpcode() == ISD::OR && "unexpected parent");
580  Out = N;
581  return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
582 }
583 
584 
585 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
586  SDValue &Base,
587  SDValue &OffImm) {
588  // Match simple R + imm12 operands.
589 
590  // Base only.
591  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
592  !CurDAG->isBaseWithConstantOffset(N)) {
593  if (N.getOpcode() == ISD::FrameIndex) {
594  // Match frame index.
595  int FI = cast<FrameIndexSDNode>(N)->getIndex();
596  Base = CurDAG->getTargetFrameIndex(
597  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
598  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
599  return true;
600  }
601 
602  if (N.getOpcode() == ARMISD::Wrapper &&
606  Base = N.getOperand(0);
607  } else
608  Base = N;
609  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
610  return true;
611  }
612 
613  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
614  int RHSC = (int)RHS->getSExtValue();
615  if (N.getOpcode() == ISD::SUB)
616  RHSC = -RHSC;
617 
618  if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
619  Base = N.getOperand(0);
620  if (Base.getOpcode() == ISD::FrameIndex) {
621  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
622  Base = CurDAG->getTargetFrameIndex(
623  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
624  }
625  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
626  return true;
627  }
628  }
629 
630  // Base only.
631  Base = N;
632  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
633  return true;
634 }
635 
636 
637 
638 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
639  SDValue &Opc) {
640  if (N.getOpcode() == ISD::MUL &&
641  ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
642  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
643  // X * [3,5,9] -> X + X * [2,4,8] etc.
644  int RHSC = (int)RHS->getZExtValue();
645  if (RHSC & 1) {
646  RHSC = RHSC & ~1;
647  ARM_AM::AddrOpc AddSub = ARM_AM::add;
648  if (RHSC < 0) {
649  AddSub = ARM_AM::sub;
650  RHSC = - RHSC;
651  }
652  if (isPowerOf2_32(RHSC)) {
653  unsigned ShAmt = Log2_32(RHSC);
654  Base = Offset = N.getOperand(0);
655  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
656  ARM_AM::lsl),
657  SDLoc(N), MVT::i32);
658  return true;
659  }
660  }
661  }
662  }
663 
664  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
665  // ISD::OR that is equivalent to an ISD::ADD.
666  !CurDAG->isBaseWithConstantOffset(N))
667  return false;
668 
669  // Leave simple R +/- imm12 operands for LDRi12
670  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
671  int RHSC;
672  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
673  -0x1000+1, 0x1000, RHSC)) // 12 bits.
674  return false;
675  }
676 
677  // Otherwise this is R +/- [possibly shifted] R.
679  ARM_AM::ShiftOpc ShOpcVal =
681  unsigned ShAmt = 0;
682 
683  Base = N.getOperand(0);
684  Offset = N.getOperand(1);
685 
686  if (ShOpcVal != ARM_AM::no_shift) {
687  // Check to see if the RHS of the shift is a constant, if not, we can't fold
688  // it.
689  if (ConstantSDNode *Sh =
690  dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
691  ShAmt = Sh->getZExtValue();
692  if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
693  Offset = N.getOperand(1).getOperand(0);
694  else {
695  ShAmt = 0;
696  ShOpcVal = ARM_AM::no_shift;
697  }
698  } else {
699  ShOpcVal = ARM_AM::no_shift;
700  }
701  }
702 
703  // Try matching (R shl C) + (R).
704  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
705  !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
706  N.getOperand(0).hasOneUse())) {
708  if (ShOpcVal != ARM_AM::no_shift) {
709  // Check to see if the RHS of the shift is a constant, if not, we can't
710  // fold it.
711  if (ConstantSDNode *Sh =
712  dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
713  ShAmt = Sh->getZExtValue();
714  if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
715  Offset = N.getOperand(0).getOperand(0);
716  Base = N.getOperand(1);
717  } else {
718  ShAmt = 0;
719  ShOpcVal = ARM_AM::no_shift;
720  }
721  } else {
722  ShOpcVal = ARM_AM::no_shift;
723  }
724  }
725  }
726 
727  // If Offset is a multiply-by-constant and it's profitable to extract a shift
728  // and use it in a shifted operand do so.
729  if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
730  unsigned PowerOfTwo = 0;
731  SDValue NewMulConst;
732  if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
733  HandleSDNode Handle(Offset);
734  replaceDAGValue(Offset.getOperand(1), NewMulConst);
735  Offset = Handle.getValue();
736  ShAmt = PowerOfTwo;
737  ShOpcVal = ARM_AM::lsl;
738  }
739  }
740 
741  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
742  SDLoc(N), MVT::i32);
743  return true;
744 }
745 
746 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
747  SDValue &Offset, SDValue &Opc) {
748  unsigned Opcode = Op->getOpcode();
749  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
750  ? cast<LoadSDNode>(Op)->getAddressingMode()
751  : cast<StoreSDNode>(Op)->getAddressingMode();
752  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
754  int Val;
755  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
756  return false;
757 
758  Offset = N;
760  unsigned ShAmt = 0;
761  if (ShOpcVal != ARM_AM::no_shift) {
762  // Check to see if the RHS of the shift is a constant, if not, we can't fold
763  // it.
764  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
765  ShAmt = Sh->getZExtValue();
766  if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
767  Offset = N.getOperand(0);
768  else {
769  ShAmt = 0;
770  ShOpcVal = ARM_AM::no_shift;
771  }
772  } else {
773  ShOpcVal = ARM_AM::no_shift;
774  }
775  }
776 
777  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
778  SDLoc(N), MVT::i32);
779  return true;
780 }
781 
782 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
783  SDValue &Offset, SDValue &Opc) {
784  unsigned Opcode = Op->getOpcode();
785  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
786  ? cast<LoadSDNode>(Op)->getAddressingMode()
787  : cast<StoreSDNode>(Op)->getAddressingMode();
788  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
790  int Val;
791  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
792  if (AddSub == ARM_AM::sub) Val *= -1;
793  Offset = CurDAG->getRegister(0, MVT::i32);
794  Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
795  return true;
796  }
797 
798  return false;
799 }
800 
801 
802 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
803  SDValue &Offset, SDValue &Opc) {
804  unsigned Opcode = Op->getOpcode();
805  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
806  ? cast<LoadSDNode>(Op)->getAddressingMode()
807  : cast<StoreSDNode>(Op)->getAddressingMode();
808  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
810  int Val;
811  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
812  Offset = CurDAG->getRegister(0, MVT::i32);
813  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
815  SDLoc(Op), MVT::i32);
816  return true;
817  }
818 
819  return false;
820 }
821 
822 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
823  Base = N;
824  return true;
825 }
826 
827 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
828  SDValue &Base, SDValue &Offset,
829  SDValue &Opc) {
830  if (N.getOpcode() == ISD::SUB) {
831  // X - C is canonicalize to X + -C, no need to handle it here.
832  Base = N.getOperand(0);
833  Offset = N.getOperand(1);
834  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
835  MVT::i32);
836  return true;
837  }
838 
839  if (!CurDAG->isBaseWithConstantOffset(N)) {
840  Base = N;
841  if (N.getOpcode() == ISD::FrameIndex) {
842  int FI = cast<FrameIndexSDNode>(N)->getIndex();
843  Base = CurDAG->getTargetFrameIndex(
844  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
845  }
846  Offset = CurDAG->getRegister(0, MVT::i32);
847  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
848  MVT::i32);
849  return true;
850  }
851 
852  // If the RHS is +/- imm8, fold into addr mode.
853  int RHSC;
854  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
855  -256 + 1, 256, RHSC)) { // 8 bits.
856  Base = N.getOperand(0);
857  if (Base.getOpcode() == ISD::FrameIndex) {
858  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
859  Base = CurDAG->getTargetFrameIndex(
860  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
861  }
862  Offset = CurDAG->getRegister(0, MVT::i32);
863 
864  ARM_AM::AddrOpc AddSub = ARM_AM::add;
865  if (RHSC < 0) {
866  AddSub = ARM_AM::sub;
867  RHSC = -RHSC;
868  }
869  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
870  MVT::i32);
871  return true;
872  }
873 
874  Base = N.getOperand(0);
875  Offset = N.getOperand(1);
876  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
877  MVT::i32);
878  return true;
879 }
880 
881 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
882  SDValue &Offset, SDValue &Opc) {
883  unsigned Opcode = Op->getOpcode();
884  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
885  ? cast<LoadSDNode>(Op)->getAddressingMode()
886  : cast<StoreSDNode>(Op)->getAddressingMode();
887  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
889  int Val;
890  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
891  Offset = CurDAG->getRegister(0, MVT::i32);
892  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
893  MVT::i32);
894  return true;
895  }
896 
897  Offset = N;
898  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
899  MVT::i32);
900  return true;
901 }
902 
903 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
904  bool FP16) {
905  if (!CurDAG->isBaseWithConstantOffset(N)) {
906  Base = N;
907  if (N.getOpcode() == ISD::FrameIndex) {
908  int FI = cast<FrameIndexSDNode>(N)->getIndex();
909  Base = CurDAG->getTargetFrameIndex(
910  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
911  } else if (N.getOpcode() == ARMISD::Wrapper &&
915  Base = N.getOperand(0);
916  }
917  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
918  SDLoc(N), MVT::i32);
919  return true;
920  }
921 
922  // If the RHS is +/- imm8, fold into addr mode.
923  int RHSC;
924  const int Scale = FP16 ? 2 : 4;
925 
926  if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) {
927  Base = N.getOperand(0);
928  if (Base.getOpcode() == ISD::FrameIndex) {
929  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
930  Base = CurDAG->getTargetFrameIndex(
931  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
932  }
933 
934  ARM_AM::AddrOpc AddSub = ARM_AM::add;
935  if (RHSC < 0) {
936  AddSub = ARM_AM::sub;
937  RHSC = -RHSC;
938  }
939 
940  if (FP16)
941  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
942  SDLoc(N), MVT::i32);
943  else
944  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
945  SDLoc(N), MVT::i32);
946 
947  return true;
948  }
949 
950  Base = N;
951 
952  if (FP16)
953  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
954  SDLoc(N), MVT::i32);
955  else
956  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
957  SDLoc(N), MVT::i32);
958 
959  return true;
960 }
961 
962 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
963  SDValue &Base, SDValue &Offset) {
964  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ false);
965 }
966 
967 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
968  SDValue &Base, SDValue &Offset) {
969  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ true);
970 }
971 
972 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
973  SDValue &Align) {
974  Addr = N;
975 
976  unsigned Alignment = 0;
977 
978  MemSDNode *MemN = cast<MemSDNode>(Parent);
979 
980  if (isa<LSBaseSDNode>(MemN) ||
981  ((MemN->getOpcode() == ARMISD::VST1_UPD ||
982  MemN->getOpcode() == ARMISD::VLD1_UPD) &&
983  MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
984  // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
985  // The maximum alignment is equal to the memory size being referenced.
986  unsigned MMOAlign = MemN->getAlignment();
987  unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
988  if (MMOAlign >= MemSize && MemSize > 1)
989  Alignment = MemSize;
990  } else {
991  // All other uses of addrmode6 are for intrinsics. For now just record
992  // the raw alignment value; it will be refined later based on the legal
993  // alignment operands for the intrinsic.
994  Alignment = MemN->getAlignment();
995  }
996 
997  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
998  return true;
999 }
1000 
1001 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1002  SDValue &Offset) {
1003  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1005  if (AM != ISD::POST_INC)
1006  return false;
1007  Offset = N;
1008  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1009  if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1010  Offset = CurDAG->getRegister(0, MVT::i32);
1011  }
1012  return true;
1013 }
1014 
1015 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1016  SDValue &Offset, SDValue &Label) {
1017  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1018  Offset = N.getOperand(0);
1019  SDValue N1 = N.getOperand(1);
1020  Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1021  SDLoc(N), MVT::i32);
1022  return true;
1023  }
1024 
1025  return false;
1026 }
1027 
1028 
1029 //===----------------------------------------------------------------------===//
1030 // Thumb Addressing Modes
1031 //===----------------------------------------------------------------------===//
1032 
1034  // Negative numbers are difficult to materialise in thumb1. If we are
1035  // selecting the add of a negative, instead try to select ri with a zero
1036  // offset, so create the add node directly which will become a sub.
1037  if (N.getOpcode() != ISD::ADD)
1038  return false;
1039 
1040  // Look for an imm which is not legal for ld/st, but is legal for sub.
1041  if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1)))
1042  return C->getSExtValue() < 0 && C->getSExtValue() >= -255;
1043 
1044  return false;
1045 }
1046 
1047 bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1048  SDValue &Offset) {
1049  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1051  if (!NC || !NC->isNullValue())
1052  return false;
1053 
1054  Base = Offset = N;
1055  return true;
1056  }
1057 
1058  Base = N.getOperand(0);
1059  Offset = N.getOperand(1);
1060  return true;
1061 }
1062 
1063 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1064  SDValue &Offset) {
1065  if (shouldUseZeroOffsetLdSt(N))
1066  return false; // Select ri instead
1067  return SelectThumbAddrModeRRSext(N, Base, Offset);
1068 }
1069 
1070 bool
1071 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1072  SDValue &Base, SDValue &OffImm) {
1073  if (shouldUseZeroOffsetLdSt(N)) {
1074  Base = N;
1075  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1076  return true;
1077  }
1078 
1079  if (!CurDAG->isBaseWithConstantOffset(N)) {
1080  if (N.getOpcode() == ISD::ADD) {
1081  return false; // We want to select register offset instead
1082  } else if (N.getOpcode() == ARMISD::Wrapper &&
1087  Base = N.getOperand(0);
1088  } else {
1089  Base = N;
1090  }
1091 
1092  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1093  return true;
1094  }
1095 
1096  // If the RHS is + imm5 * scale, fold into addr mode.
1097  int RHSC;
1098  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1099  Base = N.getOperand(0);
1100  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1101  return true;
1102  }
1103 
1104  // Offset is too large, so use register offset instead.
1105  return false;
1106 }
1107 
1108 bool
1109 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1110  SDValue &OffImm) {
1111  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1112 }
1113 
1114 bool
1115 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1116  SDValue &OffImm) {
1117  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1118 }
1119 
1120 bool
1121 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1122  SDValue &OffImm) {
1123  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1124 }
1125 
1126 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1127  SDValue &Base, SDValue &OffImm) {
1128  if (N.getOpcode() == ISD::FrameIndex) {
1129  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1130  // Only multiples of 4 are allowed for the offset, so the frame object
1131  // alignment must be at least 4.
1132  MachineFrameInfo &MFI = MF->getFrameInfo();
1133  if (MFI.getObjectAlignment(FI) < 4)
1134  MFI.setObjectAlignment(FI, 4);
1135  Base = CurDAG->getTargetFrameIndex(
1136  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1137  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1138  return true;
1139  }
1140 
1141  if (!CurDAG->isBaseWithConstantOffset(N))
1142  return false;
1143 
1144  if (N.getOperand(0).getOpcode() == ISD::FrameIndex) {
1145  // If the RHS is + imm8 * scale, fold into addr mode.
1146  int RHSC;
1147  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1148  Base = N.getOperand(0);
1149  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1150  // For LHS+RHS to result in an offset that's a multiple of 4 the object
1151  // indexed by the LHS must be 4-byte aligned.
1152  MachineFrameInfo &MFI = MF->getFrameInfo();
1153  if (MFI.getObjectAlignment(FI) < 4)
1154  MFI.setObjectAlignment(FI, 4);
1155  Base = CurDAG->getTargetFrameIndex(
1156  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1157  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1158  return true;
1159  }
1160  }
1161 
1162  return false;
1163 }
1164 
1165 
1166 //===----------------------------------------------------------------------===//
1167 // Thumb 2 Addressing Modes
1168 //===----------------------------------------------------------------------===//
1169 
1170 
1171 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1172  SDValue &Base, SDValue &OffImm) {
1173  // Match simple R + imm12 operands.
1174 
1175  // Base only.
1176  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1177  !CurDAG->isBaseWithConstantOffset(N)) {
1178  if (N.getOpcode() == ISD::FrameIndex) {
1179  // Match frame index.
1180  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1181  Base = CurDAG->getTargetFrameIndex(
1182  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1183  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1184  return true;
1185  }
1186 
1187  if (N.getOpcode() == ARMISD::Wrapper &&
1191  Base = N.getOperand(0);
1192  if (Base.getOpcode() == ISD::TargetConstantPool)
1193  return false; // We want to select t2LDRpci instead.
1194  } else
1195  Base = N;
1196  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1197  return true;
1198  }
1199 
1200  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1201  if (SelectT2AddrModeImm8(N, Base, OffImm))
1202  // Let t2LDRi8 handle (R - imm8).
1203  return false;
1204 
1205  int RHSC = (int)RHS->getZExtValue();
1206  if (N.getOpcode() == ISD::SUB)
1207  RHSC = -RHSC;
1208 
1209  if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1210  Base = N.getOperand(0);
1211  if (Base.getOpcode() == ISD::FrameIndex) {
1212  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1213  Base = CurDAG->getTargetFrameIndex(
1214  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1215  }
1216  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1217  return true;
1218  }
1219  }
1220 
1221  // Base only.
1222  Base = N;
1223  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1224  return true;
1225 }
1226 
1227 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1228  SDValue &Base, SDValue &OffImm) {
1229  // Match simple R - imm8 operands.
1230  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1231  !CurDAG->isBaseWithConstantOffset(N))
1232  return false;
1233 
1234  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1235  int RHSC = (int)RHS->getSExtValue();
1236  if (N.getOpcode() == ISD::SUB)
1237  RHSC = -RHSC;
1238 
1239  if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1240  Base = N.getOperand(0);
1241  if (Base.getOpcode() == ISD::FrameIndex) {
1242  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1243  Base = CurDAG->getTargetFrameIndex(
1244  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1245  }
1246  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1247  return true;
1248  }
1249  }
1250 
1251  return false;
1252 }
1253 
1254 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1255  SDValue &OffImm){
1256  unsigned Opcode = Op->getOpcode();
1257  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1258  ? cast<LoadSDNode>(Op)->getAddressingMode()
1259  : cast<StoreSDNode>(Op)->getAddressingMode();
1260  int RHSC;
1261  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1262  OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1263  ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
1264  : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1265  return true;
1266  }
1267 
1268  return false;
1269 }
1270 
1271 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1272  SDValue &Base,
1273  SDValue &OffReg, SDValue &ShImm) {
1274  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1275  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1276  return false;
1277 
1278  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1279  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1280  int RHSC = (int)RHS->getZExtValue();
1281  if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1282  return false;
1283  else if (RHSC < 0 && RHSC >= -255) // 8 bits
1284  return false;
1285  }
1286 
1287  // Look for (R + R) or (R + (R << [1,2,3])).
1288  unsigned ShAmt = 0;
1289  Base = N.getOperand(0);
1290  OffReg = N.getOperand(1);
1291 
1292  // Swap if it is ((R << c) + R).
1294  if (ShOpcVal != ARM_AM::lsl) {
1295  ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1296  if (ShOpcVal == ARM_AM::lsl)
1297  std::swap(Base, OffReg);
1298  }
1299 
1300  if (ShOpcVal == ARM_AM::lsl) {
1301  // Check to see if the RHS of the shift is a constant, if not, we can't fold
1302  // it.
1303  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1304  ShAmt = Sh->getZExtValue();
1305  if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1306  OffReg = OffReg.getOperand(0);
1307  else {
1308  ShAmt = 0;
1309  }
1310  }
1311  }
1312 
1313  // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1314  // and use it in a shifted operand do so.
1315  if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
1316  unsigned PowerOfTwo = 0;
1317  SDValue NewMulConst;
1318  if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1319  HandleSDNode Handle(OffReg);
1320  replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1321  OffReg = Handle.getValue();
1322  ShAmt = PowerOfTwo;
1323  }
1324  }
1325 
1326  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1327 
1328  return true;
1329 }
1330 
1331 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1332  SDValue &OffImm) {
1333  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1334  // instructions.
1335  Base = N;
1336  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1337 
1338  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1339  return true;
1340 
1342  if (!RHS)
1343  return true;
1344 
1345  uint32_t RHSC = (int)RHS->getZExtValue();
1346  if (RHSC > 1020 || RHSC % 4 != 0)
1347  return true;
1348 
1349  Base = N.getOperand(0);
1350  if (Base.getOpcode() == ISD::FrameIndex) {
1351  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1352  Base = CurDAG->getTargetFrameIndex(
1353  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1354  }
1355 
1356  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1357  return true;
1358 }
1359 
1360 //===--------------------------------------------------------------------===//
1361 
1362 /// getAL - Returns a ARMCC::AL immediate node.
1363 static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1364  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1365 }
1366 
1367 void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1368  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1369  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
1370 }
1371 
1372 bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1373  LoadSDNode *LD = cast<LoadSDNode>(N);
1375  if (AM == ISD::UNINDEXED)
1376  return false;
1377 
1378  EVT LoadedVT = LD->getMemoryVT();
1379  SDValue Offset, AMOpc;
1380  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1381  unsigned Opcode = 0;
1382  bool Match = false;
1383  if (LoadedVT == MVT::i32 && isPre &&
1384  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1385  Opcode = ARM::LDR_PRE_IMM;
1386  Match = true;
1387  } else if (LoadedVT == MVT::i32 && !isPre &&
1388  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1389  Opcode = ARM::LDR_POST_IMM;
1390  Match = true;
1391  } else if (LoadedVT == MVT::i32 &&
1392  SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1393  Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1394  Match = true;
1395 
1396  } else if (LoadedVT == MVT::i16 &&
1397  SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1398  Match = true;
1399  Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1400  ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1401  : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1402  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1403  if (LD->getExtensionType() == ISD::SEXTLOAD) {
1404  if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1405  Match = true;
1406  Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1407  }
1408  } else {
1409  if (isPre &&
1410  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1411  Match = true;
1412  Opcode = ARM::LDRB_PRE_IMM;
1413  } else if (!isPre &&
1414  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1415  Match = true;
1416  Opcode = ARM::LDRB_POST_IMM;
1417  } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1418  Match = true;
1419  Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1420  }
1421  }
1422  }
1423 
1424  if (Match) {
1425  if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1426  SDValue Chain = LD->getChain();
1427  SDValue Base = LD->getBasePtr();
1428  SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1429  CurDAG->getRegister(0, MVT::i32), Chain };
1430  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1431  MVT::Other, Ops);
1432  transferMemOperands(N, New);
1433  ReplaceNode(N, New);
1434  return true;
1435  } else {
1436  SDValue Chain = LD->getChain();
1437  SDValue Base = LD->getBasePtr();
1438  SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1439  CurDAG->getRegister(0, MVT::i32), Chain };
1440  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1441  MVT::Other, Ops);
1442  transferMemOperands(N, New);
1443  ReplaceNode(N, New);
1444  return true;
1445  }
1446  }
1447 
1448  return false;
1449 }
1450 
1451 bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1452  LoadSDNode *LD = cast<LoadSDNode>(N);
1453  EVT LoadedVT = LD->getMemoryVT();
1455  if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
1456  LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
1457  return false;
1458 
1459  auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1460  if (!COffs || COffs->getZExtValue() != 4)
1461  return false;
1462 
1463  // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1464  // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1465  // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1466  // ISel.
1467  SDValue Chain = LD->getChain();
1468  SDValue Base = LD->getBasePtr();
1469  SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1470  CurDAG->getRegister(0, MVT::i32), Chain };
1471  SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1472  MVT::i32, MVT::Other, Ops);
1473  transferMemOperands(N, New);
1474  ReplaceNode(N, New);
1475  return true;
1476 }
1477 
1478 bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1479  LoadSDNode *LD = cast<LoadSDNode>(N);
1481  if (AM == ISD::UNINDEXED)
1482  return false;
1483 
1484  EVT LoadedVT = LD->getMemoryVT();
1485  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1486  SDValue Offset;
1487  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1488  unsigned Opcode = 0;
1489  bool Match = false;
1490  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1491  switch (LoadedVT.getSimpleVT().SimpleTy) {
1492  case MVT::i32:
1493  Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1494  break;
1495  case MVT::i16:
1496  if (isSExtLd)
1497  Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1498  else
1499  Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1500  break;
1501  case MVT::i8:
1502  case MVT::i1:
1503  if (isSExtLd)
1504  Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1505  else
1506  Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1507  break;
1508  default:
1509  return false;
1510  }
1511  Match = true;
1512  }
1513 
1514  if (Match) {
1515  SDValue Chain = LD->getChain();
1516  SDValue Base = LD->getBasePtr();
1517  SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1518  CurDAG->getRegister(0, MVT::i32), Chain };
1519  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1520  MVT::Other, Ops);
1521  transferMemOperands(N, New);
1522  ReplaceNode(N, New);
1523  return true;
1524  }
1525 
1526  return false;
1527 }
1528 
1529 /// Form a GPRPair pseudo register from a pair of GPR regs.
1531  SDLoc dl(V0.getNode());
1532  SDValue RegClass =
1533  CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1534  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1535  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1536  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1537  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1538 }
1539 
1540 /// Form a D register from a pair of S registers.
1541 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1542  SDLoc dl(V0.getNode());
1543  SDValue RegClass =
1544  CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1545  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1546  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1547  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1548  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1549 }
1550 
1551 /// Form a quad register from a pair of D registers.
1552 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1553  SDLoc dl(V0.getNode());
1554  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1555  MVT::i32);
1556  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1557  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1558  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1559  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1560 }
1561 
1562 /// Form 4 consecutive D registers from a pair of Q registers.
1563 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1564  SDLoc dl(V0.getNode());
1565  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1566  MVT::i32);
1567  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1568  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1569  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1570  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1571 }
1572 
1573 /// Form 4 consecutive S registers.
1574 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1575  SDValue V2, SDValue V3) {
1576  SDLoc dl(V0.getNode());
1577  SDValue RegClass =
1578  CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1579  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1580  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1581  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1582  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1583  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1584  V2, SubReg2, V3, SubReg3 };
1585  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1586 }
1587 
1588 /// Form 4 consecutive D registers.
1589 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1590  SDValue V2, SDValue V3) {
1591  SDLoc dl(V0.getNode());
1592  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1593  MVT::i32);
1594  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1595  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1596  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1597  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1598  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1599  V2, SubReg2, V3, SubReg3 };
1600  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1601 }
1602 
1603 /// Form 4 consecutive Q registers.
1604 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1605  SDValue V2, SDValue V3) {
1606  SDLoc dl(V0.getNode());
1607  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1608  MVT::i32);
1609  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1610  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1611  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1612  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1613  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1614  V2, SubReg2, V3, SubReg3 };
1615  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1616 }
1617 
1618 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1619 /// of a NEON VLD or VST instruction. The supported values depend on the
1620 /// number of registers being loaded.
1621 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1622  unsigned NumVecs, bool is64BitVector) {
1623  unsigned NumRegs = NumVecs;
1624  if (!is64BitVector && NumVecs < 3)
1625  NumRegs *= 2;
1626 
1627  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1628  if (Alignment >= 32 && NumRegs == 4)
1629  Alignment = 32;
1630  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1631  Alignment = 16;
1632  else if (Alignment >= 8)
1633  Alignment = 8;
1634  else
1635  Alignment = 0;
1636 
1637  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1638 }
1639 
1640 static bool isVLDfixed(unsigned Opc)
1641 {
1642  switch (Opc) {
1643  default: return false;
1644  case ARM::VLD1d8wb_fixed : return true;
1645  case ARM::VLD1d16wb_fixed : return true;
1646  case ARM::VLD1d64Qwb_fixed : return true;
1647  case ARM::VLD1d32wb_fixed : return true;
1648  case ARM::VLD1d64wb_fixed : return true;
1649  case ARM::VLD1d64TPseudoWB_fixed : return true;
1650  case ARM::VLD1d64QPseudoWB_fixed : return true;
1651  case ARM::VLD1q8wb_fixed : return true;
1652  case ARM::VLD1q16wb_fixed : return true;
1653  case ARM::VLD1q32wb_fixed : return true;
1654  case ARM::VLD1q64wb_fixed : return true;
1655  case ARM::VLD1DUPd8wb_fixed : return true;
1656  case ARM::VLD1DUPd16wb_fixed : return true;
1657  case ARM::VLD1DUPd32wb_fixed : return true;
1658  case ARM::VLD1DUPq8wb_fixed : return true;
1659  case ARM::VLD1DUPq16wb_fixed : return true;
1660  case ARM::VLD1DUPq32wb_fixed : return true;
1661  case ARM::VLD2d8wb_fixed : return true;
1662  case ARM::VLD2d16wb_fixed : return true;
1663  case ARM::VLD2d32wb_fixed : return true;
1664  case ARM::VLD2q8PseudoWB_fixed : return true;
1665  case ARM::VLD2q16PseudoWB_fixed : return true;
1666  case ARM::VLD2q32PseudoWB_fixed : return true;
1667  case ARM::VLD2DUPd8wb_fixed : return true;
1668  case ARM::VLD2DUPd16wb_fixed : return true;
1669  case ARM::VLD2DUPd32wb_fixed : return true;
1670  }
1671 }
1672 
1673 static bool isVSTfixed(unsigned Opc)
1674 {
1675  switch (Opc) {
1676  default: return false;
1677  case ARM::VST1d8wb_fixed : return true;
1678  case ARM::VST1d16wb_fixed : return true;
1679  case ARM::VST1d32wb_fixed : return true;
1680  case ARM::VST1d64wb_fixed : return true;
1681  case ARM::VST1q8wb_fixed : return true;
1682  case ARM::VST1q16wb_fixed : return true;
1683  case ARM::VST1q32wb_fixed : return true;
1684  case ARM::VST1q64wb_fixed : return true;
1685  case ARM::VST1d64TPseudoWB_fixed : return true;
1686  case ARM::VST1d64QPseudoWB_fixed : return true;
1687  case ARM::VST2d8wb_fixed : return true;
1688  case ARM::VST2d16wb_fixed : return true;
1689  case ARM::VST2d32wb_fixed : return true;
1690  case ARM::VST2q8PseudoWB_fixed : return true;
1691  case ARM::VST2q16PseudoWB_fixed : return true;
1692  case ARM::VST2q32PseudoWB_fixed : return true;
1693  }
1694 }
1695 
1696 // Get the register stride update opcode of a VLD/VST instruction that
1697 // is otherwise equivalent to the given fixed stride updating instruction.
1698 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1699  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1700  && "Incorrect fixed stride updating instruction.");
1701  switch (Opc) {
1702  default: break;
1703  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1704  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1705  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1706  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1707  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1708  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1709  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1710  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1711  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1712  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1713  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1714  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1715  case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
1716  case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
1717  case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
1718  case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
1719  case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
1720  case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
1721 
1722  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1723  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1724  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1725  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1726  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1727  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1728  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1729  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1730  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1731  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1732 
1733  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1734  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1735  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1736  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1737  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1738  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1739 
1740  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1741  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1742  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1743  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1744  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1745  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1746 
1747  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1748  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1749  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1750  }
1751  return Opc; // If not one we handle, return it unchanged.
1752 }
1753 
1754 /// Returns true if the given increment is a Constant known to be equal to the
1755 /// access size performed by a NEON load/store. This means the "[rN]!" form can
1756 /// be used.
1757 static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1758  auto C = dyn_cast<ConstantSDNode>(Inc);
1759  return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
1760 }
1761 
1762 void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1763  const uint16_t *DOpcodes,
1764  const uint16_t *QOpcodes0,
1765  const uint16_t *QOpcodes1) {
1766  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1767  SDLoc dl(N);
1768 
1769  SDValue MemAddr, Align;
1770  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1771  // nodes are not intrinsics.
1772  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1773  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1774  return;
1775 
1776  SDValue Chain = N->getOperand(0);
1777  EVT VT = N->getValueType(0);
1778  bool is64BitVector = VT.is64BitVector();
1779  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1780 
1781  unsigned OpcodeIndex;
1782  switch (VT.getSimpleVT().SimpleTy) {
1783  default: llvm_unreachable("unhandled vld type");
1784  // Double-register operations:
1785  case MVT::v8i8: OpcodeIndex = 0; break;
1786  case MVT::v4f16:
1787  case MVT::v4i16: OpcodeIndex = 1; break;
1788  case MVT::v2f32:
1789  case MVT::v2i32: OpcodeIndex = 2; break;
1790  case MVT::v1i64: OpcodeIndex = 3; break;
1791  // Quad-register operations:
1792  case MVT::v16i8: OpcodeIndex = 0; break;
1793  case MVT::v8f16:
1794  case MVT::v8i16: OpcodeIndex = 1; break;
1795  case MVT::v4f32:
1796  case MVT::v4i32: OpcodeIndex = 2; break;
1797  case MVT::v2f64:
1798  case MVT::v2i64: OpcodeIndex = 3; break;
1799  }
1800 
1801  EVT ResTy;
1802  if (NumVecs == 1)
1803  ResTy = VT;
1804  else {
1805  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1806  if (!is64BitVector)
1807  ResTyElts *= 2;
1808  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1809  }
1810  std::vector<EVT> ResTys;
1811  ResTys.push_back(ResTy);
1812  if (isUpdating)
1813  ResTys.push_back(MVT::i32);
1814  ResTys.push_back(MVT::Other);
1815 
1816  SDValue Pred = getAL(CurDAG, dl);
1817  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1818  SDNode *VLd;
1820 
1821  // Double registers and VLD1/VLD2 quad registers are directly supported.
1822  if (is64BitVector || NumVecs <= 2) {
1823  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1824  QOpcodes0[OpcodeIndex]);
1825  Ops.push_back(MemAddr);
1826  Ops.push_back(Align);
1827  if (isUpdating) {
1828  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1829  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1830  if (!IsImmUpdate) {
1831  // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1832  // check for the opcode rather than the number of vector elements.
1833  if (isVLDfixed(Opc))
1834  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1835  Ops.push_back(Inc);
1836  // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
1837  // the operands if not such an opcode.
1838  } else if (!isVLDfixed(Opc))
1839  Ops.push_back(Reg0);
1840  }
1841  Ops.push_back(Pred);
1842  Ops.push_back(Reg0);
1843  Ops.push_back(Chain);
1844  VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1845 
1846  } else {
1847  // Otherwise, quad registers are loaded with two separate instructions,
1848  // where one loads the even registers and the other loads the odd registers.
1849  EVT AddrTy = MemAddr.getValueType();
1850 
1851  // Load the even subregs. This is always an updating load, so that it
1852  // provides the address to the second load for the odd subregs.
1853  SDValue ImplDef =
1854  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1855  const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1856  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1857  ResTy, AddrTy, MVT::Other, OpsA);
1858  Chain = SDValue(VLdA, 2);
1859 
1860  // Load the odd subregs.
1861  Ops.push_back(SDValue(VLdA, 1));
1862  Ops.push_back(Align);
1863  if (isUpdating) {
1864  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1865  assert(isa<ConstantSDNode>(Inc.getNode()) &&
1866  "only constant post-increment update allowed for VLD3/4");
1867  (void)Inc;
1868  Ops.push_back(Reg0);
1869  }
1870  Ops.push_back(SDValue(VLdA, 0));
1871  Ops.push_back(Pred);
1872  Ops.push_back(Reg0);
1873  Ops.push_back(Chain);
1874  VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1875  }
1876 
1877  // Transfer memoperands.
1878  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1879  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp});
1880 
1881  if (NumVecs == 1) {
1882  ReplaceNode(N, VLd);
1883  return;
1884  }
1885 
1886  // Extract out the subregisters.
1887  SDValue SuperReg = SDValue(VLd, 0);
1888  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
1889  ARM::qsub_3 == ARM::qsub_0 + 3,
1890  "Unexpected subreg numbering");
1891  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1892  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1893  ReplaceUses(SDValue(N, Vec),
1894  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1895  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1896  if (isUpdating)
1897  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1898  CurDAG->RemoveDeadNode(N);
1899 }
1900 
1901 void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1902  const uint16_t *DOpcodes,
1903  const uint16_t *QOpcodes0,
1904  const uint16_t *QOpcodes1) {
1905  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1906  SDLoc dl(N);
1907 
1908  SDValue MemAddr, Align;
1909  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1910  // nodes are not intrinsics.
1911  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1912  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1913  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1914  return;
1915 
1916  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1917 
1918  SDValue Chain = N->getOperand(0);
1919  EVT VT = N->getOperand(Vec0Idx).getValueType();
1920  bool is64BitVector = VT.is64BitVector();
1921  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1922 
1923  unsigned OpcodeIndex;
1924  switch (VT.getSimpleVT().SimpleTy) {
1925  default: llvm_unreachable("unhandled vst type");
1926  // Double-register operations:
1927  case MVT::v8i8: OpcodeIndex = 0; break;
1928  case MVT::v4f16:
1929  case MVT::v4i16: OpcodeIndex = 1; break;
1930  case MVT::v2f32:
1931  case MVT::v2i32: OpcodeIndex = 2; break;
1932  case MVT::v1i64: OpcodeIndex = 3; break;
1933  // Quad-register operations:
1934  case MVT::v16i8: OpcodeIndex = 0; break;
1935  case MVT::v8f16:
1936  case MVT::v8i16: OpcodeIndex = 1; break;
1937  case MVT::v4f32:
1938  case MVT::v4i32: OpcodeIndex = 2; break;
1939  case MVT::v2f64:
1940  case MVT::v2i64: OpcodeIndex = 3; break;
1941  }
1942 
1943  std::vector<EVT> ResTys;
1944  if (isUpdating)
1945  ResTys.push_back(MVT::i32);
1946  ResTys.push_back(MVT::Other);
1947 
1948  SDValue Pred = getAL(CurDAG, dl);
1949  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1951 
1952  // Double registers and VST1/VST2 quad registers are directly supported.
1953  if (is64BitVector || NumVecs <= 2) {
1954  SDValue SrcReg;
1955  if (NumVecs == 1) {
1956  SrcReg = N->getOperand(Vec0Idx);
1957  } else if (is64BitVector) {
1958  // Form a REG_SEQUENCE to force register allocation.
1959  SDValue V0 = N->getOperand(Vec0Idx + 0);
1960  SDValue V1 = N->getOperand(Vec0Idx + 1);
1961  if (NumVecs == 2)
1962  SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1963  else {
1964  SDValue V2 = N->getOperand(Vec0Idx + 2);
1965  // If it's a vst3, form a quad D-register and leave the last part as
1966  // an undef.
1967  SDValue V3 = (NumVecs == 3)
1968  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1969  : N->getOperand(Vec0Idx + 3);
1970  SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1971  }
1972  } else {
1973  // Form a QQ register.
1974  SDValue Q0 = N->getOperand(Vec0Idx);
1975  SDValue Q1 = N->getOperand(Vec0Idx + 1);
1976  SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1977  }
1978 
1979  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1980  QOpcodes0[OpcodeIndex]);
1981  Ops.push_back(MemAddr);
1982  Ops.push_back(Align);
1983  if (isUpdating) {
1984  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1985  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1986  if (!IsImmUpdate) {
1987  // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
1988  // check for the opcode rather than the number of vector elements.
1989  if (isVSTfixed(Opc))
1990  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1991  Ops.push_back(Inc);
1992  }
1993  // VST1/VST2 fixed increment does not need Reg0 so only include it in
1994  // the operands if not such an opcode.
1995  else if (!isVSTfixed(Opc))
1996  Ops.push_back(Reg0);
1997  }
1998  Ops.push_back(SrcReg);
1999  Ops.push_back(Pred);
2000  Ops.push_back(Reg0);
2001  Ops.push_back(Chain);
2002  SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2003 
2004  // Transfer memoperands.
2005  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp});
2006 
2007  ReplaceNode(N, VSt);
2008  return;
2009  }
2010 
2011  // Otherwise, quad registers are stored with two separate instructions,
2012  // where one stores the even registers and the other stores the odd registers.
2013 
2014  // Form the QQQQ REG_SEQUENCE.
2015  SDValue V0 = N->getOperand(Vec0Idx + 0);
2016  SDValue V1 = N->getOperand(Vec0Idx + 1);
2017  SDValue V2 = N->getOperand(Vec0Idx + 2);
2018  SDValue V3 = (NumVecs == 3)
2019  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2020  : N->getOperand(Vec0Idx + 3);
2021  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2022 
2023  // Store the even D registers. This is always an updating store, so that it
2024  // provides the address to the second store for the odd subregs.
2025  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2026  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2027  MemAddr.getValueType(),
2028  MVT::Other, OpsA);
2029  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp});
2030  Chain = SDValue(VStA, 1);
2031 
2032  // Store the odd D registers.
2033  Ops.push_back(SDValue(VStA, 0));
2034  Ops.push_back(Align);
2035  if (isUpdating) {
2036  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2037  assert(isa<ConstantSDNode>(Inc.getNode()) &&
2038  "only constant post-increment update allowed for VST3/4");
2039  (void)Inc;
2040  Ops.push_back(Reg0);
2041  }
2042  Ops.push_back(RegSeq);
2043  Ops.push_back(Pred);
2044  Ops.push_back(Reg0);
2045  Ops.push_back(Chain);
2046  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2047  Ops);
2048  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp});
2049  ReplaceNode(N, VStB);
2050 }
2051 
2052 void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2053  unsigned NumVecs,
2054  const uint16_t *DOpcodes,
2055  const uint16_t *QOpcodes) {
2056  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2057  SDLoc dl(N);
2058 
2059  SDValue MemAddr, Align;
2060  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2061  // nodes are not intrinsics.
2062  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2063  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2064  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2065  return;
2066 
2067  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2068 
2069  SDValue Chain = N->getOperand(0);
2070  unsigned Lane =
2071  cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2072  EVT VT = N->getOperand(Vec0Idx).getValueType();
2073  bool is64BitVector = VT.is64BitVector();
2074 
2075  unsigned Alignment = 0;
2076  if (NumVecs != 3) {
2077  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2078  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2079  if (Alignment > NumBytes)
2080  Alignment = NumBytes;
2081  if (Alignment < 8 && Alignment < NumBytes)
2082  Alignment = 0;
2083  // Alignment must be a power of two; make sure of that.
2084  Alignment = (Alignment & -Alignment);
2085  if (Alignment == 1)
2086  Alignment = 0;
2087  }
2088  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2089 
2090  unsigned OpcodeIndex;
2091  switch (VT.getSimpleVT().SimpleTy) {
2092  default: llvm_unreachable("unhandled vld/vst lane type");
2093  // Double-register operations:
2094  case MVT::v8i8: OpcodeIndex = 0; break;
2095  case MVT::v4i16: OpcodeIndex = 1; break;
2096  case MVT::v2f32:
2097  case MVT::v2i32: OpcodeIndex = 2; break;
2098  // Quad-register operations:
2099  case MVT::v8i16: OpcodeIndex = 0; break;
2100  case MVT::v4f32:
2101  case MVT::v4i32: OpcodeIndex = 1; break;
2102  }
2103 
2104  std::vector<EVT> ResTys;
2105  if (IsLoad) {
2106  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2107  if (!is64BitVector)
2108  ResTyElts *= 2;
2109  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2110  MVT::i64, ResTyElts));
2111  }
2112  if (isUpdating)
2113  ResTys.push_back(MVT::i32);
2114  ResTys.push_back(MVT::Other);
2115 
2116  SDValue Pred = getAL(CurDAG, dl);
2117  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2118 
2120  Ops.push_back(MemAddr);
2121  Ops.push_back(Align);
2122  if (isUpdating) {
2123  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2124  bool IsImmUpdate =
2125  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2126  Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2127  }
2128 
2129  SDValue SuperReg;
2130  SDValue V0 = N->getOperand(Vec0Idx + 0);
2131  SDValue V1 = N->getOperand(Vec0Idx + 1);
2132  if (NumVecs == 2) {
2133  if (is64BitVector)
2134  SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2135  else
2136  SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2137  } else {
2138  SDValue V2 = N->getOperand(Vec0Idx + 2);
2139  SDValue V3 = (NumVecs == 3)
2140  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2141  : N->getOperand(Vec0Idx + 3);
2142  if (is64BitVector)
2143  SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2144  else
2145  SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2146  }
2147  Ops.push_back(SuperReg);
2148  Ops.push_back(getI32Imm(Lane, dl));
2149  Ops.push_back(Pred);
2150  Ops.push_back(Reg0);
2151  Ops.push_back(Chain);
2152 
2153  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2154  QOpcodes[OpcodeIndex]);
2155  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2156  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp});
2157  if (!IsLoad) {
2158  ReplaceNode(N, VLdLn);
2159  return;
2160  }
2161 
2162  // Extract the subregisters.
2163  SuperReg = SDValue(VLdLn, 0);
2164  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2165  ARM::qsub_3 == ARM::qsub_0 + 3,
2166  "Unexpected subreg numbering");
2167  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2168  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2169  ReplaceUses(SDValue(N, Vec),
2170  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2171  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2172  if (isUpdating)
2173  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2174  CurDAG->RemoveDeadNode(N);
2175 }
2176 
2177 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2178  bool isUpdating, unsigned NumVecs,
2179  const uint16_t *DOpcodes,
2180  const uint16_t *QOpcodes0,
2181  const uint16_t *QOpcodes1) {
2182  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2183  SDLoc dl(N);
2184 
2185  SDValue MemAddr, Align;
2186  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2187  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2188  return;
2189 
2190  SDValue Chain = N->getOperand(0);
2191  EVT VT = N->getValueType(0);
2192  bool is64BitVector = VT.is64BitVector();
2193 
2194  unsigned Alignment = 0;
2195  if (NumVecs != 3) {
2196  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2197  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2198  if (Alignment > NumBytes)
2199  Alignment = NumBytes;
2200  if (Alignment < 8 && Alignment < NumBytes)
2201  Alignment = 0;
2202  // Alignment must be a power of two; make sure of that.
2203  Alignment = (Alignment & -Alignment);
2204  if (Alignment == 1)
2205  Alignment = 0;
2206  }
2207  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2208 
2209  unsigned OpcodeIndex;
2210  switch (VT.getSimpleVT().SimpleTy) {
2211  default: llvm_unreachable("unhandled vld-dup type");
2212  case MVT::v8i8:
2213  case MVT::v16i8: OpcodeIndex = 0; break;
2214  case MVT::v4i16:
2215  case MVT::v8i16:
2216  case MVT::v4f16:
2217  case MVT::v8f16:
2218  OpcodeIndex = 1; break;
2219  case MVT::v2f32:
2220  case MVT::v2i32:
2221  case MVT::v4f32:
2222  case MVT::v4i32: OpcodeIndex = 2; break;
2223  case MVT::v1f64:
2224  case MVT::v1i64: OpcodeIndex = 3; break;
2225  }
2226 
2227  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2228  if (!is64BitVector)
2229  ResTyElts *= 2;
2230  EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2231 
2232  std::vector<EVT> ResTys;
2233  ResTys.push_back(ResTy);
2234  if (isUpdating)
2235  ResTys.push_back(MVT::i32);
2236  ResTys.push_back(MVT::Other);
2237 
2238  SDValue Pred = getAL(CurDAG, dl);
2239  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2240 
2241  SDNode *VLdDup;
2242  if (is64BitVector || NumVecs == 1) {
2244  Ops.push_back(MemAddr);
2245  Ops.push_back(Align);
2246  unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex] :
2247  QOpcodes0[OpcodeIndex];
2248  if (isUpdating) {
2249  // fixed-stride update instructions don't have an explicit writeback
2250  // operand. It's implicit in the opcode itself.
2251  SDValue Inc = N->getOperand(2);
2252  bool IsImmUpdate =
2253  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2254  if (NumVecs <= 2 && !IsImmUpdate)
2255  Opc = getVLDSTRegisterUpdateOpcode(Opc);
2256  if (!IsImmUpdate)
2257  Ops.push_back(Inc);
2258  // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2259  else if (NumVecs > 2)
2260  Ops.push_back(Reg0);
2261  }
2262  Ops.push_back(Pred);
2263  Ops.push_back(Reg0);
2264  Ops.push_back(Chain);
2265  VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2266  } else if (NumVecs == 2) {
2267  const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
2268  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2269  dl, ResTys, OpsA);
2270 
2271  Chain = SDValue(VLdA, 1);
2272  const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
2273  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2274  } else {
2275  SDValue ImplDef =
2276  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2277  const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
2278  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2279  dl, ResTys, OpsA);
2280 
2281  SDValue SuperReg = SDValue(VLdA, 0);
2282  Chain = SDValue(VLdA, 1);
2283  const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
2284  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2285  }
2286 
2287  // Transfer memoperands.
2288  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2289  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
2290 
2291  // Extract the subregisters.
2292  if (NumVecs == 1) {
2293  ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
2294  } else {
2295  SDValue SuperReg = SDValue(VLdDup, 0);
2296  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
2297  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2298  for (unsigned Vec = 0; Vec != NumVecs; ++Vec) {
2299  ReplaceUses(SDValue(N, Vec),
2300  CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2301  }
2302  }
2303  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2304  if (isUpdating)
2305  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2306  CurDAG->RemoveDeadNode(N);
2307 }
2308 
2309 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
2310  if (!Subtarget->hasV6T2Ops())
2311  return false;
2312 
2313  unsigned Opc = isSigned
2314  ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2315  : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2316  SDLoc dl(N);
2317 
2318  // For unsigned extracts, check for a shift right and mask
2319  unsigned And_imm = 0;
2320  if (N->getOpcode() == ISD::AND) {
2321  if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2322 
2323  // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2324  if (And_imm & (And_imm + 1))
2325  return false;
2326 
2327  unsigned Srl_imm = 0;
2329  Srl_imm)) {
2330  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2331 
2332  // Mask off the unnecessary bits of the AND immediate; normally
2333  // DAGCombine will do this, but that might not happen if
2334  // targetShrinkDemandedConstant chooses a different immediate.
2335  And_imm &= -1U >> Srl_imm;
2336 
2337  // Note: The width operand is encoded as width-1.
2338  unsigned Width = countTrailingOnes(And_imm) - 1;
2339  unsigned LSB = Srl_imm;
2340 
2341  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2342 
2343  if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2344  // It's cheaper to use a right shift to extract the top bits.
2345  if (Subtarget->isThumb()) {
2346  Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2347  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2348  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2349  getAL(CurDAG, dl), Reg0, Reg0 };
2350  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2351  return true;
2352  }
2353 
2354  // ARM models shift instructions as MOVsi with shifter operand.
2356  SDValue ShOpc =
2357  CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
2358  MVT::i32);
2359  SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2360  getAL(CurDAG, dl), Reg0, Reg0 };
2361  CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2362  return true;
2363  }
2364 
2365  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2366  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2367  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2368  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2369  getAL(CurDAG, dl), Reg0 };
2370  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2371  return true;
2372  }
2373  }
2374  return false;
2375  }
2376 
2377  // Otherwise, we're looking for a shift of a shift
2378  unsigned Shl_imm = 0;
2379  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2380  assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2381  unsigned Srl_imm = 0;
2382  if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2383  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2384  // Note: The width operand is encoded as width-1.
2385  unsigned Width = 32 - Srl_imm - 1;
2386  int LSB = Srl_imm - Shl_imm;
2387  if (LSB < 0)
2388  return false;
2389  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2390  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2391  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2392  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2393  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2394  getAL(CurDAG, dl), Reg0 };
2395  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2396  return true;
2397  }
2398  }
2399 
2400  // Or we are looking for a shift of an and, with a mask operand
2401  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
2402  isShiftedMask_32(And_imm)) {
2403  unsigned Srl_imm = 0;
2404  unsigned LSB = countTrailingZeros(And_imm);
2405  // Shift must be the same as the ands lsb
2406  if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
2407  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2408  unsigned MSB = 31 - countLeadingZeros(And_imm);
2409  // Note: The width operand is encoded as width-1.
2410  unsigned Width = MSB - LSB;
2411  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2412  assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2413  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2414  CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
2415  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2416  getAL(CurDAG, dl), Reg0 };
2417  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2418  return true;
2419  }
2420  }
2421 
2422  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2423  unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2424  unsigned LSB = 0;
2425  if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2427  return false;
2428 
2429  if (LSB + Width > 32)
2430  return false;
2431 
2432  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2433  assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx");
2434  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2435  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2436  CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
2437  getAL(CurDAG, dl), Reg0 };
2438  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2439  return true;
2440  }
2441 
2442  return false;
2443 }
2444 
2445 /// Target-specific DAG combining for ISD::XOR.
2446 /// Target-independent combining lowers SELECT_CC nodes of the form
2447 /// select_cc setg[ge] X, 0, X, -X
2448 /// select_cc setgt X, -1, X, -X
2449 /// select_cc setl[te] X, 0, -X, X
2450 /// select_cc setlt X, 1, -X, X
2451 /// which represent Integer ABS into:
2452 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2453 /// ARM instruction selection detects the latter and matches it to
2454 /// ARM::ABS or ARM::t2ABS machine node.
2455 bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
2456  SDValue XORSrc0 = N->getOperand(0);
2457  SDValue XORSrc1 = N->getOperand(1);
2458  EVT VT = N->getValueType(0);
2459 
2460  if (Subtarget->isThumb1Only())
2461  return false;
2462 
2463  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2464  return false;
2465 
2466  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2467  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2468  SDValue SRASrc0 = XORSrc1.getOperand(0);
2469  SDValue SRASrc1 = XORSrc1.getOperand(1);
2470  ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
2471  EVT XType = SRASrc0.getValueType();
2472  unsigned Size = XType.getSizeInBits() - 1;
2473 
2474  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2475  XType.isInteger() && SRAConstant != nullptr &&
2476  Size == SRAConstant->getZExtValue()) {
2477  unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2478  CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2479  return true;
2480  }
2481 
2482  return false;
2483 }
2484 
2485 /// We've got special pseudo-instructions for these
2486 void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2487  unsigned Opcode;
2488  EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2489  if (MemTy == MVT::i8)
2490  Opcode = ARM::CMP_SWAP_8;
2491  else if (MemTy == MVT::i16)
2492  Opcode = ARM::CMP_SWAP_16;
2493  else if (MemTy == MVT::i32)
2494  Opcode = ARM::CMP_SWAP_32;
2495  else
2496  llvm_unreachable("Unknown AtomicCmpSwap type");
2497 
2498  SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2499  N->getOperand(0)};
2500  SDNode *CmpSwap = CurDAG->getMachineNode(
2501  Opcode, SDLoc(N),
2502  CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
2503 
2504  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2505  CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2506 
2507  ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2508  ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2509  CurDAG->RemoveDeadNode(N);
2510 }
2511 
2514  unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
2515  unsigned LastOne = A.countTrailingZeros();
2516  if (A.countPopulation() != (FirstOne - LastOne + 1))
2518  return std::make_pair(FirstOne, LastOne);
2519 }
2520 
2521 void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
2522  assert(N->getOpcode() == ARMISD::CMPZ);
2523  SwitchEQNEToPLMI = false;
2524 
2525  if (!Subtarget->isThumb())
2526  // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
2527  // LSR don't exist as standalone instructions - they need the barrel shifter.
2528  return;
2529 
2530  // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
2531  SDValue And = N->getOperand(0);
2532  if (!And->hasOneUse())
2533  return;
2534 
2535  SDValue Zero = N->getOperand(1);
2536  if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isNullValue() ||
2537  And->getOpcode() != ISD::AND)
2538  return;
2539  SDValue X = And.getOperand(0);
2540  auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
2541 
2542  if (!C)
2543  return;
2544  auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
2545  if (!Range)
2546  return;
2547 
2548  // There are several ways to lower this:
2549  SDNode *NewN;
2550  SDLoc dl(N);
2551 
2552  auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
2553  if (Subtarget->isThumb2()) {
2554  Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
2555  SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2556  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2557  CurDAG->getRegister(0, MVT::i32) };
2558  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2559  } else {
2560  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2561  CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2562  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2563  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2564  }
2565  };
2566 
2567  if (Range->second == 0) {
2568  // 1. Mask includes the LSB -> Simply shift the top N bits off
2569  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2570  ReplaceNode(And.getNode(), NewN);
2571  } else if (Range->first == 31) {
2572  // 2. Mask includes the MSB -> Simply shift the bottom N bits off
2573  NewN = EmitShift(ARM::tLSRri, X, Range->second);
2574  ReplaceNode(And.getNode(), NewN);
2575  } else if (Range->first == Range->second) {
2576  // 3. Only one bit is set. We can shift this into the sign bit and use a
2577  // PL/MI comparison.
2578  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2579  ReplaceNode(And.getNode(), NewN);
2580 
2581  SwitchEQNEToPLMI = true;
2582  } else if (!Subtarget->hasV6T2Ops()) {
2583  // 4. Do a double shift to clear bottom and top bits, but only in
2584  // thumb-1 mode as in thumb-2 we can use UBFX.
2585  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2586  NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
2587  Range->second + (31 - Range->first));
2588  ReplaceNode(And.getNode(), NewN);
2589  }
2590 
2591 }
2592 
2594  SDLoc dl(N);
2595 
2596  if (N->isMachineOpcode()) {
2597  N->setNodeId(-1);
2598  return; // Already selected.
2599  }
2600 
2601  switch (N->getOpcode()) {
2602  default: break;
2603  case ISD::STORE: {
2604  // For Thumb1, match an sp-relative store in C++. This is a little
2605  // unfortunate, but I don't think I can make the chain check work
2606  // otherwise. (The chain of the store has to be the same as the chain
2607  // of the CopyFromReg, or else we can't replace the CopyFromReg with
2608  // a direct reference to "SP".)
2609  //
2610  // This is only necessary on Thumb1 because Thumb1 sp-relative stores use
2611  // a different addressing mode from other four-byte stores.
2612  //
2613  // This pattern usually comes up with call arguments.
2614  StoreSDNode *ST = cast<StoreSDNode>(N);
2615  SDValue Ptr = ST->getBasePtr();
2616  if (Subtarget->isThumb1Only() && ST->isUnindexed()) {
2617  int RHSC = 0;
2618  if (Ptr.getOpcode() == ISD::ADD &&
2619  isScaledConstantInRange(Ptr.getOperand(1), /*Scale=*/4, 0, 256, RHSC))
2620  Ptr = Ptr.getOperand(0);
2621 
2622  if (Ptr.getOpcode() == ISD::CopyFromReg &&
2623  cast<RegisterSDNode>(Ptr.getOperand(1))->getReg() == ARM::SP &&
2624  Ptr.getOperand(0) == ST->getChain()) {
2625  SDValue Ops[] = {ST->getValue(),
2626  CurDAG->getRegister(ARM::SP, MVT::i32),
2627  CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
2628  getAL(CurDAG, dl),
2629  CurDAG->getRegister(0, MVT::i32),
2630  ST->getChain()};
2631  MachineSDNode *ResNode =
2632  CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
2633  MachineMemOperand *MemOp = ST->getMemOperand();
2634  CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2635  ReplaceNode(N, ResNode);
2636  return;
2637  }
2638  }
2639  break;
2640  }
2641  case ISD::WRITE_REGISTER:
2642  if (tryWriteRegister(N))
2643  return;
2644  break;
2645  case ISD::READ_REGISTER:
2646  if (tryReadRegister(N))
2647  return;
2648  break;
2649  case ISD::INLINEASM:
2650  case ISD::INLINEASM_BR:
2651  if (tryInlineAsm(N))
2652  return;
2653  break;
2654  case ISD::XOR:
2655  // Select special operations if XOR node forms integer ABS pattern
2656  if (tryABSOp(N))
2657  return;
2658  // Other cases are autogenerated.
2659  break;
2660  case ISD::Constant: {
2661  unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2662  // If we can't materialize the constant we need to use a literal pool
2663  if (ConstantMaterializationCost(Val) > 2) {
2664  SDValue CPIdx = CurDAG->getTargetConstantPool(
2665  ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2666  TLI->getPointerTy(CurDAG->getDataLayout()));
2667 
2668  SDNode *ResNode;
2669  if (Subtarget->isThumb()) {
2670  SDValue Ops[] = {
2671  CPIdx,
2672  getAL(CurDAG, dl),
2673  CurDAG->getRegister(0, MVT::i32),
2674  CurDAG->getEntryNode()
2675  };
2676  ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2677  Ops);
2678  } else {
2679  SDValue Ops[] = {
2680  CPIdx,
2681  CurDAG->getTargetConstant(0, dl, MVT::i32),
2682  getAL(CurDAG, dl),
2683  CurDAG->getRegister(0, MVT::i32),
2684  CurDAG->getEntryNode()
2685  };
2686  ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2687  Ops);
2688  }
2689  // Annotate the Node with memory operand information so that MachineInstr
2690  // queries work properly. This e.g. gives the register allocation the
2691  // required information for rematerialization.
2692  MachineFunction& MF = CurDAG->getMachineFunction();
2693  MachineMemOperand *MemOp =
2696 
2697  CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2698 
2699  ReplaceNode(N, ResNode);
2700  return;
2701  }
2702 
2703  // Other cases are autogenerated.
2704  break;
2705  }
2706  case ISD::FrameIndex: {
2707  // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2708  int FI = cast<FrameIndexSDNode>(N)->getIndex();
2709  SDValue TFI = CurDAG->getTargetFrameIndex(
2710  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2711  if (Subtarget->isThumb1Only()) {
2712  // Set the alignment of the frame object to 4, to avoid having to generate
2713  // more than one ADD
2714  MachineFrameInfo &MFI = MF->getFrameInfo();
2715  if (MFI.getObjectAlignment(FI) < 4)
2716  MFI.setObjectAlignment(FI, 4);
2717  CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2718  CurDAG->getTargetConstant(0, dl, MVT::i32));
2719  return;
2720  } else {
2721  unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2722  ARM::t2ADDri : ARM::ADDri);
2723  SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
2724  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2725  CurDAG->getRegister(0, MVT::i32) };
2726  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2727  return;
2728  }
2729  }
2730  case ISD::SRL:
2731  if (tryV6T2BitfieldExtractOp(N, false))
2732  return;
2733  break;
2735  case ISD::SRA:
2736  if (tryV6T2BitfieldExtractOp(N, true))
2737  return;
2738  break;
2739  case ISD::MUL:
2740  if (Subtarget->isThumb1Only())
2741  break;
2742  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2743  unsigned RHSV = C->getZExtValue();
2744  if (!RHSV) break;
2745  if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
2746  unsigned ShImm = Log2_32(RHSV-1);
2747  if (ShImm >= 32)
2748  break;
2749  SDValue V = N->getOperand(0);
2750  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2751  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2752  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2753  if (Subtarget->isThumb()) {
2754  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2755  CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2756  return;
2757  } else {
2758  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2759  Reg0 };
2760  CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2761  return;
2762  }
2763  }
2764  if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
2765  unsigned ShImm = Log2_32(RHSV+1);
2766  if (ShImm >= 32)
2767  break;
2768  SDValue V = N->getOperand(0);
2769  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2770  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2771  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2772  if (Subtarget->isThumb()) {
2773  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2774  CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2775  return;
2776  } else {
2777  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2778  Reg0 };
2779  CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2780  return;
2781  }
2782  }
2783  }
2784  break;
2785  case ISD::AND: {
2786  // Check for unsigned bitfield extract
2787  if (tryV6T2BitfieldExtractOp(N, false))
2788  return;
2789 
2790  // If an immediate is used in an AND node, it is possible that the immediate
2791  // can be more optimally materialized when negated. If this is the case we
2792  // can negate the immediate and use a BIC instead.
2793  auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2794  if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
2795  uint32_t Imm = (uint32_t) N1C->getZExtValue();
2796 
2797  // In Thumb2 mode, an AND can take a 12-bit immediate. If this
2798  // immediate can be negated and fit in the immediate operand of
2799  // a t2BIC, don't do any manual transform here as this can be
2800  // handled by the generic ISel machinery.
2801  bool PreferImmediateEncoding =
2802  Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
2803  if (!PreferImmediateEncoding &&
2804  ConstantMaterializationCost(Imm) >
2805  ConstantMaterializationCost(~Imm)) {
2806  // The current immediate costs more to materialize than a negated
2807  // immediate, so negate the immediate and use a BIC.
2808  SDValue NewImm =
2809  CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
2810  // If the new constant didn't exist before, reposition it in the topological
2811  // ordering so it is just before N. Otherwise, don't touch its location.
2812  if (NewImm->getNodeId() == -1)
2813  CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
2814 
2815  if (!Subtarget->hasThumb2()) {
2816  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
2817  N->getOperand(0), NewImm, getAL(CurDAG, dl),
2818  CurDAG->getRegister(0, MVT::i32)};
2819  ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
2820  return;
2821  } else {
2822  SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
2823  CurDAG->getRegister(0, MVT::i32),
2824  CurDAG->getRegister(0, MVT::i32)};
2825  ReplaceNode(N,
2826  CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
2827  return;
2828  }
2829  }
2830  }
2831 
2832  // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2833  // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2834  // are entirely contributed by c2 and lower 16-bits are entirely contributed
2835  // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2836  // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2837  EVT VT = N->getValueType(0);
2838  if (VT != MVT::i32)
2839  break;
2840  unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2841  ? ARM::t2MOVTi16
2842  : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2843  if (!Opc)
2844  break;
2845  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2846  N1C = dyn_cast<ConstantSDNode>(N1);
2847  if (!N1C)
2848  break;
2849  if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2850  SDValue N2 = N0.getOperand(1);
2852  if (!N2C)
2853  break;
2854  unsigned N1CVal = N1C->getZExtValue();
2855  unsigned N2CVal = N2C->getZExtValue();
2856  if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2857  (N1CVal & 0xffffU) == 0xffffU &&
2858  (N2CVal & 0xffffU) == 0x0U) {
2859  SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2860  dl, MVT::i32);
2861  SDValue Ops[] = { N0.getOperand(0), Imm16,
2862  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
2863  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
2864  return;
2865  }
2866  }
2867 
2868  break;
2869  }
2870  case ARMISD::UMAAL: {
2871  unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
2872  SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2873  N->getOperand(2), N->getOperand(3),
2874  getAL(CurDAG, dl),
2875  CurDAG->getRegister(0, MVT::i32) };
2876  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
2877  return;
2878  }
2879  case ARMISD::UMLAL:{
2880  if (Subtarget->isThumb()) {
2881  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2882  N->getOperand(3), getAL(CurDAG, dl),
2883  CurDAG->getRegister(0, MVT::i32)};
2884  ReplaceNode(
2885  N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
2886  return;
2887  }else{
2888  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2889  N->getOperand(3), getAL(CurDAG, dl),
2890  CurDAG->getRegister(0, MVT::i32),
2891  CurDAG->getRegister(0, MVT::i32) };
2892  ReplaceNode(N, CurDAG->getMachineNode(
2893  Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
2894  MVT::i32, MVT::i32, Ops));
2895  return;
2896  }
2897  }
2898  case ARMISD::SMLAL:{
2899  if (Subtarget->isThumb()) {
2900  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2901  N->getOperand(3), getAL(CurDAG, dl),
2902  CurDAG->getRegister(0, MVT::i32)};
2903  ReplaceNode(
2904  N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
2905  return;
2906  }else{
2907  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2908  N->getOperand(3), getAL(CurDAG, dl),
2909  CurDAG->getRegister(0, MVT::i32),
2910  CurDAG->getRegister(0, MVT::i32) };
2911  ReplaceNode(N, CurDAG->getMachineNode(
2912  Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
2913  MVT::i32, MVT::i32, Ops));
2914  return;
2915  }
2916  }
2917  case ARMISD::SUBE: {
2918  if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
2919  break;
2920  // Look for a pattern to match SMMLS
2921  // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
2922  if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
2923  N->getOperand(2).getOpcode() != ARMISD::SUBC ||
2924  !SDValue(N, 1).use_empty())
2925  break;
2926 
2927  if (Subtarget->isThumb())
2928  assert(Subtarget->hasThumb2() &&
2929  "This pattern should not be generated for Thumb");
2930 
2931  SDValue SmulLoHi = N->getOperand(1);
2932  SDValue Subc = N->getOperand(2);
2933  auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
2934 
2935  if (!Zero || Zero->getZExtValue() != 0 ||
2936  Subc.getOperand(1) != SmulLoHi.getValue(0) ||
2937  N->getOperand(1) != SmulLoHi.getValue(1) ||
2938  N->getOperand(2) != Subc.getValue(1))
2939  break;
2940 
2941  unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
2942  SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
2943  N->getOperand(0), getAL(CurDAG, dl),
2944  CurDAG->getRegister(0, MVT::i32) };
2945  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
2946  return;
2947  }
2948  case ISD::LOAD: {
2949  if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
2950  if (tryT2IndexedLoad(N))
2951  return;
2952  } else if (Subtarget->isThumb()) {
2953  if (tryT1IndexedLoad(N))
2954  return;
2955  } else if (tryARMIndexedLoad(N))
2956  return;
2957  // Other cases are autogenerated.
2958  break;
2959  }
2960  case ARMISD::BRCOND: {
2961  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2962  // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2963  // Pattern complexity = 6 cost = 1 size = 0
2964 
2965  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2966  // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2967  // Pattern complexity = 6 cost = 1 size = 0
2968 
2969  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2970  // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2971  // Pattern complexity = 6 cost = 1 size = 0
2972 
2973  unsigned Opc = Subtarget->isThumb() ?
2974  ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2975  SDValue Chain = N->getOperand(0);
2976  SDValue N1 = N->getOperand(1);
2977  SDValue N2 = N->getOperand(2);
2978  SDValue N3 = N->getOperand(3);
2979  SDValue InFlag = N->getOperand(4);
2980  assert(N1.getOpcode() == ISD::BasicBlock);
2981  assert(N2.getOpcode() == ISD::Constant);
2982  assert(N3.getOpcode() == ISD::Register);
2983 
2984  unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
2985 
2986  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2987  bool SwitchEQNEToPLMI;
2988  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2989  InFlag = N->getOperand(4);
2990 
2991  if (SwitchEQNEToPLMI) {
2992  switch ((ARMCC::CondCodes)CC) {
2993  default: llvm_unreachable("CMPZ must be either NE or EQ!");
2994  case ARMCC::NE:
2995  CC = (unsigned)ARMCC::MI;
2996  break;
2997  case ARMCC::EQ:
2998  CC = (unsigned)ARMCC::PL;
2999  break;
3000  }
3001  }
3002  }
3003 
3004  SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
3005  SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
3006  SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
3007  MVT::Glue, Ops);
3008  Chain = SDValue(ResNode, 0);
3009  if (N->getNumValues() == 2) {
3010  InFlag = SDValue(ResNode, 1);
3011  ReplaceUses(SDValue(N, 1), InFlag);
3012  }
3013  ReplaceUses(SDValue(N, 0),
3014  SDValue(Chain.getNode(), Chain.getResNo()));
3015  CurDAG->RemoveDeadNode(N);
3016  return;
3017  }
3018 
3019  case ARMISD::CMPZ: {
3020  // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
3021  // This allows us to avoid materializing the expensive negative constant.
3022  // The CMPZ #0 is useless and will be peepholed away but we need to keep it
3023  // for its glue output.
3024  SDValue X = N->getOperand(0);
3025  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
3026  if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
3027  int64_t Addend = -C->getSExtValue();
3028 
3029  SDNode *Add = nullptr;
3030  // ADDS can be better than CMN if the immediate fits in a
3031  // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
3032  // Outside that range we can just use a CMN which is 32-bit but has a
3033  // 12-bit immediate range.
3034  if (Addend < 1<<8) {
3035  if (Subtarget->isThumb2()) {
3036  SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3037  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3038  CurDAG->getRegister(0, MVT::i32) };
3039  Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
3040  } else {
3041  unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
3042  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
3043  CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3044  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3045  Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3046  }
3047  }
3048  if (Add) {
3049  SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
3050  CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
3051  }
3052  }
3053  // Other cases are autogenerated.
3054  break;
3055  }
3056 
3057  case ARMISD::CMOV: {
3058  SDValue InFlag = N->getOperand(4);
3059 
3060  if (InFlag.getOpcode() == ARMISD::CMPZ) {
3061  bool SwitchEQNEToPLMI;
3062  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
3063 
3064  if (SwitchEQNEToPLMI) {
3065  SDValue ARMcc = N->getOperand(2);
3066  ARMCC::CondCodes CC =
3067  (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
3068 
3069  switch (CC) {
3070  default: llvm_unreachable("CMPZ must be either NE or EQ!");
3071  case ARMCC::NE:
3072  CC = ARMCC::MI;
3073  break;
3074  case ARMCC::EQ:
3075  CC = ARMCC::PL;
3076  break;
3077  }
3078  SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
3079  SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
3080  N->getOperand(3), N->getOperand(4)};
3081  CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
3082  }
3083 
3084  }
3085  // Other cases are autogenerated.
3086  break;
3087  }
3088 
3089  case ARMISD::VZIP: {
3090  unsigned Opc = 0;
3091  EVT VT = N->getValueType(0);
3092  switch (VT.getSimpleVT().SimpleTy) {
3093  default: return;
3094  case MVT::v8i8: Opc = ARM::VZIPd8; break;
3095  case MVT::v4f16:
3096  case MVT::v4i16: Opc = ARM::VZIPd16; break;
3097  case MVT::v2f32:
3098  // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3099  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3100  case MVT::v16i8: Opc = ARM::VZIPq8; break;
3101  case MVT::v8f16:
3102  case MVT::v8i16: Opc = ARM::VZIPq16; break;
3103  case MVT::v4f32:
3104  case MVT::v4i32: Opc = ARM::VZIPq32; break;
3105  }
3106  SDValue Pred = getAL(CurDAG, dl);
3107  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3108  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3109  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3110  return;
3111  }
3112  case ARMISD::VUZP: {
3113  unsigned Opc = 0;
3114  EVT VT = N->getValueType(0);
3115  switch (VT.getSimpleVT().SimpleTy) {
3116  default: return;
3117  case MVT::v8i8: Opc = ARM::VUZPd8; break;
3118  case MVT::v4f16:
3119  case MVT::v4i16: Opc = ARM::VUZPd16; break;
3120  case MVT::v2f32:
3121  // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3122  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3123  case MVT::v16i8: Opc = ARM::VUZPq8; break;
3124  case MVT::v8f16:
3125  case MVT::v8i16: Opc = ARM::VUZPq16; break;
3126  case MVT::v4f32:
3127  case MVT::v4i32: Opc = ARM::VUZPq32; break;
3128  }
3129  SDValue Pred = getAL(CurDAG, dl);
3130  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3131  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3132  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3133  return;
3134  }
3135  case ARMISD::VTRN: {
3136  unsigned Opc = 0;
3137  EVT VT = N->getValueType(0);
3138  switch (VT.getSimpleVT().SimpleTy) {
3139  default: return;
3140  case MVT::v8i8: Opc = ARM::VTRNd8; break;
3141  case MVT::v4f16:
3142  case MVT::v4i16: Opc = ARM::VTRNd16; break;
3143  case MVT::v2f32:
3144  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3145  case MVT::v16i8: Opc = ARM::VTRNq8; break;
3146  case MVT::v8f16:
3147  case MVT::v8i16: Opc = ARM::VTRNq16; break;
3148  case MVT::v4f32:
3149  case MVT::v4i32: Opc = ARM::VTRNq32; break;
3150  }
3151  SDValue Pred = getAL(CurDAG, dl);
3152  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3153  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3154  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3155  return;
3156  }
3157  case ARMISD::BUILD_VECTOR: {
3158  EVT VecVT = N->getValueType(0);
3159  EVT EltVT = VecVT.getVectorElementType();
3160  unsigned NumElts = VecVT.getVectorNumElements();
3161  if (EltVT == MVT::f64) {
3162  assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
3163  ReplaceNode(
3164  N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3165  return;
3166  }
3167  assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
3168  if (NumElts == 2) {
3169  ReplaceNode(
3170  N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3171  return;
3172  }
3173  assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
3174  ReplaceNode(N,
3175  createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
3176  N->getOperand(2), N->getOperand(3)));
3177  return;
3178  }
3179 
3180  case ARMISD::VLD1DUP: {
3181  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
3182  ARM::VLD1DUPd32 };
3183  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
3184  ARM::VLD1DUPq32 };
3185  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes);
3186  return;
3187  }
3188 
3189  case ARMISD::VLD2DUP: {
3190  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3191  ARM::VLD2DUPd32 };
3192  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes);
3193  return;
3194  }
3195 
3196  case ARMISD::VLD3DUP: {
3197  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
3198  ARM::VLD3DUPd16Pseudo,
3199  ARM::VLD3DUPd32Pseudo };
3200  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes);
3201  return;
3202  }
3203 
3204  case ARMISD::VLD4DUP: {
3205  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
3206  ARM::VLD4DUPd16Pseudo,
3207  ARM::VLD4DUPd32Pseudo };
3208  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes);
3209  return;
3210  }
3211 
3212  case ARMISD::VLD1DUP_UPD: {
3213  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
3214  ARM::VLD1DUPd16wb_fixed,
3215  ARM::VLD1DUPd32wb_fixed };
3216  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
3217  ARM::VLD1DUPq16wb_fixed,
3218  ARM::VLD1DUPq32wb_fixed };
3219  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
3220  return;
3221  }
3222 
3223  case ARMISD::VLD2DUP_UPD: {
3224  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
3225  ARM::VLD2DUPd16wb_fixed,
3226  ARM::VLD2DUPd32wb_fixed };
3227  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, Opcodes);
3228  return;
3229  }
3230 
3231  case ARMISD::VLD3DUP_UPD: {
3232  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
3233  ARM::VLD3DUPd16Pseudo_UPD,
3234  ARM::VLD3DUPd32Pseudo_UPD };
3235  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, Opcodes);
3236  return;
3237  }
3238 
3239  case ARMISD::VLD4DUP_UPD: {
3240  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
3241  ARM::VLD4DUPd16Pseudo_UPD,
3242  ARM::VLD4DUPd32Pseudo_UPD };
3243  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, Opcodes);
3244  return;
3245  }
3246 
3247  case ARMISD::VLD1_UPD: {
3248  static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
3249  ARM::VLD1d16wb_fixed,
3250  ARM::VLD1d32wb_fixed,
3251  ARM::VLD1d64wb_fixed };
3252  static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
3253  ARM::VLD1q16wb_fixed,
3254  ARM::VLD1q32wb_fixed,
3255  ARM::VLD1q64wb_fixed };
3256  SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
3257  return;
3258  }
3259 
3260  case ARMISD::VLD2_UPD: {
3261  static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
3262  ARM::VLD2d16wb_fixed,
3263  ARM::VLD2d32wb_fixed,
3264  ARM::VLD1q64wb_fixed};
3265  static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
3266  ARM::VLD2q16PseudoWB_fixed,
3267  ARM::VLD2q32PseudoWB_fixed };
3268  SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
3269  return;
3270  }
3271 
3272  case ARMISD::VLD3_UPD: {
3273  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
3274  ARM::VLD3d16Pseudo_UPD,
3275  ARM::VLD3d32Pseudo_UPD,
3276  ARM::VLD1d64TPseudoWB_fixed};
3277  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3278  ARM::VLD3q16Pseudo_UPD,
3279  ARM::VLD3q32Pseudo_UPD };
3280  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
3281  ARM::VLD3q16oddPseudo_UPD,
3282  ARM::VLD3q32oddPseudo_UPD };
3283  SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3284  return;
3285  }
3286 
3287  case ARMISD::VLD4_UPD: {
3288  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
3289  ARM::VLD4d16Pseudo_UPD,
3290  ARM::VLD4d32Pseudo_UPD,
3291  ARM::VLD1d64QPseudoWB_fixed};
3292  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3293  ARM::VLD4q16Pseudo_UPD,
3294  ARM::VLD4q32Pseudo_UPD };
3295  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
3296  ARM::VLD4q16oddPseudo_UPD,
3297  ARM::VLD4q32oddPseudo_UPD };
3298  SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3299  return;
3300  }
3301 
3302  case ARMISD::VLD2LN_UPD: {
3303  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
3304  ARM::VLD2LNd16Pseudo_UPD,
3305  ARM::VLD2LNd32Pseudo_UPD };
3306  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
3307  ARM::VLD2LNq32Pseudo_UPD };
3308  SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
3309  return;
3310  }
3311 
3312  case ARMISD::VLD3LN_UPD: {
3313  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
3314  ARM::VLD3LNd16Pseudo_UPD,
3315  ARM::VLD3LNd32Pseudo_UPD };
3316  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
3317  ARM::VLD3LNq32Pseudo_UPD };
3318  SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
3319  return;
3320  }
3321 
3322  case ARMISD::VLD4LN_UPD: {
3323  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3324  ARM::VLD4LNd16Pseudo_UPD,
3325  ARM::VLD4LNd32Pseudo_UPD };
3326  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3327  ARM::VLD4LNq32Pseudo_UPD };
3328  SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3329  return;
3330  }
3331 
3332  case ARMISD::VST1_UPD: {
3333  static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3334  ARM::VST1d16wb_fixed,
3335  ARM::VST1d32wb_fixed,
3336  ARM::VST1d64wb_fixed };
3337  static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3338  ARM::VST1q16wb_fixed,
3339  ARM::VST1q32wb_fixed,
3340  ARM::VST1q64wb_fixed };
3341  SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
3342  return;
3343  }
3344 
3345  case ARMISD::VST2_UPD: {
3346  static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3347  ARM::VST2d16wb_fixed,
3348  ARM::VST2d32wb_fixed,
3349  ARM::VST1q64wb_fixed};
3350  static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3351  ARM::VST2q16PseudoWB_fixed,
3352  ARM::VST2q32PseudoWB_fixed };
3353  SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
3354  return;
3355  }
3356 
3357  case ARMISD::VST3_UPD: {
3358  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3359  ARM::VST3d16Pseudo_UPD,
3360  ARM::VST3d32Pseudo_UPD,
3361  ARM::VST1d64TPseudoWB_fixed};
3362  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3363  ARM::VST3q16Pseudo_UPD,
3364  ARM::VST3q32Pseudo_UPD };
3365  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3366  ARM::VST3q16oddPseudo_UPD,
3367  ARM::VST3q32oddPseudo_UPD };
3368  SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3369  return;
3370  }
3371 
3372  case ARMISD::VST4_UPD: {
3373  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3374  ARM::VST4d16Pseudo_UPD,
3375  ARM::VST4d32Pseudo_UPD,
3376  ARM::VST1d64QPseudoWB_fixed};
3377  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3378  ARM::VST4q16Pseudo_UPD,
3379  ARM::VST4q32Pseudo_UPD };
3380  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3381  ARM::VST4q16oddPseudo_UPD,
3382  ARM::VST4q32oddPseudo_UPD };
3383  SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3384  return;
3385  }
3386 
3387  case ARMISD::VST2LN_UPD: {
3388  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3389  ARM::VST2LNd16Pseudo_UPD,
3390  ARM::VST2LNd32Pseudo_UPD };
3391  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3392  ARM::VST2LNq32Pseudo_UPD };
3393  SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3394  return;
3395  }
3396 
3397  case ARMISD::VST3LN_UPD: {
3398  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3399  ARM::VST3LNd16Pseudo_UPD,
3400  ARM::VST3LNd32Pseudo_UPD };
3401  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3402  ARM::VST3LNq32Pseudo_UPD };
3403  SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3404  return;
3405  }
3406 
3407  case ARMISD::VST4LN_UPD: {
3408  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3409  ARM::VST4LNd16Pseudo_UPD,
3410  ARM::VST4LNd32Pseudo_UPD };
3411  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3412  ARM::VST4LNq32Pseudo_UPD };
3413  SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3414  return;
3415  }
3416 
3417  case ISD::INTRINSIC_VOID:
3418  case ISD::INTRINSIC_W_CHAIN: {
3419  unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3420  switch (IntNo) {
3421  default:
3422  break;
3423 
3424  case Intrinsic::arm_mrrc:
3425  case Intrinsic::arm_mrrc2: {
3426  SDLoc dl(N);
3427  SDValue Chain = N->getOperand(0);
3428  unsigned Opc;
3429 
3430  if (Subtarget->isThumb())
3431  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
3432  else
3433  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
3434 
3436  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
3437  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
3438  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
3439 
3440  // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
3441  // instruction will always be '1111' but it is possible in assembly language to specify
3442  // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
3443  if (Opc != ARM::MRRC2) {
3444  Ops.push_back(getAL(CurDAG, dl));
3445  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3446  }
3447 
3448  Ops.push_back(Chain);
3449 
3450  // Writes to two registers.
3451  const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
3452 
3453  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
3454  return;
3455  }
3456  case Intrinsic::arm_ldaexd:
3457  case Intrinsic::arm_ldrexd: {
3458  SDLoc dl(N);
3459  SDValue Chain = N->getOperand(0);
3460  SDValue MemAddr = N->getOperand(2);
3461  bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
3462 
3463  bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3464  unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3465  : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3466 
3467  // arm_ldrexd returns a i64 value in {i32, i32}
3468  std::vector<EVT> ResTys;
3469  if (isThumb) {
3470  ResTys.push_back(MVT::i32);
3471  ResTys.push_back(MVT::i32);
3472  } else
3473  ResTys.push_back(MVT::Untyped);
3474  ResTys.push_back(MVT::Other);
3475 
3476  // Place arguments in the right order.
3477  SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3478  CurDAG->getRegister(0, MVT::i32), Chain};
3479  SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3480  // Transfer memoperands.
3481  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3482  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3483 
3484  // Remap uses.
3485  SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3486  if (!SDValue(N, 0).use_empty()) {
3487  SDValue Result;
3488  if (isThumb)
3489  Result = SDValue(Ld, 0);
3490  else {
3491  SDValue SubRegIdx =
3492  CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
3493  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3494  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3495  Result = SDValue(ResNode,0);
3496  }
3497  ReplaceUses(SDValue(N, 0), Result);
3498  }
3499  if (!SDValue(N, 1).use_empty()) {
3500  SDValue Result;
3501  if (isThumb)
3502  Result = SDValue(Ld, 1);
3503  else {
3504  SDValue SubRegIdx =
3505  CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
3506  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3507  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3508  Result = SDValue(ResNode,0);
3509  }
3510  ReplaceUses(SDValue(N, 1), Result);
3511  }
3512  ReplaceUses(SDValue(N, 2), OutChain);
3513  CurDAG->RemoveDeadNode(N);
3514  return;
3515  }
3516  case Intrinsic::arm_stlexd:
3517  case Intrinsic::arm_strexd: {
3518  SDLoc dl(N);
3519  SDValue Chain = N->getOperand(0);
3520  SDValue Val0 = N->getOperand(2);
3521  SDValue Val1 = N->getOperand(3);
3522  SDValue MemAddr = N->getOperand(4);
3523 
3524  // Store exclusive double return a i32 value which is the return status
3525  // of the issued store.
3526  const EVT ResTys[] = {MVT::i32, MVT::Other};
3527 
3528  bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3529  // Place arguments in the right order.
3531  if (isThumb) {
3532  Ops.push_back(Val0);
3533  Ops.push_back(Val1);
3534  } else
3535  // arm_strexd uses GPRPair.
3536  Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3537  Ops.push_back(MemAddr);
3538  Ops.push_back(getAL(CurDAG, dl));
3539  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3540  Ops.push_back(Chain);
3541 
3542  bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3543  unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3544  : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3545 
3546  SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3547  // Transfer memoperands.
3548  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3549  CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3550 
3551  ReplaceNode(N, St);
3552  return;
3553  }
3554 
3555  case Intrinsic::arm_neon_vld1: {
3556  static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3557  ARM::VLD1d32, ARM::VLD1d64 };
3558  static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3559  ARM::VLD1q32, ARM::VLD1q64};
3560  SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3561  return;
3562  }
3563 
3564  case Intrinsic::arm_neon_vld1x2: {
3565  static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3566  ARM::VLD1q32, ARM::VLD1q64 };
3567  static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
3568  ARM::VLD1d16QPseudo,
3569  ARM::VLD1d32QPseudo,
3570  ARM::VLD1d64QPseudo };
3571  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3572  return;
3573  }
3574 
3575  case Intrinsic::arm_neon_vld1x3: {
3576  static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
3577  ARM::VLD1d16TPseudo,
3578  ARM::VLD1d32TPseudo,
3579  ARM::VLD1d64TPseudo };
3580  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
3581  ARM::VLD1q16LowTPseudo_UPD,
3582  ARM::VLD1q32LowTPseudo_UPD,
3583  ARM::VLD1q64LowTPseudo_UPD };
3584  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
3585  ARM::VLD1q16HighTPseudo,
3586  ARM::VLD1q32HighTPseudo,
3587  ARM::VLD1q64HighTPseudo };
3588  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3589  return;
3590  }
3591 
3592  case Intrinsic::arm_neon_vld1x4: {
3593  static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
3594  ARM::VLD1d16QPseudo,
3595  ARM::VLD1d32QPseudo,
3596  ARM::VLD1d64QPseudo };
3597  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
3598  ARM::VLD1q16LowQPseudo_UPD,
3599  ARM::VLD1q32LowQPseudo_UPD,
3600  ARM::VLD1q64LowQPseudo_UPD };
3601  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
3602  ARM::VLD1q16HighQPseudo,
3603  ARM::VLD1q32HighQPseudo,
3604  ARM::VLD1q64HighQPseudo };
3605  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3606  return;
3607  }
3608 
3609  case Intrinsic::arm_neon_vld2: {
3610  static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3611  ARM::VLD2d32, ARM::VLD1q64 };
3612  static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3613  ARM::VLD2q32Pseudo };
3614  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3615  return;
3616  }
3617 
3618  case Intrinsic::arm_neon_vld3: {
3619  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3620  ARM::VLD3d16Pseudo,
3621  ARM::VLD3d32Pseudo,
3622  ARM::VLD1d64TPseudo };
3623  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3624  ARM::VLD3q16Pseudo_UPD,
3625  ARM::VLD3q32Pseudo_UPD };
3626  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3627  ARM::VLD3q16oddPseudo,
3628  ARM::VLD3q32oddPseudo };
3629  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3630  return;
3631  }
3632 
3633  case Intrinsic::arm_neon_vld4: {
3634  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3635  ARM::VLD4d16Pseudo,
3636  ARM::VLD4d32Pseudo,
3637  ARM::VLD1d64QPseudo };
3638  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3639  ARM::VLD4q16Pseudo_UPD,
3640  ARM::VLD4q32Pseudo_UPD };
3641  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3642  ARM::VLD4q16oddPseudo,
3643  ARM::VLD4q32oddPseudo };
3644  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3645  return;
3646  }
3647 
3648  case Intrinsic::arm_neon_vld2dup: {
3649  static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3650  ARM::VLD2DUPd32, ARM::VLD1q64 };
3651  static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
3652  ARM::VLD2DUPq16EvenPseudo,
3653  ARM::VLD2DUPq32EvenPseudo };
3654  static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
3655  ARM::VLD2DUPq16OddPseudo,
3656  ARM::VLD2DUPq32OddPseudo };
3657  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2,
3658  DOpcodes, QOpcodes0, QOpcodes1);
3659  return;
3660  }
3661 
3662  case Intrinsic::arm_neon_vld3dup: {
3663  static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
3664  ARM::VLD3DUPd16Pseudo,
3665  ARM::VLD3DUPd32Pseudo,
3666  ARM::VLD1d64TPseudo };
3667  static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
3668  ARM::VLD3DUPq16EvenPseudo,
3669  ARM::VLD3DUPq32EvenPseudo };
3670  static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
3671  ARM::VLD3DUPq16OddPseudo,
3672  ARM::VLD3DUPq32OddPseudo };
3673  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3,
3674  DOpcodes, QOpcodes0, QOpcodes1);
3675  return;
3676  }
3677 
3678  case Intrinsic::arm_neon_vld4dup: {
3679  static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
3680  ARM::VLD4DUPd16Pseudo,
3681  ARM::VLD4DUPd32Pseudo,
3682  ARM::VLD1d64QPseudo };
3683  static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
3684  ARM::VLD4DUPq16EvenPseudo,
3685  ARM::VLD4DUPq32EvenPseudo };
3686  static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
3687  ARM::VLD4DUPq16OddPseudo,
3688  ARM::VLD4DUPq32OddPseudo };
3689  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4,
3690  DOpcodes, QOpcodes0, QOpcodes1);
3691  return;
3692  }
3693 
3694  case Intrinsic::arm_neon_vld2lane: {
3695  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3696  ARM::VLD2LNd16Pseudo,
3697  ARM::VLD2LNd32Pseudo };
3698  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3699  ARM::VLD2LNq32Pseudo };
3700  SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3701  return;
3702  }
3703 
3704  case Intrinsic::arm_neon_vld3lane: {
3705  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3706  ARM::VLD3LNd16Pseudo,
3707  ARM::VLD3LNd32Pseudo };
3708  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3709  ARM::VLD3LNq32Pseudo };
3710  SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3711  return;
3712  }
3713 
3714  case Intrinsic::arm_neon_vld4lane: {
3715  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3716  ARM::VLD4LNd16Pseudo,
3717  ARM::VLD4LNd32Pseudo };
3718  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3719  ARM::VLD4LNq32Pseudo };
3720  SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3721  return;
3722  }
3723 
3724  case Intrinsic::arm_neon_vst1: {
3725  static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3726  ARM::VST1d32, ARM::VST1d64 };
3727  static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3728  ARM::VST1q32, ARM::VST1q64 };
3729  SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3730  return;
3731  }
3732 
3733  case Intrinsic::arm_neon_vst1x2: {
3734  static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3735  ARM::VST1q32, ARM::VST1q64 };
3736  static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
3737  ARM::VST1d16QPseudo,
3738  ARM::VST1d32QPseudo,
3739  ARM::VST1d64QPseudo };
3740  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3741  return;
3742  }
3743 
3744  case Intrinsic::arm_neon_vst1x3: {
3745  static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
3746  ARM::VST1d16TPseudo,
3747  ARM::VST1d32TPseudo,
3748  ARM::VST1d64TPseudo };
3749  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
3750  ARM::VST1q16LowTPseudo_UPD,
3751  ARM::VST1q32LowTPseudo_UPD,
3752  ARM::VST1q64LowTPseudo_UPD };
3753  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
3754  ARM::VST1q16HighTPseudo,
3755  ARM::VST1q32HighTPseudo,
3756  ARM::VST1q64HighTPseudo };
3757  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3758  return;
3759  }
3760 
3761  case Intrinsic::arm_neon_vst1x4: {
3762  static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
3763  ARM::VST1d16QPseudo,
3764  ARM::VST1d32QPseudo,
3765  ARM::VST1d64QPseudo };
3766  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
3767  ARM::VST1q16LowQPseudo_UPD,
3768  ARM::VST1q32LowQPseudo_UPD,
3769  ARM::VST1q64LowQPseudo_UPD };
3770  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
3771  ARM::VST1q16HighQPseudo,
3772  ARM::VST1q32HighQPseudo,
3773  ARM::VST1q64HighQPseudo };
3774  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3775  return;
3776  }
3777 
3778  case Intrinsic::arm_neon_vst2: {
3779  static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3780  ARM::VST2d32, ARM::VST1q64 };
3781  static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3782  ARM::VST2q32Pseudo };
3783  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3784  return;
3785  }
3786 
3787  case Intrinsic::arm_neon_vst3: {
3788  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3789  ARM::VST3d16Pseudo,
3790  ARM::VST3d32Pseudo,
3791  ARM::VST1d64TPseudo };
3792  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3793  ARM::VST3q16Pseudo_UPD,
3794  ARM::VST3q32Pseudo_UPD };
3795  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3796  ARM::VST3q16oddPseudo,
3797  ARM::VST3q32oddPseudo };
3798  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3799  return;
3800  }
3801 
3802  case Intrinsic::arm_neon_vst4: {
3803  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3804  ARM::VST4d16Pseudo,
3805  ARM::VST4d32Pseudo,
3806  ARM::VST1d64QPseudo };
3807  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3808  ARM::VST4q16Pseudo_UPD,
3809  ARM::VST4q32Pseudo_UPD };
3810  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3811  ARM::VST4q16oddPseudo,
3812  ARM::VST4q32oddPseudo };
3813  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3814  return;
3815  }
3816 
3817  case Intrinsic::arm_neon_vst2lane: {
3818  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3819  ARM::VST2LNd16Pseudo,
3820  ARM::VST2LNd32Pseudo };
3821  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3822  ARM::VST2LNq32Pseudo };
3823  SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3824  return;
3825  }
3826 
3827  case Intrinsic::arm_neon_vst3lane: {
3828  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3829  ARM::VST3LNd16Pseudo,
3830  ARM::VST3LNd32Pseudo };
3831  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3832  ARM::VST3LNq32Pseudo };
3833  SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3834  return;
3835  }
3836 
3837  case Intrinsic::arm_neon_vst4lane: {
3838  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3839  ARM::VST4LNd16Pseudo,
3840  ARM::VST4LNd32Pseudo };
3841  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3842  ARM::VST4LNq32Pseudo };
3843  SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3844  return;
3845  }
3846  }
3847  break;
3848  }
3849 
3850  case ISD::ATOMIC_CMP_SWAP:
3851  SelectCMP_SWAP(N);
3852  return;
3853  }
3854 
3855  SelectCode(N);
3856 }
3857 
3858 // Inspect a register string of the form
3859 // cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
3860 // cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
3861 // and obtain the integer operands from them, adding these operands to the
3862 // provided vector.
3864  SelectionDAG *CurDAG,
3865  const SDLoc &DL,
3866  std::vector<SDValue> &Ops) {
3868  RegString.split(Fields, ':');
3869 
3870  if (Fields.size() > 1) {
3871  bool AllIntFields = true;
3872 
3873  for (StringRef Field : Fields) {
3874  // Need to trim out leading 'cp' characters and get the integer field.
3875  unsigned IntField;
3876  AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
3877  Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
3878  }
3879 
3880  assert(AllIntFields &&
3881  "Unexpected non-integer value in special register string.");
3882  }
3883 }
3884 
3885 // Maps a Banked Register string to its mask value. The mask value returned is
3886 // for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
3887 // mask operand, which expresses which register is to be used, e.g. r8, and in
3888 // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
3889 // was invalid.
3890 static inline int getBankedRegisterMask(StringRef RegString) {
3891  auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
3892  if (!TheReg)
3893  return -1;
3894  return TheReg->Encoding;
3895 }
3896 
3897 // The flags here are common to those allowed for apsr in the A class cores and
3898 // those allowed for the special registers in the M class cores. Returns a
3899 // value representing which flags were present, -1 if invalid.
3900 static inline int getMClassFlagsMask(StringRef Flags) {
3901  return StringSwitch<int>(Flags)
3902  .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
3903  // correct when flags are not permitted
3904  .Case("g", 0x1)
3905  .Case("nzcvq", 0x2)
3906  .Case("nzcvqg", 0x3)
3907  .Default(-1);
3908 }
3909 
3910 // Maps MClass special registers string to its value for use in the
3911 // t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
3912 // Returns -1 to signify that the string was invalid.
3913 static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
3914  auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
3915  const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
3916  if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
3917  return -1;
3918  return (int)(TheReg->Encoding & 0xFFF); // SYSm value
3919 }
3920 
3922  // The mask operand contains the special register (R Bit) in bit 4, whether
3923  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
3924  // bits 3-0 contains the fields to be accessed in the special register, set by
3925  // the flags provided with the register.
3926  int Mask = 0;
3927  if (Reg == "apsr") {
3928  // The flags permitted for apsr are the same flags that are allowed in
3929  // M class registers. We get the flag value and then shift the flags into
3930  // the correct place to combine with the mask.
3931  Mask = getMClassFlagsMask(Flags);
3932  if (Mask == -1)
3933  return -1;
3934  return Mask << 2;
3935  }
3936 
3937  if (Reg != "cpsr" && Reg != "spsr") {
3938  return -1;
3939  }
3940 
3941  // This is the same as if the flags were "fc"
3942  if (Flags.empty() || Flags == "all")
3943  return Mask | 0x9;
3944 
3945  // Inspect the supplied flags string and set the bits in the mask for
3946  // the relevant and valid flags allowed for cpsr and spsr.
3947  for (char Flag : Flags) {
3948  int FlagVal;
3949  switch (Flag) {
3950  case 'c':
3951  FlagVal = 0x1;
3952  break;
3953  case 'x':
3954  FlagVal = 0x2;
3955  break;
3956  case 's':
3957  FlagVal = 0x4;
3958  break;
3959  case 'f':
3960  FlagVal = 0x8;
3961  break;
3962  default:
3963  FlagVal = 0;
3964  }
3965 
3966  // This avoids allowing strings where the same flag bit appears twice.
3967  if (!FlagVal || (Mask & FlagVal))
3968  return -1;
3969  Mask |= FlagVal;
3970  }
3971 
3972  // If the register is spsr then we need to set the R bit.
3973  if (Reg == "spsr")
3974  Mask |= 0x10;
3975 
3976  return Mask;
3977 }
3978 
3979 // Lower the read_register intrinsic to ARM specific DAG nodes
3980 // using the supplied metadata string to select the instruction node to use
3981 // and the registers/masks to construct as operands for the node.
3982 bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
3983  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
3984  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
3985  bool IsThumb2 = Subtarget->isThumb2();
3986  SDLoc DL(N);
3987 
3988  std::vector<SDValue> Ops;
3989  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
3990 
3991  if (!Ops.empty()) {
3992  // If the special register string was constructed of fields (as defined
3993  // in the ACLE) then need to lower to MRC node (32 bit) or
3994  // MRRC node(64 bit), we can make the distinction based on the number of
3995  // operands we have.
3996  unsigned Opcode;
3997  SmallVector<EVT, 3> ResTypes;
3998  if (Ops.size() == 5){
3999  Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
4000  ResTypes.append({ MVT::i32, MVT::Other });
4001  } else {
4002  assert(Ops.size() == 3 &&
4003  "Invalid number of fields in special register string.");
4004  Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
4005  ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
4006  }
4007 
4008  Ops.push_back(getAL(CurDAG, DL));
4009  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4010  Ops.push_back(N->getOperand(0));
4011  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
4012  return true;
4013  }
4014 
4015  std::string SpecialReg = RegString->getString().lower();
4016 
4017  int BankedReg = getBankedRegisterMask(SpecialReg);
4018  if (BankedReg != -1) {
4019  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
4020  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4021  N->getOperand(0) };
4022  ReplaceNode(
4023  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
4024  DL, MVT::i32, MVT::Other, Ops));
4025  return true;
4026  }
4027 
4028  // The VFP registers are read by creating SelectionDAG nodes with opcodes
4029  // corresponding to the register that is being read from. So we switch on the
4030  // string to find which opcode we need to use.
4031  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4032  .Case("fpscr", ARM::VMRS)
4033  .Case("fpexc", ARM::VMRS_FPEXC)
4034  .Case("fpsid", ARM::VMRS_FPSID)
4035  .Case("mvfr0", ARM::VMRS_MVFR0)
4036  .Case("mvfr1", ARM::VMRS_MVFR1)
4037  .Case("mvfr2", ARM::VMRS_MVFR2)
4038  .Case("fpinst", ARM::VMRS_FPINST)
4039  .Case("fpinst2", ARM::VMRS_FPINST2)
4040  .Default(0);
4041 
4042  // If an opcode was found then we can lower the read to a VFP instruction.
4043  if (Opcode) {
4044  if (!Subtarget->hasVFP2())
4045  return false;
4046  if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8())
4047  return false;
4048 
4049  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4050  N->getOperand(0) };
4051  ReplaceNode(N,
4052  CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
4053  return true;
4054  }
4055 
4056  // If the target is M Class then need to validate that the register string
4057  // is an acceptable value, so check that a mask can be constructed from the
4058  // string.
4059  if (Subtarget->isMClass()) {
4060  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4061  if (SYSmValue == -1)
4062  return false;
4063 
4064  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4065  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4066  N->getOperand(0) };
4067  ReplaceNode(
4068  N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
4069  return true;
4070  }
4071 
4072  // Here we know the target is not M Class so we need to check if it is one
4073  // of the remaining possible values which are apsr, cpsr or spsr.
4074  if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
4075  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4076  N->getOperand(0) };
4077  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
4078  DL, MVT::i32, MVT::Other, Ops));
4079  return true;
4080  }
4081 
4082  if (SpecialReg == "spsr") {
4083  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4084  N->getOperand(0) };
4085  ReplaceNode(
4086  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
4087  MVT::i32, MVT::Other, Ops));
4088  return true;
4089  }
4090 
4091  return false;
4092 }
4093 
4094 // Lower the write_register intrinsic to ARM specific DAG nodes
4095 // using the supplied metadata string to select the instruction node to use
4096 // and the registers/masks to use in the nodes
4097 bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
4098  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
4099  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
4100  bool IsThumb2 = Subtarget->isThumb2();
4101  SDLoc DL(N);
4102 
4103  std::vector<SDValue> Ops;
4104  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
4105 
4106  if (!Ops.empty()) {
4107  // If the special register string was constructed of fields (as defined
4108  // in the ACLE) then need to lower to MCR node (32 bit) or
4109  // MCRR node(64 bit), we can make the distinction based on the number of
4110  // operands we have.
4111  unsigned Opcode;
4112  if (Ops.size() == 5) {
4113  Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
4114  Ops.insert(Ops.begin()+2, N->getOperand(2));
4115  } else {
4116  assert(Ops.size() == 3 &&
4117  "Invalid number of fields in special register string.");
4118  Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
4119  SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
4120  Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
4121  }
4122 
4123  Ops.push_back(getAL(CurDAG, DL));
4124  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4125  Ops.push_back(N->getOperand(0));
4126 
4127  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4128  return true;
4129  }
4130 
4131  std::string SpecialReg = RegString->getString().lower();
4132  int BankedReg = getBankedRegisterMask(SpecialReg);
4133  if (BankedReg != -1) {
4134  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
4135  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4136  N->getOperand(0) };
4137  ReplaceNode(
4138  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
4139  DL, MVT::Other, Ops));
4140  return true;
4141  }
4142 
4143  // The VFP registers are written to by creating SelectionDAG nodes with
4144  // opcodes corresponding to the register that is being written. So we switch
4145  // on the string to find which opcode we need to use.
4146  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4147  .Case("fpscr", ARM::VMSR)
4148  .Case("fpexc", ARM::VMSR_FPEXC)
4149  .Case("fpsid", ARM::VMSR_FPSID)
4150  .Case("fpinst", ARM::VMSR_FPINST)
4151  .Case("fpinst2", ARM::VMSR_FPINST2)
4152  .Default(0);
4153 
4154  if (Opcode) {
4155  if (!Subtarget->hasVFP2())
4156  return false;
4157  Ops = { N->getOperand(2), getAL(CurDAG, DL),
4158  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4159  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4160  return true;
4161  }
4162 
4163  std::pair<StringRef, StringRef> Fields;
4164  Fields = StringRef(SpecialReg).rsplit('_');
4165  std::string Reg = Fields.first.str();
4166  StringRef Flags = Fields.second;
4167 
4168  // If the target was M Class then need to validate the special register value
4169  // and retrieve the mask for use in the instruction node.
4170  if (Subtarget->isMClass()) {
4171  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4172  if (SYSmValue == -1)
4173  return false;
4174 
4175  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4176  N->getOperand(2), getAL(CurDAG, DL),
4177  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4178  ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
4179  return true;
4180  }
4181 
4182  // We then check to see if a valid mask can be constructed for one of the
4183  // register string values permitted for the A and R class cores. These values
4184  // are apsr, spsr and cpsr; these are also valid on older cores.
4185  int Mask = getARClassRegisterMask(Reg, Flags);
4186  if (Mask != -1) {
4187  Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
4188  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4189  N->getOperand(0) };
4190  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
4191  DL, MVT::Other, Ops));
4192  return true;
4193  }
4194 
4195  return false;
4196 }
4197 
4198 bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
4199  std::vector<SDValue> AsmNodeOperands;
4200  unsigned Flag, Kind;
4201  bool Changed = false;
4202  unsigned NumOps = N->getNumOperands();
4203 
4204  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
4205  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
4206  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
4207  // respectively. Since there is no constraint to explicitly specify a
4208  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
4209  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
4210  // them into a GPRPair.
4211 
4212  SDLoc dl(N);
4213  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
4214  : SDValue(nullptr,0);
4215 
4216  SmallVector<bool, 8> OpChanged;
4217  // Glue node will be appended late.
4218  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
4219  SDValue op = N->getOperand(i);
4220  AsmNodeOperands.push_back(op);
4221 
4223  continue;
4224 
4225  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
4226  Flag = C->getZExtValue();
4227  Kind = InlineAsm::getKind(Flag);
4228  }
4229  else
4230  continue;
4231 
4232  // Immediate operands to inline asm in the SelectionDAG are modeled with
4233  // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
4234  // the second is a constant with the value of the immediate. If we get here
4235  // and we have a Kind_Imm, skip the next operand, and continue.
4236  if (Kind == InlineAsm::Kind_Imm) {
4237  SDValue op = N->getOperand(++i);
4238  AsmNodeOperands.push_back(op);
4239  continue;
4240  }
4241 
4242  unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
4243  if (NumRegs)
4244  OpChanged.push_back(false);
4245 
4246  unsigned DefIdx = 0;
4247  bool IsTiedToChangedOp = false;
4248  // If it's a use that is tied with a previous def, it has no
4249  // reg class constraint.
4250  if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
4251  IsTiedToChangedOp = OpChanged[DefIdx];
4252 
4253  // Memory operands to inline asm in the SelectionDAG are modeled with two
4254  // operands: a constant of value InlineAsm::Kind_Mem followed by the input
4255  // operand. If we get here and we have a Kind_Mem, skip the next operand (so
4256  // it doesn't get misinterpreted), and continue. We do this here because
4257  // it's important to update the OpChanged array correctly before moving on.
4258  if (Kind == InlineAsm::Kind_Mem) {
4259  SDValue op = N->getOperand(++i);
4260  AsmNodeOperands.push_back(op);
4261  continue;
4262  }
4263 
4264  if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
4266  continue;
4267 
4268  unsigned RC;
4269  bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
4270  if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
4271  || NumRegs != 2)
4272  continue;
4273 
4274  assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
4275  SDValue V0 = N->getOperand(i+1);
4276  SDValue V1 = N->getOperand(i+2);
4277  unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
4278  unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
4279  SDValue PairedReg;
4280  MachineRegisterInfo &MRI = MF->getRegInfo();
4281 
4282  if (Kind == InlineAsm::Kind_RegDef ||
4284  // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
4285  // the original GPRs.
4286 
4287  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4288  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4289  SDValue Chain = SDValue(N,0);
4290 
4291  SDNode *GU = N->getGluedUser();
4292  SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4293  Chain.getValue(1));
4294 
4295  // Extract values from a GPRPair reg and copy to the original GPR reg.
4296  SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4297  RegCopy);
4298  SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4299  RegCopy);
4300  SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4301  RegCopy.getValue(1));
4302  SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4303 
4304  // Update the original glue user.
4305  std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4306  Ops.push_back(T1.getValue(1));
4307  CurDAG->UpdateNodeOperands(GU, Ops);
4308  }
4309  else {
4310  // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a
4311  // GPRPair and then pass the GPRPair to the inline asm.
4312  SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4313 
4314  // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
4315  SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4316  Chain.getValue(1));
4317  SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4318  T0.getValue(1));
4319  SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4320 
4321  // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
4322  // i32 VRs of inline asm with it.
4323  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4324  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4325  Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
4326 
4327  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
4328  Glue = Chain.getValue(1);
4329  }
4330 
4331  Changed = true;
4332 
4333  if(PairedReg.getNode()) {
4334  OpChanged[OpChanged.size() -1 ] = true;
4335  Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
4336  if (IsTiedToChangedOp)
4337  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
4338  else
4339  Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
4340  // Replace the current flag.
4341  AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
4342  Flag, dl, MVT::i32);
4343  // Add the new register node and skip the original two GPRs.
4344  AsmNodeOperands.push_back(PairedReg);
4345  // Skip the next two GPRs.
4346  i += 2;
4347  }
4348  }
4349 
4350  if (Glue.getNode())
4351  AsmNodeOperands.push_back(Glue);
4352  if (!Changed)
4353  return false;
4354 
4355  SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
4356  CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
4357  New->setNodeId(-1);
4358  ReplaceNode(N, New.getNode());
4359  return true;
4360 }
4361 
4362 
4363 bool ARMDAGToDAGISel::
4364 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4365  std::vector<SDValue> &OutOps) {
4366  switch(ConstraintID) {
4367  default:
4368  llvm_unreachable("Unexpected asm memory constraint");
4370  // FIXME: It seems strange that 'i' is needed here since it's supposed to
4371  // be an immediate and not a memory constraint.
4383  // Require the address to be in a register. That is safe for all ARM
4384  // variants and it is hard to do anything much smarter without knowing
4385  // how the operand is used.
4386  OutOps.push_back(Op);
4387  return false;
4388  }
4389  return true;
4390 }
4391 
4392 /// createARMISelDag - This pass converts a legalized DAG into a
4393 /// ARM-specific DAG, ready for instruction scheduling.
4394 ///
4396  CodeGenOpt::Level OptLevel) {
4397  return new ARMDAGToDAGISel(TM, OptLevel);
4398 }
uint64_t CallInst * C
bool useMovt() const
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
EVT getValueType() const
Return the ValueType of the referenced return value.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const SDValue & getOffset() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isThumb() const
Definition: ARMSubtarget.h:717
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Various leaf nodes.
Definition: ISDOpcodes.h:59
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
const SDValue & getValue() const
SDVTList getVTList() const
static Optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
unsigned Reg
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:130
unsigned getAlignment() const
static bool isVSTfixed(unsigned Opc)
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:729
bool isSOImmTwoPartVal(unsigned V)
isSOImmTwoPartVal - Return true if the specified value can be obtained by or&#39;ing together two SOImmVa...
bool hasV6Ops() const
Definition: ARMSubtarget.h:541
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1068
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
bool isThumb1Only() const
Definition: ARMSubtarget.h:719
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
#define op(i)
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:188
static bool isThumb(const MCSubtargetInfo &STI)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:341
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1508
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:158
bool hasV8MBaselineOps() const
Definition: ARMSubtarget.h:552
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.h:1631
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isFpMLxInstruction(unsigned Opcode) const
isFpMLxInstruction - Return true if the specified opcode is a fp MLA / MLS instruction.
A description of a memory reference used in the backend.
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:299
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:409
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:477
Base class for LoadSDNode and StoreSDNode.
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:392
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:169
const MDNode * getMD() const
op_iterator op_end() const
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:668
bool hasVFP2() const
Definition: ARMSubtarget.h:572
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
bool hasDSP() const
Definition: ARMSubtarget.h:633
static bool isVLDfixed(unsigned Opc)
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
bool hasV6T2Ops() const
Definition: ARMSubtarget.h:544
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:581
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
This class is used to represent ISD::STORE nodes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:117
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
const SDValue & getBasePtr() const
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:427
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:165
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:84
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
Container class for subtarget features.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:119
static int getBankedRegisterMask(StringRef RegString)
unsigned countPopulation() const
Count the number of bits set.
Definition: APInt.h:1657
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
bool isMClass() const
Definition: ARMSubtarget.h:722
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
const SDValue & getOperand(unsigned Num) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool hasFPARMv8() const
Definition: ARMSubtarget.h:575
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:335
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
static unsigned getKind(unsigned Flags)
Definition: InlineAsm.h:324
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
self_iterator getIterator()
Definition: ilist_node.h:81
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:800
Extended Value Type.
Definition: ValueTypes.h:33
size_t size() const
Definition: SmallVector.h:52
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
Iterator for intrusive lists based on ilist_node.
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:264
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
Represents one node in the SelectionDAG.
#define NC
Definition: regutils.h:42
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:72
EVT getMemoryVT() const
Return the type of the in-memory value.
Class for arbitrary precision integers.
Definition: APInt.h:69
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:404
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:387
static bool shouldUseZeroOffsetLdSt(SDValue N)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getNodeId() const
Return the unique node id.
static bool hasRegClassConstraint(unsigned Flag, unsigned &RC)
hasRegClassConstraint - Returns true if the flag contains a register class constraint.
Definition: InlineAsm.h:350
bool isThumb2() const
Definition: ARMSubtarget.h:720
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: ValueTypes.h:176
const SDValue & getValue() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:386
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:485
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:615
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:174
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOpt::Level OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
bool hasVMLxHazards() const
Definition: ARMSubtarget.h:616
LLVM_NODISCARD std::string lower() const
Definition: StringRef.cpp:107
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
INLINEASM_BR - Terminator version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:671
bool isSwift() const
Definition: ARMSubtarget.h:563
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:203
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition: MathExtras.h:416
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
A single uniqued string.
Definition: Metadata.h:603
static int getMClassFlagsMask(StringRef Flags)
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1595
const SDValue & getOperand(unsigned i) const
uint64_t getZExtValue() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void setObjectAlignment(int ObjectIdx, unsigned Align)
setObjectAlignment - Change the alignment of the specified stack object.
bool hasThumb2() const
Definition: ARMSubtarget.h:721
bool isLikeA9() const
Definition: ARMSubtarget.h:565
#define T1
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:287
This file describes how to lower LLVM code to machine code.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:923
This class is used to represent ISD::LOAD nodes.