LLVM  9.0.0svn
ARMISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the ARM target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "ARM.h"
14 #include "ARMBaseInstrInfo.h"
15 #include "ARMTargetMachine.h"
17 #include "Utils/ARMBaseInfo.h"
18 #include "llvm/ADT/StringSwitch.h"
26 #include "llvm/IR/CallingConv.h"
27 #include "llvm/IR/Constants.h"
28 #include "llvm/IR/DerivedTypes.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/Intrinsics.h"
31 #include "llvm/IR/LLVMContext.h"
33 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "arm-isel"
40 
41 static cl::opt<bool>
42 DisableShifterOp("disable-shifter-op", cl::Hidden,
43  cl::desc("Disable isel of shifter-op"),
44  cl::init(false));
45 
46 //===--------------------------------------------------------------------===//
47 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
48 /// instructions for SelectionDAG operations.
49 ///
50 namespace {
51 
52 class ARMDAGToDAGISel : public SelectionDAGISel {
53  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
54  /// make the right decision when generating code for different targets.
55  const ARMSubtarget *Subtarget;
56 
57 public:
58  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
59  : SelectionDAGISel(tm, OptLevel) {}
60 
61  bool runOnMachineFunction(MachineFunction &MF) override {
62  // Reset the subtarget each time through.
63  Subtarget = &MF.getSubtarget<ARMSubtarget>();
65  return true;
66  }
67 
68  StringRef getPassName() const override { return "ARM Instruction Selection"; }
69 
70  void PreprocessISelDAG() override;
71 
72  /// getI32Imm - Return a target constant of type i32 with the specified
73  /// value.
74  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
75  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
76  }
77 
78  void Select(SDNode *N) override;
79 
80  bool hasNoVMLxHazardUse(SDNode *N) const;
81  bool isShifterOpProfitable(const SDValue &Shift,
82  ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
83  bool SelectRegShifterOperand(SDValue N, SDValue &A,
84  SDValue &B, SDValue &C,
85  bool CheckProfitability = true);
86  bool SelectImmShifterOperand(SDValue N, SDValue &A,
87  SDValue &B, bool CheckProfitability = true);
88  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
89  SDValue &B, SDValue &C) {
90  // Don't apply the profitability check
91  return SelectRegShifterOperand(N, A, B, C, false);
92  }
93  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
94  SDValue &B) {
95  // Don't apply the profitability check
96  return SelectImmShifterOperand(N, A, B, false);
97  }
98 
99  bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
100 
101  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
102  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
103 
104  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
105  const ConstantSDNode *CN = cast<ConstantSDNode>(N);
106  Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
107  Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
108  return true;
109  }
110 
111  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
112  SDValue &Offset, SDValue &Opc);
113  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
114  SDValue &Offset, SDValue &Opc);
115  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
116  SDValue &Offset, SDValue &Opc);
117  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
118  bool SelectAddrMode3(SDValue N, SDValue &Base,
119  SDValue &Offset, SDValue &Opc);
120  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
121  SDValue &Offset, SDValue &Opc);
122  bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset, bool FP16);
123  bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
124  bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
125  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
126  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
127 
128  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
129 
130  // Thumb Addressing Modes:
131  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
132  bool SelectThumbAddrModeRRSext(SDValue N, SDValue &Base, SDValue &Offset);
133  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
134  SDValue &OffImm);
135  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
136  SDValue &OffImm);
137  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
138  SDValue &OffImm);
139  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
140  SDValue &OffImm);
141  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
142 
143  // Thumb 2 Addressing Modes:
144  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
145  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
146  SDValue &OffImm);
147  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
148  SDValue &OffImm);
149  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
150  SDValue &OffReg, SDValue &ShImm);
151  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
152 
153  inline bool is_so_imm(unsigned Imm) const {
154  return ARM_AM::getSOImmVal(Imm) != -1;
155  }
156 
157  inline bool is_so_imm_not(unsigned Imm) const {
158  return ARM_AM::getSOImmVal(~Imm) != -1;
159  }
160 
161  inline bool is_t2_so_imm(unsigned Imm) const {
162  return ARM_AM::getT2SOImmVal(Imm) != -1;
163  }
164 
165  inline bool is_t2_so_imm_not(unsigned Imm) const {
166  return ARM_AM::getT2SOImmVal(~Imm) != -1;
167  }
168 
169  // Include the pieces autogenerated from the target description.
170 #include "ARMGenDAGISel.inc"
171 
172 private:
173  void transferMemOperands(SDNode *Src, SDNode *Dst);
174 
175  /// Indexed (pre/post inc/dec) load matching code for ARM.
176  bool tryARMIndexedLoad(SDNode *N);
177  bool tryT1IndexedLoad(SDNode *N);
178  bool tryT2IndexedLoad(SDNode *N);
179 
180  /// SelectVLD - Select NEON load intrinsics. NumVecs should be
181  /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
182  /// loads of D registers and even subregs and odd subregs of Q registers.
183  /// For NumVecs <= 2, QOpcodes1 is not used.
184  void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
185  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
186  const uint16_t *QOpcodes1);
187 
188  /// SelectVST - Select NEON store intrinsics. NumVecs should
189  /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
190  /// stores of D registers and even subregs and odd subregs of Q registers.
191  /// For NumVecs <= 2, QOpcodes1 is not used.
192  void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
193  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
194  const uint16_t *QOpcodes1);
195 
196  /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
197  /// be 2, 3 or 4. The opcode arrays specify the instructions used for
198  /// load/store of D registers and Q registers.
199  void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
200  unsigned NumVecs, const uint16_t *DOpcodes,
201  const uint16_t *QOpcodes);
202 
203  /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
204  /// should be 1, 2, 3 or 4. The opcode array specifies the instructions used
205  /// for loading D registers.
206  void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
207  unsigned NumVecs, const uint16_t *DOpcodes,
208  const uint16_t *QOpcodes0 = nullptr,
209  const uint16_t *QOpcodes1 = nullptr);
210 
211  /// Try to select SBFX/UBFX instructions for ARM.
212  bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
213 
214  // Select special operations if node forms integer ABS pattern
215  bool tryABSOp(SDNode *N);
216 
217  bool tryReadRegister(SDNode *N);
218  bool tryWriteRegister(SDNode *N);
219 
220  bool tryInlineAsm(SDNode *N);
221 
222  void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
223 
224  void SelectCMP_SWAP(SDNode *N);
225 
226  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
227  /// inline asm expressions.
228  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
229  std::vector<SDValue> &OutOps) override;
230 
231  // Form pairs of consecutive R, S, D, or Q registers.
233  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
234  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
235  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
236 
237  // Form sequences of 4 consecutive S, D, or Q registers.
238  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
239  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
240  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
241 
242  // Get the alignment operand for a NEON VLD or VST instruction.
243  SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
244  bool is64BitVector);
245 
246  /// Returns the number of instructions required to materialize the given
247  /// constant in a register, or 3 if a literal pool load is needed.
248  unsigned ConstantMaterializationCost(unsigned Val) const;
249 
250  /// Checks if N is a multiplication by a constant where we can extract out a
251  /// power of two from the constant so that it can be used in a shift, but only
252  /// if it simplifies the materialization of the constant. Returns true if it
253  /// is, and assigns to PowerOfTwo the power of two that should be extracted
254  /// out and to NewMulConst the new constant to be multiplied by.
255  bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
256  unsigned &PowerOfTwo, SDValue &NewMulConst) const;
257 
258  /// Replace N with M in CurDAG, in a way that also ensures that M gets
259  /// selected when N would have been selected.
260  void replaceDAGValue(const SDValue &N, SDValue M);
261 };
262 }
263 
264 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
265 /// operand. If so Imm will receive the 32-bit value.
266 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
267  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
268  Imm = cast<ConstantSDNode>(N)->getZExtValue();
269  return true;
270  }
271  return false;
272 }
273 
274 // isInt32Immediate - This method tests to see if a constant operand.
275 // If so Imm will receive the 32 bit value.
276 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
277  return isInt32Immediate(N.getNode(), Imm);
278 }
279 
280 // isOpcWithIntImmediate - This method tests to see if the node is a specific
281 // opcode and that it has a immediate integer right operand.
282 // If so Imm will receive the 32 bit value.
283 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
284  return N->getOpcode() == Opc &&
285  isInt32Immediate(N->getOperand(1).getNode(), Imm);
286 }
287 
288 /// Check whether a particular node is a constant value representable as
289 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
290 ///
291 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
292 static bool isScaledConstantInRange(SDValue Node, int Scale,
293  int RangeMin, int RangeMax,
294  int &ScaledConstant) {
295  assert(Scale > 0 && "Invalid scale!");
296 
297  // Check that this is a constant.
298  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
299  if (!C)
300  return false;
301 
302  ScaledConstant = (int) C->getZExtValue();
303  if ((ScaledConstant % Scale) != 0)
304  return false;
305 
306  ScaledConstant /= Scale;
307  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
308 }
309 
310 void ARMDAGToDAGISel::PreprocessISelDAG() {
311  if (!Subtarget->hasV6T2Ops())
312  return;
313 
314  bool isThumb2 = Subtarget->isThumb();
315  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
316  E = CurDAG->allnodes_end(); I != E; ) {
317  SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
318 
319  if (N->getOpcode() != ISD::ADD)
320  continue;
321 
322  // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
323  // leading zeros, followed by consecutive set bits, followed by 1 or 2
324  // trailing zeros, e.g. 1020.
325  // Transform the expression to
326  // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
327  // of trailing zeros of c2. The left shift would be folded as an shifter
328  // operand of 'add' and the 'and' and 'srl' would become a bits extraction
329  // node (UBFX).
330 
331  SDValue N0 = N->getOperand(0);
332  SDValue N1 = N->getOperand(1);
333  unsigned And_imm = 0;
334  if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
335  if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
336  std::swap(N0, N1);
337  }
338  if (!And_imm)
339  continue;
340 
341  // Check if the AND mask is an immediate of the form: 000.....1111111100
342  unsigned TZ = countTrailingZeros(And_imm);
343  if (TZ != 1 && TZ != 2)
344  // Be conservative here. Shifter operands aren't always free. e.g. On
345  // Swift, left shifter operand of 1 / 2 for free but others are not.
346  // e.g.
347  // ubfx r3, r1, #16, #8
348  // ldr.w r3, [r0, r3, lsl #2]
349  // vs.
350  // mov.w r9, #1020
351  // and.w r2, r9, r1, lsr #14
352  // ldr r2, [r0, r2]
353  continue;
354  And_imm >>= TZ;
355  if (And_imm & (And_imm + 1))
356  continue;
357 
358  // Look for (and (srl X, c1), c2).
359  SDValue Srl = N1.getOperand(0);
360  unsigned Srl_imm = 0;
361  if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
362  (Srl_imm <= 2))
363  continue;
364 
365  // Make sure first operand is not a shifter operand which would prevent
366  // folding of the left shift.
367  SDValue CPTmp0;
368  SDValue CPTmp1;
369  SDValue CPTmp2;
370  if (isThumb2) {
371  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
372  continue;
373  } else {
374  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
375  SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
376  continue;
377  }
378 
379  // Now make the transformation.
380  Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
381  Srl.getOperand(0),
382  CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
383  MVT::i32));
384  N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
385  Srl,
386  CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
387  N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
388  N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
389  CurDAG->UpdateNodeOperands(N, N0, N1);
390  }
391 }
392 
393 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
394 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
395 /// least on current ARM implementations) which should be avoidded.
396 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
397  if (OptLevel == CodeGenOpt::None)
398  return true;
399 
400  if (!Subtarget->hasVMLxHazards())
401  return true;
402 
403  if (!N->hasOneUse())
404  return false;
405 
406  SDNode *Use = *N->use_begin();
407  if (Use->getOpcode() == ISD::CopyToReg)
408  return true;
409  if (Use->isMachineOpcode()) {
410  const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
411  CurDAG->getSubtarget().getInstrInfo());
412 
413  const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
414  if (MCID.mayStore())
415  return true;
416  unsigned Opcode = MCID.getOpcode();
417  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
418  return true;
419  // vmlx feeding into another vmlx. We actually want to unfold
420  // the use later in the MLxExpansion pass. e.g.
421  // vmla
422  // vmla (stall 8 cycles)
423  //
424  // vmul (5 cycles)
425  // vadd (5 cycles)
426  // vmla
427  // This adds up to about 18 - 19 cycles.
428  //
429  // vmla
430  // vmul (stall 4 cycles)
431  // vadd adds up to about 14 cycles.
432  return TII->isFpMLxInstruction(Opcode);
433  }
434 
435  return false;
436 }
437 
438 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
439  ARM_AM::ShiftOpc ShOpcVal,
440  unsigned ShAmt) {
441  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
442  return true;
443  if (Shift.hasOneUse())
444  return true;
445  // R << 2 is free.
446  return ShOpcVal == ARM_AM::lsl &&
447  (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
448 }
449 
450 unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
451  if (Subtarget->isThumb()) {
452  if (Val <= 255) return 1; // MOV
453  if (Subtarget->hasV6T2Ops() &&
454  (Val <= 0xffff || // MOV
455  ARM_AM::getT2SOImmVal(Val) != -1 || // MOVW
456  ARM_AM::getT2SOImmVal(~Val) != -1)) // MVN
457  return 1;
458  if (Val <= 510) return 2; // MOV + ADDi8
459  if (~Val <= 255) return 2; // MOV + MVN
460  if (ARM_AM::isThumbImmShiftedVal(Val)) return 2; // MOV + LSL
461  } else {
462  if (ARM_AM::getSOImmVal(Val) != -1) return 1; // MOV
463  if (ARM_AM::getSOImmVal(~Val) != -1) return 1; // MVN
464  if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
465  if (ARM_AM::isSOImmTwoPartVal(Val)) return 2; // two instrs
466  }
467  if (Subtarget->useMovt()) return 2; // MOVW + MOVT
468  return 3; // Literal pool load
469 }
470 
471 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
472  unsigned MaxShift,
473  unsigned &PowerOfTwo,
474  SDValue &NewMulConst) const {
475  assert(N.getOpcode() == ISD::MUL);
476  assert(MaxShift > 0);
477 
478  // If the multiply is used in more than one place then changing the constant
479  // will make other uses incorrect, so don't.
480  if (!N.hasOneUse()) return false;
481  // Check if the multiply is by a constant
483  if (!MulConst) return false;
484  // If the constant is used in more than one place then modifying it will mean
485  // we need to materialize two constants instead of one, which is a bad idea.
486  if (!MulConst->hasOneUse()) return false;
487  unsigned MulConstVal = MulConst->getZExtValue();
488  if (MulConstVal == 0) return false;
489 
490  // Find the largest power of 2 that MulConstVal is a multiple of
491  PowerOfTwo = MaxShift;
492  while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
493  --PowerOfTwo;
494  if (PowerOfTwo == 0) return false;
495  }
496 
497  // Only optimise if the new cost is better
498  unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
499  NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
500  unsigned OldCost = ConstantMaterializationCost(MulConstVal);
501  unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
502  return NewCost < OldCost;
503 }
504 
505 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
506  CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
507  ReplaceUses(N, M);
508 }
509 
510 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
511  SDValue &BaseReg,
512  SDValue &Opc,
513  bool CheckProfitability) {
514  if (DisableShifterOp)
515  return false;
516 
517  // If N is a multiply-by-constant and it's profitable to extract a shift and
518  // use it in a shifted operand do so.
519  if (N.getOpcode() == ISD::MUL) {
520  unsigned PowerOfTwo = 0;
521  SDValue NewMulConst;
522  if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
523  HandleSDNode Handle(N);
524  SDLoc Loc(N);
525  replaceDAGValue(N.getOperand(1), NewMulConst);
526  BaseReg = Handle.getValue();
527  Opc = CurDAG->getTargetConstant(
528  ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
529  return true;
530  }
531  }
532 
534 
535  // Don't match base register only case. That is matched to a separate
536  // lower complexity pattern with explicit register operand.
537  if (ShOpcVal == ARM_AM::no_shift) return false;
538 
539  BaseReg = N.getOperand(0);
540  unsigned ShImmVal = 0;
542  if (!RHS) return false;
543  ShImmVal = RHS->getZExtValue() & 31;
544  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
545  SDLoc(N), MVT::i32);
546  return true;
547 }
548 
549 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
550  SDValue &BaseReg,
551  SDValue &ShReg,
552  SDValue &Opc,
553  bool CheckProfitability) {
554  if (DisableShifterOp)
555  return false;
556 
558 
559  // Don't match base register only case. That is matched to a separate
560  // lower complexity pattern with explicit register operand.
561  if (ShOpcVal == ARM_AM::no_shift) return false;
562 
563  BaseReg = N.getOperand(0);
564  unsigned ShImmVal = 0;
566  if (RHS) return false;
567 
568  ShReg = N.getOperand(1);
569  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
570  return false;
571  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
572  SDLoc(N), MVT::i32);
573  return true;
574 }
575 
576 // Determine whether an ISD::OR's operands are suitable to turn the operation
577 // into an addition, which often has more compact encodings.
578 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
579  assert(Parent->getOpcode() == ISD::OR && "unexpected parent");
580  Out = N;
581  return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
582 }
583 
584 
585 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
586  SDValue &Base,
587  SDValue &OffImm) {
588  // Match simple R + imm12 operands.
589 
590  // Base only.
591  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
592  !CurDAG->isBaseWithConstantOffset(N)) {
593  if (N.getOpcode() == ISD::FrameIndex) {
594  // Match frame index.
595  int FI = cast<FrameIndexSDNode>(N)->getIndex();
596  Base = CurDAG->getTargetFrameIndex(
597  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
598  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
599  return true;
600  }
601 
602  if (N.getOpcode() == ARMISD::Wrapper &&
606  Base = N.getOperand(0);
607  } else
608  Base = N;
609  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
610  return true;
611  }
612 
613  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
614  int RHSC = (int)RHS->getSExtValue();
615  if (N.getOpcode() == ISD::SUB)
616  RHSC = -RHSC;
617 
618  if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
619  Base = N.getOperand(0);
620  if (Base.getOpcode() == ISD::FrameIndex) {
621  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
622  Base = CurDAG->getTargetFrameIndex(
623  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
624  }
625  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
626  return true;
627  }
628  }
629 
630  // Base only.
631  Base = N;
632  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
633  return true;
634 }
635 
636 
637 
638 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
639  SDValue &Opc) {
640  if (N.getOpcode() == ISD::MUL &&
641  ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
642  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
643  // X * [3,5,9] -> X + X * [2,4,8] etc.
644  int RHSC = (int)RHS->getZExtValue();
645  if (RHSC & 1) {
646  RHSC = RHSC & ~1;
647  ARM_AM::AddrOpc AddSub = ARM_AM::add;
648  if (RHSC < 0) {
649  AddSub = ARM_AM::sub;
650  RHSC = - RHSC;
651  }
652  if (isPowerOf2_32(RHSC)) {
653  unsigned ShAmt = Log2_32(RHSC);
654  Base = Offset = N.getOperand(0);
655  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
656  ARM_AM::lsl),
657  SDLoc(N), MVT::i32);
658  return true;
659  }
660  }
661  }
662  }
663 
664  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
665  // ISD::OR that is equivalent to an ISD::ADD.
666  !CurDAG->isBaseWithConstantOffset(N))
667  return false;
668 
669  // Leave simple R +/- imm12 operands for LDRi12
670  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
671  int RHSC;
672  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
673  -0x1000+1, 0x1000, RHSC)) // 12 bits.
674  return false;
675  }
676 
677  // Otherwise this is R +/- [possibly shifted] R.
679  ARM_AM::ShiftOpc ShOpcVal =
681  unsigned ShAmt = 0;
682 
683  Base = N.getOperand(0);
684  Offset = N.getOperand(1);
685 
686  if (ShOpcVal != ARM_AM::no_shift) {
687  // Check to see if the RHS of the shift is a constant, if not, we can't fold
688  // it.
689  if (ConstantSDNode *Sh =
690  dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
691  ShAmt = Sh->getZExtValue();
692  if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
693  Offset = N.getOperand(1).getOperand(0);
694  else {
695  ShAmt = 0;
696  ShOpcVal = ARM_AM::no_shift;
697  }
698  } else {
699  ShOpcVal = ARM_AM::no_shift;
700  }
701  }
702 
703  // Try matching (R shl C) + (R).
704  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
705  !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
706  N.getOperand(0).hasOneUse())) {
708  if (ShOpcVal != ARM_AM::no_shift) {
709  // Check to see if the RHS of the shift is a constant, if not, we can't
710  // fold it.
711  if (ConstantSDNode *Sh =
712  dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
713  ShAmt = Sh->getZExtValue();
714  if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
715  Offset = N.getOperand(0).getOperand(0);
716  Base = N.getOperand(1);
717  } else {
718  ShAmt = 0;
719  ShOpcVal = ARM_AM::no_shift;
720  }
721  } else {
722  ShOpcVal = ARM_AM::no_shift;
723  }
724  }
725  }
726 
727  // If Offset is a multiply-by-constant and it's profitable to extract a shift
728  // and use it in a shifted operand do so.
729  if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
730  unsigned PowerOfTwo = 0;
731  SDValue NewMulConst;
732  if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
733  HandleSDNode Handle(Offset);
734  replaceDAGValue(Offset.getOperand(1), NewMulConst);
735  Offset = Handle.getValue();
736  ShAmt = PowerOfTwo;
737  ShOpcVal = ARM_AM::lsl;
738  }
739  }
740 
741  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
742  SDLoc(N), MVT::i32);
743  return true;
744 }
745 
746 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
747  SDValue &Offset, SDValue &Opc) {
748  unsigned Opcode = Op->getOpcode();
749  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
750  ? cast<LoadSDNode>(Op)->getAddressingMode()
751  : cast<StoreSDNode>(Op)->getAddressingMode();
752  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
754  int Val;
755  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
756  return false;
757 
758  Offset = N;
760  unsigned ShAmt = 0;
761  if (ShOpcVal != ARM_AM::no_shift) {
762  // Check to see if the RHS of the shift is a constant, if not, we can't fold
763  // it.
764  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
765  ShAmt = Sh->getZExtValue();
766  if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
767  Offset = N.getOperand(0);
768  else {
769  ShAmt = 0;
770  ShOpcVal = ARM_AM::no_shift;
771  }
772  } else {
773  ShOpcVal = ARM_AM::no_shift;
774  }
775  }
776 
777  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
778  SDLoc(N), MVT::i32);
779  return true;
780 }
781 
782 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
783  SDValue &Offset, SDValue &Opc) {
784  unsigned Opcode = Op->getOpcode();
785  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
786  ? cast<LoadSDNode>(Op)->getAddressingMode()
787  : cast<StoreSDNode>(Op)->getAddressingMode();
788  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
790  int Val;
791  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
792  if (AddSub == ARM_AM::sub) Val *= -1;
793  Offset = CurDAG->getRegister(0, MVT::i32);
794  Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
795  return true;
796  }
797 
798  return false;
799 }
800 
801 
802 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
803  SDValue &Offset, SDValue &Opc) {
804  unsigned Opcode = Op->getOpcode();
805  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
806  ? cast<LoadSDNode>(Op)->getAddressingMode()
807  : cast<StoreSDNode>(Op)->getAddressingMode();
808  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
810  int Val;
811  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
812  Offset = CurDAG->getRegister(0, MVT::i32);
813  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
815  SDLoc(Op), MVT::i32);
816  return true;
817  }
818 
819  return false;
820 }
821 
822 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
823  Base = N;
824  return true;
825 }
826 
827 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
828  SDValue &Base, SDValue &Offset,
829  SDValue &Opc) {
830  if (N.getOpcode() == ISD::SUB) {
831  // X - C is canonicalize to X + -C, no need to handle it here.
832  Base = N.getOperand(0);
833  Offset = N.getOperand(1);
834  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
835  MVT::i32);
836  return true;
837  }
838 
839  if (!CurDAG->isBaseWithConstantOffset(N)) {
840  Base = N;
841  if (N.getOpcode() == ISD::FrameIndex) {
842  int FI = cast<FrameIndexSDNode>(N)->getIndex();
843  Base = CurDAG->getTargetFrameIndex(
844  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
845  }
846  Offset = CurDAG->getRegister(0, MVT::i32);
847  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
848  MVT::i32);
849  return true;
850  }
851 
852  // If the RHS is +/- imm8, fold into addr mode.
853  int RHSC;
854  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
855  -256 + 1, 256, RHSC)) { // 8 bits.
856  Base = N.getOperand(0);
857  if (Base.getOpcode() == ISD::FrameIndex) {
858  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
859  Base = CurDAG->getTargetFrameIndex(
860  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
861  }
862  Offset = CurDAG->getRegister(0, MVT::i32);
863 
864  ARM_AM::AddrOpc AddSub = ARM_AM::add;
865  if (RHSC < 0) {
866  AddSub = ARM_AM::sub;
867  RHSC = -RHSC;
868  }
869  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
870  MVT::i32);
871  return true;
872  }
873 
874  Base = N.getOperand(0);
875  Offset = N.getOperand(1);
876  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
877  MVT::i32);
878  return true;
879 }
880 
881 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
882  SDValue &Offset, SDValue &Opc) {
883  unsigned Opcode = Op->getOpcode();
884  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
885  ? cast<LoadSDNode>(Op)->getAddressingMode()
886  : cast<StoreSDNode>(Op)->getAddressingMode();
887  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
889  int Val;
890  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
891  Offset = CurDAG->getRegister(0, MVT::i32);
892  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
893  MVT::i32);
894  return true;
895  }
896 
897  Offset = N;
898  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
899  MVT::i32);
900  return true;
901 }
902 
903 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
904  bool FP16) {
905  if (!CurDAG->isBaseWithConstantOffset(N)) {
906  Base = N;
907  if (N.getOpcode() == ISD::FrameIndex) {
908  int FI = cast<FrameIndexSDNode>(N)->getIndex();
909  Base = CurDAG->getTargetFrameIndex(
910  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
911  } else if (N.getOpcode() == ARMISD::Wrapper &&
915  Base = N.getOperand(0);
916  }
917  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
918  SDLoc(N), MVT::i32);
919  return true;
920  }
921 
922  // If the RHS is +/- imm8, fold into addr mode.
923  int RHSC;
924  const int Scale = FP16 ? 2 : 4;
925 
926  if (isScaledConstantInRange(N.getOperand(1), Scale, -255, 256, RHSC)) {
927  Base = N.getOperand(0);
928  if (Base.getOpcode() == ISD::FrameIndex) {
929  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
930  Base = CurDAG->getTargetFrameIndex(
931  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
932  }
933 
934  ARM_AM::AddrOpc AddSub = ARM_AM::add;
935  if (RHSC < 0) {
936  AddSub = ARM_AM::sub;
937  RHSC = -RHSC;
938  }
939 
940  if (FP16)
941  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
942  SDLoc(N), MVT::i32);
943  else
944  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
945  SDLoc(N), MVT::i32);
946 
947  return true;
948  }
949 
950  Base = N;
951 
952  if (FP16)
953  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
954  SDLoc(N), MVT::i32);
955  else
956  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
957  SDLoc(N), MVT::i32);
958 
959  return true;
960 }
961 
962 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
963  SDValue &Base, SDValue &Offset) {
964  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ false);
965 }
966 
967 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
968  SDValue &Base, SDValue &Offset) {
969  return IsAddressingMode5(N, Base, Offset, /*FP16=*/ true);
970 }
971 
972 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
973  SDValue &Align) {
974  Addr = N;
975 
976  unsigned Alignment = 0;
977 
978  MemSDNode *MemN = cast<MemSDNode>(Parent);
979 
980  if (isa<LSBaseSDNode>(MemN) ||
981  ((MemN->getOpcode() == ARMISD::VST1_UPD ||
982  MemN->getOpcode() == ARMISD::VLD1_UPD) &&
983  MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
984  // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
985  // The maximum alignment is equal to the memory size being referenced.
986  unsigned MMOAlign = MemN->getAlignment();
987  unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
988  if (MMOAlign >= MemSize && MemSize > 1)
989  Alignment = MemSize;
990  } else {
991  // All other uses of addrmode6 are for intrinsics. For now just record
992  // the raw alignment value; it will be refined later based on the legal
993  // alignment operands for the intrinsic.
994  Alignment = MemN->getAlignment();
995  }
996 
997  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
998  return true;
999 }
1000 
1001 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1002  SDValue &Offset) {
1003  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1005  if (AM != ISD::POST_INC)
1006  return false;
1007  Offset = N;
1008  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1009  if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1010  Offset = CurDAG->getRegister(0, MVT::i32);
1011  }
1012  return true;
1013 }
1014 
1015 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1016  SDValue &Offset, SDValue &Label) {
1017  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1018  Offset = N.getOperand(0);
1019  SDValue N1 = N.getOperand(1);
1020  Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1021  SDLoc(N), MVT::i32);
1022  return true;
1023  }
1024 
1025  return false;
1026 }
1027 
1028 
1029 //===----------------------------------------------------------------------===//
1030 // Thumb Addressing Modes
1031 //===----------------------------------------------------------------------===//
1032 
1034  // Negative numbers are difficult to materialise in thumb1. If we are
1035  // selecting the add of a negative, instead try to select ri with a zero
1036  // offset, so create the add node directly which will become a sub.
1037  if (N.getOpcode() != ISD::ADD)
1038  return false;
1039 
1040  // Look for an imm which is not legal for ld/st, but is legal for sub.
1041  if (auto C = dyn_cast<ConstantSDNode>(N.getOperand(1)))
1042  return C->getSExtValue() < 0 && C->getSExtValue() >= -255;
1043 
1044  return false;
1045 }
1046 
1047 bool ARMDAGToDAGISel::SelectThumbAddrModeRRSext(SDValue N, SDValue &Base,
1048  SDValue &Offset) {
1049  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1051  if (!NC || !NC->isNullValue())
1052  return false;
1053 
1054  Base = Offset = N;
1055  return true;
1056  }
1057 
1058  Base = N.getOperand(0);
1059  Offset = N.getOperand(1);
1060  return true;
1061 }
1062 
1063 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N, SDValue &Base,
1064  SDValue &Offset) {
1065  if (shouldUseZeroOffsetLdSt(N))
1066  return false; // Select ri instead
1067  return SelectThumbAddrModeRRSext(N, Base, Offset);
1068 }
1069 
1070 bool
1071 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1072  SDValue &Base, SDValue &OffImm) {
1073  if (shouldUseZeroOffsetLdSt(N)) {
1074  Base = N;
1075  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1076  return true;
1077  }
1078 
1079  if (!CurDAG->isBaseWithConstantOffset(N)) {
1080  if (N.getOpcode() == ISD::ADD) {
1081  return false; // We want to select register offset instead
1082  } else if (N.getOpcode() == ARMISD::Wrapper &&
1087  Base = N.getOperand(0);
1088  } else {
1089  Base = N;
1090  }
1091 
1092  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1093  return true;
1094  }
1095 
1096  // If the RHS is + imm5 * scale, fold into addr mode.
1097  int RHSC;
1098  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1099  Base = N.getOperand(0);
1100  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1101  return true;
1102  }
1103 
1104  // Offset is too large, so use register offset instead.
1105  return false;
1106 }
1107 
1108 bool
1109 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1110  SDValue &OffImm) {
1111  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1112 }
1113 
1114 bool
1115 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1116  SDValue &OffImm) {
1117  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1118 }
1119 
1120 bool
1121 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1122  SDValue &OffImm) {
1123  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1124 }
1125 
1126 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1127  SDValue &Base, SDValue &OffImm) {
1128  if (N.getOpcode() == ISD::FrameIndex) {
1129  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1130  // Only multiples of 4 are allowed for the offset, so the frame object
1131  // alignment must be at least 4.
1132  MachineFrameInfo &MFI = MF->getFrameInfo();
1133  if (MFI.getObjectAlignment(FI) < 4)
1134  MFI.setObjectAlignment(FI, 4);
1135  Base = CurDAG->getTargetFrameIndex(
1136  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1137  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1138  return true;
1139  }
1140 
1141  if (!CurDAG->isBaseWithConstantOffset(N))
1142  return false;
1143 
1144  if (N.getOperand(0).getOpcode() == ISD::FrameIndex) {
1145  // If the RHS is + imm8 * scale, fold into addr mode.
1146  int RHSC;
1147  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1148  Base = N.getOperand(0);
1149  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1150  // For LHS+RHS to result in an offset that's a multiple of 4 the object
1151  // indexed by the LHS must be 4-byte aligned.
1152  MachineFrameInfo &MFI = MF->getFrameInfo();
1153  if (MFI.getObjectAlignment(FI) < 4)
1154  MFI.setObjectAlignment(FI, 4);
1155  Base = CurDAG->getTargetFrameIndex(
1156  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1157  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1158  return true;
1159  }
1160  }
1161 
1162  return false;
1163 }
1164 
1165 
1166 //===----------------------------------------------------------------------===//
1167 // Thumb 2 Addressing Modes
1168 //===----------------------------------------------------------------------===//
1169 
1170 
1171 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1172  SDValue &Base, SDValue &OffImm) {
1173  // Match simple R + imm12 operands.
1174 
1175  // Base only.
1176  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1177  !CurDAG->isBaseWithConstantOffset(N)) {
1178  if (N.getOpcode() == ISD::FrameIndex) {
1179  // Match frame index.
1180  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1181  Base = CurDAG->getTargetFrameIndex(
1182  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1183  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1184  return true;
1185  }
1186 
1187  if (N.getOpcode() == ARMISD::Wrapper &&
1191  Base = N.getOperand(0);
1192  if (Base.getOpcode() == ISD::TargetConstantPool)
1193  return false; // We want to select t2LDRpci instead.
1194  } else
1195  Base = N;
1196  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1197  return true;
1198  }
1199 
1200  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1201  if (SelectT2AddrModeImm8(N, Base, OffImm))
1202  // Let t2LDRi8 handle (R - imm8).
1203  return false;
1204 
1205  int RHSC = (int)RHS->getZExtValue();
1206  if (N.getOpcode() == ISD::SUB)
1207  RHSC = -RHSC;
1208 
1209  if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1210  Base = N.getOperand(0);
1211  if (Base.getOpcode() == ISD::FrameIndex) {
1212  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1213  Base = CurDAG->getTargetFrameIndex(
1214  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1215  }
1216  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1217  return true;
1218  }
1219  }
1220 
1221  // Base only.
1222  Base = N;
1223  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1224  return true;
1225 }
1226 
1227 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1228  SDValue &Base, SDValue &OffImm) {
1229  // Match simple R - imm8 operands.
1230  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1231  !CurDAG->isBaseWithConstantOffset(N))
1232  return false;
1233 
1234  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1235  int RHSC = (int)RHS->getSExtValue();
1236  if (N.getOpcode() == ISD::SUB)
1237  RHSC = -RHSC;
1238 
1239  if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1240  Base = N.getOperand(0);
1241  if (Base.getOpcode() == ISD::FrameIndex) {
1242  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1243  Base = CurDAG->getTargetFrameIndex(
1244  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1245  }
1246  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1247  return true;
1248  }
1249  }
1250 
1251  return false;
1252 }
1253 
1254 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1255  SDValue &OffImm){
1256  unsigned Opcode = Op->getOpcode();
1257  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1258  ? cast<LoadSDNode>(Op)->getAddressingMode()
1259  : cast<StoreSDNode>(Op)->getAddressingMode();
1260  int RHSC;
1261  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1262  OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1263  ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
1264  : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1265  return true;
1266  }
1267 
1268  return false;
1269 }
1270 
1271 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1272  SDValue &Base,
1273  SDValue &OffReg, SDValue &ShImm) {
1274  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1275  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1276  return false;
1277 
1278  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1279  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1280  int RHSC = (int)RHS->getZExtValue();
1281  if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1282  return false;
1283  else if (RHSC < 0 && RHSC >= -255) // 8 bits
1284  return false;
1285  }
1286 
1287  // Look for (R + R) or (R + (R << [1,2,3])).
1288  unsigned ShAmt = 0;
1289  Base = N.getOperand(0);
1290  OffReg = N.getOperand(1);
1291 
1292  // Swap if it is ((R << c) + R).
1294  if (ShOpcVal != ARM_AM::lsl) {
1295  ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1296  if (ShOpcVal == ARM_AM::lsl)
1297  std::swap(Base, OffReg);
1298  }
1299 
1300  if (ShOpcVal == ARM_AM::lsl) {
1301  // Check to see if the RHS of the shift is a constant, if not, we can't fold
1302  // it.
1303  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1304  ShAmt = Sh->getZExtValue();
1305  if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1306  OffReg = OffReg.getOperand(0);
1307  else {
1308  ShAmt = 0;
1309  }
1310  }
1311  }
1312 
1313  // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1314  // and use it in a shifted operand do so.
1315  if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
1316  unsigned PowerOfTwo = 0;
1317  SDValue NewMulConst;
1318  if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1319  HandleSDNode Handle(OffReg);
1320  replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1321  OffReg = Handle.getValue();
1322  ShAmt = PowerOfTwo;
1323  }
1324  }
1325 
1326  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1327 
1328  return true;
1329 }
1330 
1331 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1332  SDValue &OffImm) {
1333  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1334  // instructions.
1335  Base = N;
1336  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1337 
1338  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1339  return true;
1340 
1342  if (!RHS)
1343  return true;
1344 
1345  uint32_t RHSC = (int)RHS->getZExtValue();
1346  if (RHSC > 1020 || RHSC % 4 != 0)
1347  return true;
1348 
1349  Base = N.getOperand(0);
1350  if (Base.getOpcode() == ISD::FrameIndex) {
1351  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1352  Base = CurDAG->getTargetFrameIndex(
1353  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1354  }
1355 
1356  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1357  return true;
1358 }
1359 
1360 //===--------------------------------------------------------------------===//
1361 
1362 /// getAL - Returns a ARMCC::AL immediate node.
1363 static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1364  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1365 }
1366 
1367 void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1368  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1369  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
1370 }
1371 
1372 bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1373  LoadSDNode *LD = cast<LoadSDNode>(N);
1375  if (AM == ISD::UNINDEXED)
1376  return false;
1377 
1378  EVT LoadedVT = LD->getMemoryVT();
1379  SDValue Offset, AMOpc;
1380  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1381  unsigned Opcode = 0;
1382  bool Match = false;
1383  if (LoadedVT == MVT::i32 && isPre &&
1384  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1385  Opcode = ARM::LDR_PRE_IMM;
1386  Match = true;
1387  } else if (LoadedVT == MVT::i32 && !isPre &&
1388  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1389  Opcode = ARM::LDR_POST_IMM;
1390  Match = true;
1391  } else if (LoadedVT == MVT::i32 &&
1392  SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1393  Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1394  Match = true;
1395 
1396  } else if (LoadedVT == MVT::i16 &&
1397  SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1398  Match = true;
1399  Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1400  ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1401  : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1402  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1403  if (LD->getExtensionType() == ISD::SEXTLOAD) {
1404  if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1405  Match = true;
1406  Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1407  }
1408  } else {
1409  if (isPre &&
1410  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1411  Match = true;
1412  Opcode = ARM::LDRB_PRE_IMM;
1413  } else if (!isPre &&
1414  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1415  Match = true;
1416  Opcode = ARM::LDRB_POST_IMM;
1417  } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1418  Match = true;
1419  Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1420  }
1421  }
1422  }
1423 
1424  if (Match) {
1425  if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1426  SDValue Chain = LD->getChain();
1427  SDValue Base = LD->getBasePtr();
1428  SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1429  CurDAG->getRegister(0, MVT::i32), Chain };
1430  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1431  MVT::Other, Ops);
1432  transferMemOperands(N, New);
1433  ReplaceNode(N, New);
1434  return true;
1435  } else {
1436  SDValue Chain = LD->getChain();
1437  SDValue Base = LD->getBasePtr();
1438  SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1439  CurDAG->getRegister(0, MVT::i32), Chain };
1440  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1441  MVT::Other, Ops);
1442  transferMemOperands(N, New);
1443  ReplaceNode(N, New);
1444  return true;
1445  }
1446  }
1447 
1448  return false;
1449 }
1450 
1451 bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1452  LoadSDNode *LD = cast<LoadSDNode>(N);
1453  EVT LoadedVT = LD->getMemoryVT();
1455  if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
1456  LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
1457  return false;
1458 
1459  auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1460  if (!COffs || COffs->getZExtValue() != 4)
1461  return false;
1462 
1463  // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1464  // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1465  // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1466  // ISel.
1467  SDValue Chain = LD->getChain();
1468  SDValue Base = LD->getBasePtr();
1469  SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1470  CurDAG->getRegister(0, MVT::i32), Chain };
1471  SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1472  MVT::i32, MVT::Other, Ops);
1473  transferMemOperands(N, New);
1474  ReplaceNode(N, New);
1475  return true;
1476 }
1477 
1478 bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1479  LoadSDNode *LD = cast<LoadSDNode>(N);
1481  if (AM == ISD::UNINDEXED)
1482  return false;
1483 
1484  EVT LoadedVT = LD->getMemoryVT();
1485  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1486  SDValue Offset;
1487  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1488  unsigned Opcode = 0;
1489  bool Match = false;
1490  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1491  switch (LoadedVT.getSimpleVT().SimpleTy) {
1492  case MVT::i32:
1493  Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1494  break;
1495  case MVT::i16:
1496  if (isSExtLd)
1497  Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1498  else
1499  Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1500  break;
1501  case MVT::i8:
1502  case MVT::i1:
1503  if (isSExtLd)
1504  Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1505  else
1506  Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1507  break;
1508  default:
1509  return false;
1510  }
1511  Match = true;
1512  }
1513 
1514  if (Match) {
1515  SDValue Chain = LD->getChain();
1516  SDValue Base = LD->getBasePtr();
1517  SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1518  CurDAG->getRegister(0, MVT::i32), Chain };
1519  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1520  MVT::Other, Ops);
1521  transferMemOperands(N, New);
1522  ReplaceNode(N, New);
1523  return true;
1524  }
1525 
1526  return false;
1527 }
1528 
1529 /// Form a GPRPair pseudo register from a pair of GPR regs.
1531  SDLoc dl(V0.getNode());
1532  SDValue RegClass =
1533  CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1534  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1535  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1536  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1537  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1538 }
1539 
1540 /// Form a D register from a pair of S registers.
1541 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1542  SDLoc dl(V0.getNode());
1543  SDValue RegClass =
1544  CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1545  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1546  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1547  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1548  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1549 }
1550 
1551 /// Form a quad register from a pair of D registers.
1552 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1553  SDLoc dl(V0.getNode());
1554  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1555  MVT::i32);
1556  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1557  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1558  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1559  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1560 }
1561 
1562 /// Form 4 consecutive D registers from a pair of Q registers.
1563 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1564  SDLoc dl(V0.getNode());
1565  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1566  MVT::i32);
1567  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1568  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1569  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1570  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1571 }
1572 
1573 /// Form 4 consecutive S registers.
1574 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1575  SDValue V2, SDValue V3) {
1576  SDLoc dl(V0.getNode());
1577  SDValue RegClass =
1578  CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1579  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1580  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1581  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1582  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1583  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1584  V2, SubReg2, V3, SubReg3 };
1585  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1586 }
1587 
1588 /// Form 4 consecutive D registers.
1589 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1590  SDValue V2, SDValue V3) {
1591  SDLoc dl(V0.getNode());
1592  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1593  MVT::i32);
1594  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1595  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1596  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1597  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1598  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1599  V2, SubReg2, V3, SubReg3 };
1600  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1601 }
1602 
1603 /// Form 4 consecutive Q registers.
1604 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1605  SDValue V2, SDValue V3) {
1606  SDLoc dl(V0.getNode());
1607  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1608  MVT::i32);
1609  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1610  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1611  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1612  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1613  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1614  V2, SubReg2, V3, SubReg3 };
1615  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1616 }
1617 
1618 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1619 /// of a NEON VLD or VST instruction. The supported values depend on the
1620 /// number of registers being loaded.
1621 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1622  unsigned NumVecs, bool is64BitVector) {
1623  unsigned NumRegs = NumVecs;
1624  if (!is64BitVector && NumVecs < 3)
1625  NumRegs *= 2;
1626 
1627  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1628  if (Alignment >= 32 && NumRegs == 4)
1629  Alignment = 32;
1630  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1631  Alignment = 16;
1632  else if (Alignment >= 8)
1633  Alignment = 8;
1634  else
1635  Alignment = 0;
1636 
1637  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1638 }
1639 
1640 static bool isVLDfixed(unsigned Opc)
1641 {
1642  switch (Opc) {
1643  default: return false;
1644  case ARM::VLD1d8wb_fixed : return true;
1645  case ARM::VLD1d16wb_fixed : return true;
1646  case ARM::VLD1d64Qwb_fixed : return true;
1647  case ARM::VLD1d32wb_fixed : return true;
1648  case ARM::VLD1d64wb_fixed : return true;
1649  case ARM::VLD1d64TPseudoWB_fixed : return true;
1650  case ARM::VLD1d64QPseudoWB_fixed : return true;
1651  case ARM::VLD1q8wb_fixed : return true;
1652  case ARM::VLD1q16wb_fixed : return true;
1653  case ARM::VLD1q32wb_fixed : return true;
1654  case ARM::VLD1q64wb_fixed : return true;
1655  case ARM::VLD1DUPd8wb_fixed : return true;
1656  case ARM::VLD1DUPd16wb_fixed : return true;
1657  case ARM::VLD1DUPd32wb_fixed : return true;
1658  case ARM::VLD1DUPq8wb_fixed : return true;
1659  case ARM::VLD1DUPq16wb_fixed : return true;
1660  case ARM::VLD1DUPq32wb_fixed : return true;
1661  case ARM::VLD2d8wb_fixed : return true;
1662  case ARM::VLD2d16wb_fixed : return true;
1663  case ARM::VLD2d32wb_fixed : return true;
1664  case ARM::VLD2q8PseudoWB_fixed : return true;
1665  case ARM::VLD2q16PseudoWB_fixed : return true;
1666  case ARM::VLD2q32PseudoWB_fixed : return true;
1667  case ARM::VLD2DUPd8wb_fixed : return true;
1668  case ARM::VLD2DUPd16wb_fixed : return true;
1669  case ARM::VLD2DUPd32wb_fixed : return true;
1670  }
1671 }
1672 
1673 static bool isVSTfixed(unsigned Opc)
1674 {
1675  switch (Opc) {
1676  default: return false;
1677  case ARM::VST1d8wb_fixed : return true;
1678  case ARM::VST1d16wb_fixed : return true;
1679  case ARM::VST1d32wb_fixed : return true;
1680  case ARM::VST1d64wb_fixed : return true;
1681  case ARM::VST1q8wb_fixed : return true;
1682  case ARM::VST1q16wb_fixed : return true;
1683  case ARM::VST1q32wb_fixed : return true;
1684  case ARM::VST1q64wb_fixed : return true;
1685  case ARM::VST1d64TPseudoWB_fixed : return true;
1686  case ARM::VST1d64QPseudoWB_fixed : return true;
1687  case ARM::VST2d8wb_fixed : return true;
1688  case ARM::VST2d16wb_fixed : return true;
1689  case ARM::VST2d32wb_fixed : return true;
1690  case ARM::VST2q8PseudoWB_fixed : return true;
1691  case ARM::VST2q16PseudoWB_fixed : return true;
1692  case ARM::VST2q32PseudoWB_fixed : return true;
1693  }
1694 }
1695 
1696 // Get the register stride update opcode of a VLD/VST instruction that
1697 // is otherwise equivalent to the given fixed stride updating instruction.
1698 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1699  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1700  && "Incorrect fixed stride updating instruction.");
1701  switch (Opc) {
1702  default: break;
1703  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1704  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1705  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1706  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1707  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1708  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1709  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1710  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1711  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1712  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1713  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1714  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1715  case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
1716  case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
1717  case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
1718  case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
1719  case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
1720  case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
1721 
1722  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1723  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1724  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1725  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1726  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1727  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1728  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1729  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1730  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1731  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1732 
1733  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1734  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1735  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1736  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1737  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1738  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1739 
1740  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1741  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1742  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1743  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1744  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1745  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1746 
1747  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1748  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1749  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1750  }
1751  return Opc; // If not one we handle, return it unchanged.
1752 }
1753 
1754 /// Returns true if the given increment is a Constant known to be equal to the
1755 /// access size performed by a NEON load/store. This means the "[rN]!" form can
1756 /// be used.
1757 static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1758  auto C = dyn_cast<ConstantSDNode>(Inc);
1759  return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
1760 }
1761 
1762 void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1763  const uint16_t *DOpcodes,
1764  const uint16_t *QOpcodes0,
1765  const uint16_t *QOpcodes1) {
1766  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1767  SDLoc dl(N);
1768 
1769  SDValue MemAddr, Align;
1770  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1771  // nodes are not intrinsics.
1772  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1773  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1774  return;
1775 
1776  SDValue Chain = N->getOperand(0);
1777  EVT VT = N->getValueType(0);
1778  bool is64BitVector = VT.is64BitVector();
1779  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1780 
1781  unsigned OpcodeIndex;
1782  switch (VT.getSimpleVT().SimpleTy) {
1783  default: llvm_unreachable("unhandled vld type");
1784  // Double-register operations:
1785  case MVT::v8i8: OpcodeIndex = 0; break;
1786  case MVT::v4f16:
1787  case MVT::v4i16: OpcodeIndex = 1; break;
1788  case MVT::v2f32:
1789  case MVT::v2i32: OpcodeIndex = 2; break;
1790  case MVT::v1i64: OpcodeIndex = 3; break;
1791  // Quad-register operations:
1792  case MVT::v16i8: OpcodeIndex = 0; break;
1793  case MVT::v8f16:
1794  case MVT::v8i16: OpcodeIndex = 1; break;
1795  case MVT::v4f32:
1796  case MVT::v4i32: OpcodeIndex = 2; break;
1797  case MVT::v2f64:
1798  case MVT::v2i64: OpcodeIndex = 3; break;
1799  }
1800 
1801  EVT ResTy;
1802  if (NumVecs == 1)
1803  ResTy = VT;
1804  else {
1805  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1806  if (!is64BitVector)
1807  ResTyElts *= 2;
1808  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1809  }
1810  std::vector<EVT> ResTys;
1811  ResTys.push_back(ResTy);
1812  if (isUpdating)
1813  ResTys.push_back(MVT::i32);
1814  ResTys.push_back(MVT::Other);
1815 
1816  SDValue Pred = getAL(CurDAG, dl);
1817  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1818  SDNode *VLd;
1820 
1821  // Double registers and VLD1/VLD2 quad registers are directly supported.
1822  if (is64BitVector || NumVecs <= 2) {
1823  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1824  QOpcodes0[OpcodeIndex]);
1825  Ops.push_back(MemAddr);
1826  Ops.push_back(Align);
1827  if (isUpdating) {
1828  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1829  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1830  if (!IsImmUpdate) {
1831  // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1832  // check for the opcode rather than the number of vector elements.
1833  if (isVLDfixed(Opc))
1834  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1835  Ops.push_back(Inc);
1836  // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
1837  // the operands if not such an opcode.
1838  } else if (!isVLDfixed(Opc))
1839  Ops.push_back(Reg0);
1840  }
1841  Ops.push_back(Pred);
1842  Ops.push_back(Reg0);
1843  Ops.push_back(Chain);
1844  VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1845 
1846  } else {
1847  // Otherwise, quad registers are loaded with two separate instructions,
1848  // where one loads the even registers and the other loads the odd registers.
1849  EVT AddrTy = MemAddr.getValueType();
1850 
1851  // Load the even subregs. This is always an updating load, so that it
1852  // provides the address to the second load for the odd subregs.
1853  SDValue ImplDef =
1854  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1855  const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1856  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1857  ResTy, AddrTy, MVT::Other, OpsA);
1858  Chain = SDValue(VLdA, 2);
1859 
1860  // Load the odd subregs.
1861  Ops.push_back(SDValue(VLdA, 1));
1862  Ops.push_back(Align);
1863  if (isUpdating) {
1864  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1865  assert(isa<ConstantSDNode>(Inc.getNode()) &&
1866  "only constant post-increment update allowed for VLD3/4");
1867  (void)Inc;
1868  Ops.push_back(Reg0);
1869  }
1870  Ops.push_back(SDValue(VLdA, 0));
1871  Ops.push_back(Pred);
1872  Ops.push_back(Reg0);
1873  Ops.push_back(Chain);
1874  VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1875  }
1876 
1877  // Transfer memoperands.
1878  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1879  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp});
1880 
1881  if (NumVecs == 1) {
1882  ReplaceNode(N, VLd);
1883  return;
1884  }
1885 
1886  // Extract out the subregisters.
1887  SDValue SuperReg = SDValue(VLd, 0);
1888  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
1889  ARM::qsub_3 == ARM::qsub_0 + 3,
1890  "Unexpected subreg numbering");
1891  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1892  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1893  ReplaceUses(SDValue(N, Vec),
1894  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1895  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1896  if (isUpdating)
1897  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1898  CurDAG->RemoveDeadNode(N);
1899 }
1900 
1901 void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1902  const uint16_t *DOpcodes,
1903  const uint16_t *QOpcodes0,
1904  const uint16_t *QOpcodes1) {
1905  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1906  SDLoc dl(N);
1907 
1908  SDValue MemAddr, Align;
1909  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1910  // nodes are not intrinsics.
1911  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1912  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1913  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1914  return;
1915 
1916  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1917 
1918  SDValue Chain = N->getOperand(0);
1919  EVT VT = N->getOperand(Vec0Idx).getValueType();
1920  bool is64BitVector = VT.is64BitVector();
1921  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1922 
1923  unsigned OpcodeIndex;
1924  switch (VT.getSimpleVT().SimpleTy) {
1925  default: llvm_unreachable("unhandled vst type");
1926  // Double-register operations:
1927  case MVT::v8i8: OpcodeIndex = 0; break;
1928  case MVT::v4f16:
1929  case MVT::v4i16: OpcodeIndex = 1; break;
1930  case MVT::v2f32:
1931  case MVT::v2i32: OpcodeIndex = 2; break;
1932  case MVT::v1i64: OpcodeIndex = 3; break;
1933  // Quad-register operations:
1934  case MVT::v16i8: OpcodeIndex = 0; break;
1935  case MVT::v8f16:
1936  case MVT::v8i16: OpcodeIndex = 1; break;
1937  case MVT::v4f32:
1938  case MVT::v4i32: OpcodeIndex = 2; break;
1939  case MVT::v2f64:
1940  case MVT::v2i64: OpcodeIndex = 3; break;
1941  }
1942 
1943  std::vector<EVT> ResTys;
1944  if (isUpdating)
1945  ResTys.push_back(MVT::i32);
1946  ResTys.push_back(MVT::Other);
1947 
1948  SDValue Pred = getAL(CurDAG, dl);
1949  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1951 
1952  // Double registers and VST1/VST2 quad registers are directly supported.
1953  if (is64BitVector || NumVecs <= 2) {
1954  SDValue SrcReg;
1955  if (NumVecs == 1) {
1956  SrcReg = N->getOperand(Vec0Idx);
1957  } else if (is64BitVector) {
1958  // Form a REG_SEQUENCE to force register allocation.
1959  SDValue V0 = N->getOperand(Vec0Idx + 0);
1960  SDValue V1 = N->getOperand(Vec0Idx + 1);
1961  if (NumVecs == 2)
1962  SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1963  else {
1964  SDValue V2 = N->getOperand(Vec0Idx + 2);
1965  // If it's a vst3, form a quad D-register and leave the last part as
1966  // an undef.
1967  SDValue V3 = (NumVecs == 3)
1968  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1969  : N->getOperand(Vec0Idx + 3);
1970  SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1971  }
1972  } else {
1973  // Form a QQ register.
1974  SDValue Q0 = N->getOperand(Vec0Idx);
1975  SDValue Q1 = N->getOperand(Vec0Idx + 1);
1976  SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1977  }
1978 
1979  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1980  QOpcodes0[OpcodeIndex]);
1981  Ops.push_back(MemAddr);
1982  Ops.push_back(Align);
1983  if (isUpdating) {
1984  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1985  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1986  if (!IsImmUpdate) {
1987  // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
1988  // check for the opcode rather than the number of vector elements.
1989  if (isVSTfixed(Opc))
1990  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1991  Ops.push_back(Inc);
1992  }
1993  // VST1/VST2 fixed increment does not need Reg0 so only include it in
1994  // the operands if not such an opcode.
1995  else if (!isVSTfixed(Opc))
1996  Ops.push_back(Reg0);
1997  }
1998  Ops.push_back(SrcReg);
1999  Ops.push_back(Pred);
2000  Ops.push_back(Reg0);
2001  Ops.push_back(Chain);
2002  SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2003 
2004  // Transfer memoperands.
2005  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp});
2006 
2007  ReplaceNode(N, VSt);
2008  return;
2009  }
2010 
2011  // Otherwise, quad registers are stored with two separate instructions,
2012  // where one stores the even registers and the other stores the odd registers.
2013 
2014  // Form the QQQQ REG_SEQUENCE.
2015  SDValue V0 = N->getOperand(Vec0Idx + 0);
2016  SDValue V1 = N->getOperand(Vec0Idx + 1);
2017  SDValue V2 = N->getOperand(Vec0Idx + 2);
2018  SDValue V3 = (NumVecs == 3)
2019  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2020  : N->getOperand(Vec0Idx + 3);
2021  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2022 
2023  // Store the even D registers. This is always an updating store, so that it
2024  // provides the address to the second store for the odd subregs.
2025  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2026  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2027  MemAddr.getValueType(),
2028  MVT::Other, OpsA);
2029  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp});
2030  Chain = SDValue(VStA, 1);
2031 
2032  // Store the odd D registers.
2033  Ops.push_back(SDValue(VStA, 0));
2034  Ops.push_back(Align);
2035  if (isUpdating) {
2036  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2037  assert(isa<ConstantSDNode>(Inc.getNode()) &&
2038  "only constant post-increment update allowed for VST3/4");
2039  (void)Inc;
2040  Ops.push_back(Reg0);
2041  }
2042  Ops.push_back(RegSeq);
2043  Ops.push_back(Pred);
2044  Ops.push_back(Reg0);
2045  Ops.push_back(Chain);
2046  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2047  Ops);
2048  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp});
2049  ReplaceNode(N, VStB);
2050 }
2051 
2052 void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2053  unsigned NumVecs,
2054  const uint16_t *DOpcodes,
2055  const uint16_t *QOpcodes) {
2056  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2057  SDLoc dl(N);
2058 
2059  SDValue MemAddr, Align;
2060  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2061  // nodes are not intrinsics.
2062  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2063  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2064  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2065  return;
2066 
2067  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2068 
2069  SDValue Chain = N->getOperand(0);
2070  unsigned Lane =
2071  cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2072  EVT VT = N->getOperand(Vec0Idx).getValueType();
2073  bool is64BitVector = VT.is64BitVector();
2074 
2075  unsigned Alignment = 0;
2076  if (NumVecs != 3) {
2077  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2078  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2079  if (Alignment > NumBytes)
2080  Alignment = NumBytes;
2081  if (Alignment < 8 && Alignment < NumBytes)
2082  Alignment = 0;
2083  // Alignment must be a power of two; make sure of that.
2084  Alignment = (Alignment & -Alignment);
2085  if (Alignment == 1)
2086  Alignment = 0;
2087  }
2088  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2089 
2090  unsigned OpcodeIndex;
2091  switch (VT.getSimpleVT().SimpleTy) {
2092  default: llvm_unreachable("unhandled vld/vst lane type");
2093  // Double-register operations:
2094  case MVT::v8i8: OpcodeIndex = 0; break;
2095  case MVT::v4f16:
2096  case MVT::v4i16: OpcodeIndex = 1; break;
2097  case MVT::v2f32:
2098  case MVT::v2i32: OpcodeIndex = 2; break;
2099  // Quad-register operations:
2100  case MVT::v8f16:
2101  case MVT::v8i16: OpcodeIndex = 0; break;
2102  case MVT::v4f32:
2103  case MVT::v4i32: OpcodeIndex = 1; break;
2104  }
2105 
2106  std::vector<EVT> ResTys;
2107  if (IsLoad) {
2108  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2109  if (!is64BitVector)
2110  ResTyElts *= 2;
2111  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2112  MVT::i64, ResTyElts));
2113  }
2114  if (isUpdating)
2115  ResTys.push_back(MVT::i32);
2116  ResTys.push_back(MVT::Other);
2117 
2118  SDValue Pred = getAL(CurDAG, dl);
2119  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2120 
2122  Ops.push_back(MemAddr);
2123  Ops.push_back(Align);
2124  if (isUpdating) {
2125  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2126  bool IsImmUpdate =
2127  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2128  Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2129  }
2130 
2131  SDValue SuperReg;
2132  SDValue V0 = N->getOperand(Vec0Idx + 0);
2133  SDValue V1 = N->getOperand(Vec0Idx + 1);
2134  if (NumVecs == 2) {
2135  if (is64BitVector)
2136  SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2137  else
2138  SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2139  } else {
2140  SDValue V2 = N->getOperand(Vec0Idx + 2);
2141  SDValue V3 = (NumVecs == 3)
2142  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2143  : N->getOperand(Vec0Idx + 3);
2144  if (is64BitVector)
2145  SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2146  else
2147  SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2148  }
2149  Ops.push_back(SuperReg);
2150  Ops.push_back(getI32Imm(Lane, dl));
2151  Ops.push_back(Pred);
2152  Ops.push_back(Reg0);
2153  Ops.push_back(Chain);
2154 
2155  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2156  QOpcodes[OpcodeIndex]);
2157  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2158  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp});
2159  if (!IsLoad) {
2160  ReplaceNode(N, VLdLn);
2161  return;
2162  }
2163 
2164  // Extract the subregisters.
2165  SuperReg = SDValue(VLdLn, 0);
2166  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2167  ARM::qsub_3 == ARM::qsub_0 + 3,
2168  "Unexpected subreg numbering");
2169  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2170  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2171  ReplaceUses(SDValue(N, Vec),
2172  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2173  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2174  if (isUpdating)
2175  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2176  CurDAG->RemoveDeadNode(N);
2177 }
2178 
2179 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2180  bool isUpdating, unsigned NumVecs,
2181  const uint16_t *DOpcodes,
2182  const uint16_t *QOpcodes0,
2183  const uint16_t *QOpcodes1) {
2184  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2185  SDLoc dl(N);
2186 
2187  SDValue MemAddr, Align;
2188  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2189  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2190  return;
2191 
2192  SDValue Chain = N->getOperand(0);
2193  EVT VT = N->getValueType(0);
2194  bool is64BitVector = VT.is64BitVector();
2195 
2196  unsigned Alignment = 0;
2197  if (NumVecs != 3) {
2198  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2199  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2200  if (Alignment > NumBytes)
2201  Alignment = NumBytes;
2202  if (Alignment < 8 && Alignment < NumBytes)
2203  Alignment = 0;
2204  // Alignment must be a power of two; make sure of that.
2205  Alignment = (Alignment & -Alignment);
2206  if (Alignment == 1)
2207  Alignment = 0;
2208  }
2209  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2210 
2211  unsigned OpcodeIndex;
2212  switch (VT.getSimpleVT().SimpleTy) {
2213  default: llvm_unreachable("unhandled vld-dup type");
2214  case MVT::v8i8:
2215  case MVT::v16i8: OpcodeIndex = 0; break;
2216  case MVT::v4i16:
2217  case MVT::v8i16:
2218  case MVT::v4f16:
2219  case MVT::v8f16:
2220  OpcodeIndex = 1; break;
2221  case MVT::v2f32:
2222  case MVT::v2i32:
2223  case MVT::v4f32:
2224  case MVT::v4i32: OpcodeIndex = 2; break;
2225  case MVT::v1f64:
2226  case MVT::v1i64: OpcodeIndex = 3; break;
2227  }
2228 
2229  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2230  if (!is64BitVector)
2231  ResTyElts *= 2;
2232  EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2233 
2234  std::vector<EVT> ResTys;
2235  ResTys.push_back(ResTy);
2236  if (isUpdating)
2237  ResTys.push_back(MVT::i32);
2238  ResTys.push_back(MVT::Other);
2239 
2240  SDValue Pred = getAL(CurDAG, dl);
2241  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2242 
2243  SDNode *VLdDup;
2244  if (is64BitVector || NumVecs == 1) {
2246  Ops.push_back(MemAddr);
2247  Ops.push_back(Align);
2248  unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex] :
2249  QOpcodes0[OpcodeIndex];
2250  if (isUpdating) {
2251  // fixed-stride update instructions don't have an explicit writeback
2252  // operand. It's implicit in the opcode itself.
2253  SDValue Inc = N->getOperand(2);
2254  bool IsImmUpdate =
2255  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2256  if (NumVecs <= 2 && !IsImmUpdate)
2257  Opc = getVLDSTRegisterUpdateOpcode(Opc);
2258  if (!IsImmUpdate)
2259  Ops.push_back(Inc);
2260  // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2261  else if (NumVecs > 2)
2262  Ops.push_back(Reg0);
2263  }
2264  Ops.push_back(Pred);
2265  Ops.push_back(Reg0);
2266  Ops.push_back(Chain);
2267  VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2268  } else if (NumVecs == 2) {
2269  const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
2270  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2271  dl, ResTys, OpsA);
2272 
2273  Chain = SDValue(VLdA, 1);
2274  const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
2275  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2276  } else {
2277  SDValue ImplDef =
2278  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2279  const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
2280  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2281  dl, ResTys, OpsA);
2282 
2283  SDValue SuperReg = SDValue(VLdA, 0);
2284  Chain = SDValue(VLdA, 1);
2285  const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
2286  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2287  }
2288 
2289  // Transfer memoperands.
2290  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2291  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
2292 
2293  // Extract the subregisters.
2294  if (NumVecs == 1) {
2295  ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
2296  } else {
2297  SDValue SuperReg = SDValue(VLdDup, 0);
2298  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
2299  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2300  for (unsigned Vec = 0; Vec != NumVecs; ++Vec) {
2301  ReplaceUses(SDValue(N, Vec),
2302  CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2303  }
2304  }
2305  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2306  if (isUpdating)
2307  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2308  CurDAG->RemoveDeadNode(N);
2309 }
2310 
2311 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
2312  if (!Subtarget->hasV6T2Ops())
2313  return false;
2314 
2315  unsigned Opc = isSigned
2316  ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2317  : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2318  SDLoc dl(N);
2319 
2320  // For unsigned extracts, check for a shift right and mask
2321  unsigned And_imm = 0;
2322  if (N->getOpcode() == ISD::AND) {
2323  if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2324 
2325  // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2326  if (And_imm & (And_imm + 1))
2327  return false;
2328 
2329  unsigned Srl_imm = 0;
2331  Srl_imm)) {
2332  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2333 
2334  // Mask off the unnecessary bits of the AND immediate; normally
2335  // DAGCombine will do this, but that might not happen if
2336  // targetShrinkDemandedConstant chooses a different immediate.
2337  And_imm &= -1U >> Srl_imm;
2338 
2339  // Note: The width operand is encoded as width-1.
2340  unsigned Width = countTrailingOnes(And_imm) - 1;
2341  unsigned LSB = Srl_imm;
2342 
2343  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2344 
2345  if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2346  // It's cheaper to use a right shift to extract the top bits.
2347  if (Subtarget->isThumb()) {
2348  Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2349  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2350  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2351  getAL(CurDAG, dl), Reg0, Reg0 };
2352  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2353  return true;
2354  }
2355 
2356  // ARM models shift instructions as MOVsi with shifter operand.
2358  SDValue ShOpc =
2359  CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
2360  MVT::i32);
2361  SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2362  getAL(CurDAG, dl), Reg0, Reg0 };
2363  CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2364  return true;
2365  }
2366 
2367  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2368  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2369  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2370  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2371  getAL(CurDAG, dl), Reg0 };
2372  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2373  return true;
2374  }
2375  }
2376  return false;
2377  }
2378 
2379  // Otherwise, we're looking for a shift of a shift
2380  unsigned Shl_imm = 0;
2381  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2382  assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2383  unsigned Srl_imm = 0;
2384  if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2385  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2386  // Note: The width operand is encoded as width-1.
2387  unsigned Width = 32 - Srl_imm - 1;
2388  int LSB = Srl_imm - Shl_imm;
2389  if (LSB < 0)
2390  return false;
2391  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2392  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2393  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2394  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2395  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2396  getAL(CurDAG, dl), Reg0 };
2397  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2398  return true;
2399  }
2400  }
2401 
2402  // Or we are looking for a shift of an and, with a mask operand
2403  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
2404  isShiftedMask_32(And_imm)) {
2405  unsigned Srl_imm = 0;
2406  unsigned LSB = countTrailingZeros(And_imm);
2407  // Shift must be the same as the ands lsb
2408  if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
2409  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2410  unsigned MSB = 31 - countLeadingZeros(And_imm);
2411  // Note: The width operand is encoded as width-1.
2412  unsigned Width = MSB - LSB;
2413  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2414  assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2415  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2416  CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
2417  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2418  getAL(CurDAG, dl), Reg0 };
2419  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2420  return true;
2421  }
2422  }
2423 
2424  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2425  unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2426  unsigned LSB = 0;
2427  if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2429  return false;
2430 
2431  if (LSB + Width > 32)
2432  return false;
2433 
2434  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2435  assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx");
2436  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2437  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2438  CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
2439  getAL(CurDAG, dl), Reg0 };
2440  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2441  return true;
2442  }
2443 
2444  return false;
2445 }
2446 
2447 /// Target-specific DAG combining for ISD::XOR.
2448 /// Target-independent combining lowers SELECT_CC nodes of the form
2449 /// select_cc setg[ge] X, 0, X, -X
2450 /// select_cc setgt X, -1, X, -X
2451 /// select_cc setl[te] X, 0, -X, X
2452 /// select_cc setlt X, 1, -X, X
2453 /// which represent Integer ABS into:
2454 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2455 /// ARM instruction selection detects the latter and matches it to
2456 /// ARM::ABS or ARM::t2ABS machine node.
2457 bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
2458  SDValue XORSrc0 = N->getOperand(0);
2459  SDValue XORSrc1 = N->getOperand(1);
2460  EVT VT = N->getValueType(0);
2461 
2462  if (Subtarget->isThumb1Only())
2463  return false;
2464 
2465  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2466  return false;
2467 
2468  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2469  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2470  SDValue SRASrc0 = XORSrc1.getOperand(0);
2471  SDValue SRASrc1 = XORSrc1.getOperand(1);
2472  ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
2473  EVT XType = SRASrc0.getValueType();
2474  unsigned Size = XType.getSizeInBits() - 1;
2475 
2476  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2477  XType.isInteger() && SRAConstant != nullptr &&
2478  Size == SRAConstant->getZExtValue()) {
2479  unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2480  CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2481  return true;
2482  }
2483 
2484  return false;
2485 }
2486 
2487 /// We've got special pseudo-instructions for these
2488 void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2489  unsigned Opcode;
2490  EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2491  if (MemTy == MVT::i8)
2492  Opcode = ARM::CMP_SWAP_8;
2493  else if (MemTy == MVT::i16)
2494  Opcode = ARM::CMP_SWAP_16;
2495  else if (MemTy == MVT::i32)
2496  Opcode = ARM::CMP_SWAP_32;
2497  else
2498  llvm_unreachable("Unknown AtomicCmpSwap type");
2499 
2500  SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2501  N->getOperand(0)};
2502  SDNode *CmpSwap = CurDAG->getMachineNode(
2503  Opcode, SDLoc(N),
2504  CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
2505 
2506  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2507  CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2508 
2509  ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2510  ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2511  CurDAG->RemoveDeadNode(N);
2512 }
2513 
2516  unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
2517  unsigned LastOne = A.countTrailingZeros();
2518  if (A.countPopulation() != (FirstOne - LastOne + 1))
2520  return std::make_pair(FirstOne, LastOne);
2521 }
2522 
2523 void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
2524  assert(N->getOpcode() == ARMISD::CMPZ);
2525  SwitchEQNEToPLMI = false;
2526 
2527  if (!Subtarget->isThumb())
2528  // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
2529  // LSR don't exist as standalone instructions - they need the barrel shifter.
2530  return;
2531 
2532  // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
2533  SDValue And = N->getOperand(0);
2534  if (!And->hasOneUse())
2535  return;
2536 
2537  SDValue Zero = N->getOperand(1);
2538  if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isNullValue() ||
2539  And->getOpcode() != ISD::AND)
2540  return;
2541  SDValue X = And.getOperand(0);
2542  auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
2543 
2544  if (!C)
2545  return;
2546  auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
2547  if (!Range)
2548  return;
2549 
2550  // There are several ways to lower this:
2551  SDNode *NewN;
2552  SDLoc dl(N);
2553 
2554  auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
2555  if (Subtarget->isThumb2()) {
2556  Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
2557  SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2558  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2559  CurDAG->getRegister(0, MVT::i32) };
2560  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2561  } else {
2562  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2563  CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2564  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2565  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2566  }
2567  };
2568 
2569  if (Range->second == 0) {
2570  // 1. Mask includes the LSB -> Simply shift the top N bits off
2571  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2572  ReplaceNode(And.getNode(), NewN);
2573  } else if (Range->first == 31) {
2574  // 2. Mask includes the MSB -> Simply shift the bottom N bits off
2575  NewN = EmitShift(ARM::tLSRri, X, Range->second);
2576  ReplaceNode(And.getNode(), NewN);
2577  } else if (Range->first == Range->second) {
2578  // 3. Only one bit is set. We can shift this into the sign bit and use a
2579  // PL/MI comparison.
2580  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2581  ReplaceNode(And.getNode(), NewN);
2582 
2583  SwitchEQNEToPLMI = true;
2584  } else if (!Subtarget->hasV6T2Ops()) {
2585  // 4. Do a double shift to clear bottom and top bits, but only in
2586  // thumb-1 mode as in thumb-2 we can use UBFX.
2587  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2588  NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
2589  Range->second + (31 - Range->first));
2590  ReplaceNode(And.getNode(), NewN);
2591  }
2592 
2593 }
2594 
2596  SDLoc dl(N);
2597 
2598  if (N->isMachineOpcode()) {
2599  N->setNodeId(-1);
2600  return; // Already selected.
2601  }
2602 
2603  switch (N->getOpcode()) {
2604  default: break;
2605  case ISD::STORE: {
2606  // For Thumb1, match an sp-relative store in C++. This is a little
2607  // unfortunate, but I don't think I can make the chain check work
2608  // otherwise. (The chain of the store has to be the same as the chain
2609  // of the CopyFromReg, or else we can't replace the CopyFromReg with
2610  // a direct reference to "SP".)
2611  //
2612  // This is only necessary on Thumb1 because Thumb1 sp-relative stores use
2613  // a different addressing mode from other four-byte stores.
2614  //
2615  // This pattern usually comes up with call arguments.
2616  StoreSDNode *ST = cast<StoreSDNode>(N);
2617  SDValue Ptr = ST->getBasePtr();
2618  if (Subtarget->isThumb1Only() && ST->isUnindexed()) {
2619  int RHSC = 0;
2620  if (Ptr.getOpcode() == ISD::ADD &&
2621  isScaledConstantInRange(Ptr.getOperand(1), /*Scale=*/4, 0, 256, RHSC))
2622  Ptr = Ptr.getOperand(0);
2623 
2624  if (Ptr.getOpcode() == ISD::CopyFromReg &&
2625  cast<RegisterSDNode>(Ptr.getOperand(1))->getReg() == ARM::SP &&
2626  Ptr.getOperand(0) == ST->getChain()) {
2627  SDValue Ops[] = {ST->getValue(),
2628  CurDAG->getRegister(ARM::SP, MVT::i32),
2629  CurDAG->getTargetConstant(RHSC, dl, MVT::i32),
2630  getAL(CurDAG, dl),
2631  CurDAG->getRegister(0, MVT::i32),
2632  ST->getChain()};
2633  MachineSDNode *ResNode =
2634  CurDAG->getMachineNode(ARM::tSTRspi, dl, MVT::Other, Ops);
2635  MachineMemOperand *MemOp = ST->getMemOperand();
2636  CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2637  ReplaceNode(N, ResNode);
2638  return;
2639  }
2640  }
2641  break;
2642  }
2643  case ISD::WRITE_REGISTER:
2644  if (tryWriteRegister(N))
2645  return;
2646  break;
2647  case ISD::READ_REGISTER:
2648  if (tryReadRegister(N))
2649  return;
2650  break;
2651  case ISD::INLINEASM:
2652  case ISD::INLINEASM_BR:
2653  if (tryInlineAsm(N))
2654  return;
2655  break;
2656  case ISD::XOR:
2657  // Select special operations if XOR node forms integer ABS pattern
2658  if (tryABSOp(N))
2659  return;
2660  // Other cases are autogenerated.
2661  break;
2662  case ISD::Constant: {
2663  unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2664  // If we can't materialize the constant we need to use a literal pool
2665  if (ConstantMaterializationCost(Val) > 2) {
2666  SDValue CPIdx = CurDAG->getTargetConstantPool(
2667  ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2668  TLI->getPointerTy(CurDAG->getDataLayout()));
2669 
2670  SDNode *ResNode;
2671  if (Subtarget->isThumb()) {
2672  SDValue Ops[] = {
2673  CPIdx,
2674  getAL(CurDAG, dl),
2675  CurDAG->getRegister(0, MVT::i32),
2676  CurDAG->getEntryNode()
2677  };
2678  ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2679  Ops);
2680  } else {
2681  SDValue Ops[] = {
2682  CPIdx,
2683  CurDAG->getTargetConstant(0, dl, MVT::i32),
2684  getAL(CurDAG, dl),
2685  CurDAG->getRegister(0, MVT::i32),
2686  CurDAG->getEntryNode()
2687  };
2688  ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2689  Ops);
2690  }
2691  // Annotate the Node with memory operand information so that MachineInstr
2692  // queries work properly. This e.g. gives the register allocation the
2693  // required information for rematerialization.
2694  MachineFunction& MF = CurDAG->getMachineFunction();
2695  MachineMemOperand *MemOp =
2698 
2699  CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2700 
2701  ReplaceNode(N, ResNode);
2702  return;
2703  }
2704 
2705  // Other cases are autogenerated.
2706  break;
2707  }
2708  case ISD::FrameIndex: {
2709  // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2710  int FI = cast<FrameIndexSDNode>(N)->getIndex();
2711  SDValue TFI = CurDAG->getTargetFrameIndex(
2712  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2713  if (Subtarget->isThumb1Only()) {
2714  // Set the alignment of the frame object to 4, to avoid having to generate
2715  // more than one ADD
2716  MachineFrameInfo &MFI = MF->getFrameInfo();
2717  if (MFI.getObjectAlignment(FI) < 4)
2718  MFI.setObjectAlignment(FI, 4);
2719  CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2720  CurDAG->getTargetConstant(0, dl, MVT::i32));
2721  return;
2722  } else {
2723  unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2724  ARM::t2ADDri : ARM::ADDri);
2725  SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
2726  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2727  CurDAG->getRegister(0, MVT::i32) };
2728  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2729  return;
2730  }
2731  }
2732  case ISD::SRL:
2733  if (tryV6T2BitfieldExtractOp(N, false))
2734  return;
2735  break;
2737  case ISD::SRA:
2738  if (tryV6T2BitfieldExtractOp(N, true))
2739  return;
2740  break;
2741  case ISD::MUL:
2742  if (Subtarget->isThumb1Only())
2743  break;
2744  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2745  unsigned RHSV = C->getZExtValue();
2746  if (!RHSV) break;
2747  if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
2748  unsigned ShImm = Log2_32(RHSV-1);
2749  if (ShImm >= 32)
2750  break;
2751  SDValue V = N->getOperand(0);
2752  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2753  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2754  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2755  if (Subtarget->isThumb()) {
2756  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2757  CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2758  return;
2759  } else {
2760  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2761  Reg0 };
2762  CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2763  return;
2764  }
2765  }
2766  if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
2767  unsigned ShImm = Log2_32(RHSV+1);
2768  if (ShImm >= 32)
2769  break;
2770  SDValue V = N->getOperand(0);
2771  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2772  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2773  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2774  if (Subtarget->isThumb()) {
2775  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2776  CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2777  return;
2778  } else {
2779  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2780  Reg0 };
2781  CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2782  return;
2783  }
2784  }
2785  }
2786  break;
2787  case ISD::AND: {
2788  // Check for unsigned bitfield extract
2789  if (tryV6T2BitfieldExtractOp(N, false))
2790  return;
2791 
2792  // If an immediate is used in an AND node, it is possible that the immediate
2793  // can be more optimally materialized when negated. If this is the case we
2794  // can negate the immediate and use a BIC instead.
2795  auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2796  if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
2797  uint32_t Imm = (uint32_t) N1C->getZExtValue();
2798 
2799  // In Thumb2 mode, an AND can take a 12-bit immediate. If this
2800  // immediate can be negated and fit in the immediate operand of
2801  // a t2BIC, don't do any manual transform here as this can be
2802  // handled by the generic ISel machinery.
2803  bool PreferImmediateEncoding =
2804  Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
2805  if (!PreferImmediateEncoding &&
2806  ConstantMaterializationCost(Imm) >
2807  ConstantMaterializationCost(~Imm)) {
2808  // The current immediate costs more to materialize than a negated
2809  // immediate, so negate the immediate and use a BIC.
2810  SDValue NewImm =
2811  CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
2812  // If the new constant didn't exist before, reposition it in the topological
2813  // ordering so it is just before N. Otherwise, don't touch its location.
2814  if (NewImm->getNodeId() == -1)
2815  CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
2816 
2817  if (!Subtarget->hasThumb2()) {
2818  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
2819  N->getOperand(0), NewImm, getAL(CurDAG, dl),
2820  CurDAG->getRegister(0, MVT::i32)};
2821  ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
2822  return;
2823  } else {
2824  SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
2825  CurDAG->getRegister(0, MVT::i32),
2826  CurDAG->getRegister(0, MVT::i32)};
2827  ReplaceNode(N,
2828  CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
2829  return;
2830  }
2831  }
2832  }
2833 
2834  // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2835  // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2836  // are entirely contributed by c2 and lower 16-bits are entirely contributed
2837  // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2838  // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2839  EVT VT = N->getValueType(0);
2840  if (VT != MVT::i32)
2841  break;
2842  unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2843  ? ARM::t2MOVTi16
2844  : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2845  if (!Opc)
2846  break;
2847  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2848  N1C = dyn_cast<ConstantSDNode>(N1);
2849  if (!N1C)
2850  break;
2851  if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2852  SDValue N2 = N0.getOperand(1);
2854  if (!N2C)
2855  break;
2856  unsigned N1CVal = N1C->getZExtValue();
2857  unsigned N2CVal = N2C->getZExtValue();
2858  if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2859  (N1CVal & 0xffffU) == 0xffffU &&
2860  (N2CVal & 0xffffU) == 0x0U) {
2861  SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2862  dl, MVT::i32);
2863  SDValue Ops[] = { N0.getOperand(0), Imm16,
2864  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
2865  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
2866  return;
2867  }
2868  }
2869 
2870  break;
2871  }
2872  case ARMISD::UMAAL: {
2873  unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
2874  SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2875  N->getOperand(2), N->getOperand(3),
2876  getAL(CurDAG, dl),
2877  CurDAG->getRegister(0, MVT::i32) };
2878  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
2879  return;
2880  }
2881  case ARMISD::UMLAL:{
2882  if (Subtarget->isThumb()) {
2883  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2884  N->getOperand(3), getAL(CurDAG, dl),
2885  CurDAG->getRegister(0, MVT::i32)};
2886  ReplaceNode(
2887  N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
2888  return;
2889  }else{
2890  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2891  N->getOperand(3), getAL(CurDAG, dl),
2892  CurDAG->getRegister(0, MVT::i32),
2893  CurDAG->getRegister(0, MVT::i32) };
2894  ReplaceNode(N, CurDAG->getMachineNode(
2895  Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
2896  MVT::i32, MVT::i32, Ops));
2897  return;
2898  }
2899  }
2900  case ARMISD::SMLAL:{
2901  if (Subtarget->isThumb()) {
2902  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2903  N->getOperand(3), getAL(CurDAG, dl),
2904  CurDAG->getRegister(0, MVT::i32)};
2905  ReplaceNode(
2906  N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
2907  return;
2908  }else{
2909  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2910  N->getOperand(3), getAL(CurDAG, dl),
2911  CurDAG->getRegister(0, MVT::i32),
2912  CurDAG->getRegister(0, MVT::i32) };
2913  ReplaceNode(N, CurDAG->getMachineNode(
2914  Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
2915  MVT::i32, MVT::i32, Ops));
2916  return;
2917  }
2918  }
2919  case ARMISD::SUBE: {
2920  if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
2921  break;
2922  // Look for a pattern to match SMMLS
2923  // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
2924  if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
2925  N->getOperand(2).getOpcode() != ARMISD::SUBC ||
2926  !SDValue(N, 1).use_empty())
2927  break;
2928 
2929  if (Subtarget->isThumb())
2930  assert(Subtarget->hasThumb2() &&
2931  "This pattern should not be generated for Thumb");
2932 
2933  SDValue SmulLoHi = N->getOperand(1);
2934  SDValue Subc = N->getOperand(2);
2935  auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
2936 
2937  if (!Zero || Zero->getZExtValue() != 0 ||
2938  Subc.getOperand(1) != SmulLoHi.getValue(0) ||
2939  N->getOperand(1) != SmulLoHi.getValue(1) ||
2940  N->getOperand(2) != Subc.getValue(1))
2941  break;
2942 
2943  unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
2944  SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
2945  N->getOperand(0), getAL(CurDAG, dl),
2946  CurDAG->getRegister(0, MVT::i32) };
2947  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
2948  return;
2949  }
2950  case ISD::LOAD: {
2951  if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
2952  if (tryT2IndexedLoad(N))
2953  return;
2954  } else if (Subtarget->isThumb()) {
2955  if (tryT1IndexedLoad(N))
2956  return;
2957  } else if (tryARMIndexedLoad(N))
2958  return;
2959  // Other cases are autogenerated.
2960  break;
2961  }
2962  case ARMISD::BRCOND: {
2963  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2964  // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2965  // Pattern complexity = 6 cost = 1 size = 0
2966 
2967  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2968  // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2969  // Pattern complexity = 6 cost = 1 size = 0
2970 
2971  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2972  // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2973  // Pattern complexity = 6 cost = 1 size = 0
2974 
2975  unsigned Opc = Subtarget->isThumb() ?
2976  ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2977  SDValue Chain = N->getOperand(0);
2978  SDValue N1 = N->getOperand(1);
2979  SDValue N2 = N->getOperand(2);
2980  SDValue N3 = N->getOperand(3);
2981  SDValue InFlag = N->getOperand(4);
2982  assert(N1.getOpcode() == ISD::BasicBlock);
2983  assert(N2.getOpcode() == ISD::Constant);
2984  assert(N3.getOpcode() == ISD::Register);
2985 
2986  unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
2987 
2988  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2989  bool SwitchEQNEToPLMI;
2990  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2991  InFlag = N->getOperand(4);
2992 
2993  if (SwitchEQNEToPLMI) {
2994  switch ((ARMCC::CondCodes)CC) {
2995  default: llvm_unreachable("CMPZ must be either NE or EQ!");
2996  case ARMCC::NE:
2997  CC = (unsigned)ARMCC::MI;
2998  break;
2999  case ARMCC::EQ:
3000  CC = (unsigned)ARMCC::PL;
3001  break;
3002  }
3003  }
3004  }
3005 
3006  SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
3007  SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
3008  SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
3009  MVT::Glue, Ops);
3010  Chain = SDValue(ResNode, 0);
3011  if (N->getNumValues() == 2) {
3012  InFlag = SDValue(ResNode, 1);
3013  ReplaceUses(SDValue(N, 1), InFlag);
3014  }
3015  ReplaceUses(SDValue(N, 0),
3016  SDValue(Chain.getNode(), Chain.getResNo()));
3017  CurDAG->RemoveDeadNode(N);
3018  return;
3019  }
3020 
3021  case ARMISD::CMPZ: {
3022  // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
3023  // This allows us to avoid materializing the expensive negative constant.
3024  // The CMPZ #0 is useless and will be peepholed away but we need to keep it
3025  // for its glue output.
3026  SDValue X = N->getOperand(0);
3027  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
3028  if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
3029  int64_t Addend = -C->getSExtValue();
3030 
3031  SDNode *Add = nullptr;
3032  // ADDS can be better than CMN if the immediate fits in a
3033  // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
3034  // Outside that range we can just use a CMN which is 32-bit but has a
3035  // 12-bit immediate range.
3036  if (Addend < 1<<8) {
3037  if (Subtarget->isThumb2()) {
3038  SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3039  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
3040  CurDAG->getRegister(0, MVT::i32) };
3041  Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
3042  } else {
3043  unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
3044  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
3045  CurDAG->getTargetConstant(Addend, dl, MVT::i32),
3046  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
3047  Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
3048  }
3049  }
3050  if (Add) {
3051  SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
3052  CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
3053  }
3054  }
3055  // Other cases are autogenerated.
3056  break;
3057  }
3058 
3059  case ARMISD::CMOV: {
3060  SDValue InFlag = N->getOperand(4);
3061 
3062  if (InFlag.getOpcode() == ARMISD::CMPZ) {
3063  bool SwitchEQNEToPLMI;
3064  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
3065 
3066  if (SwitchEQNEToPLMI) {
3067  SDValue ARMcc = N->getOperand(2);
3068  ARMCC::CondCodes CC =
3069  (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
3070 
3071  switch (CC) {
3072  default: llvm_unreachable("CMPZ must be either NE or EQ!");
3073  case ARMCC::NE:
3074  CC = ARMCC::MI;
3075  break;
3076  case ARMCC::EQ:
3077  CC = ARMCC::PL;
3078  break;
3079  }
3080  SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
3081  SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
3082  N->getOperand(3), N->getOperand(4)};
3083  CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
3084  }
3085 
3086  }
3087  // Other cases are autogenerated.
3088  break;
3089  }
3090 
3091  case ARMISD::VZIP: {
3092  unsigned Opc = 0;
3093  EVT VT = N->getValueType(0);
3094  switch (VT.getSimpleVT().SimpleTy) {
3095  default: return;
3096  case MVT::v8i8: Opc = ARM::VZIPd8; break;
3097  case MVT::v4f16:
3098  case MVT::v4i16: Opc = ARM::VZIPd16; break;
3099  case MVT::v2f32:
3100  // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3101  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3102  case MVT::v16i8: Opc = ARM::VZIPq8; break;
3103  case MVT::v8f16:
3104  case MVT::v8i16: Opc = ARM::VZIPq16; break;
3105  case MVT::v4f32:
3106  case MVT::v4i32: Opc = ARM::VZIPq32; break;
3107  }
3108  SDValue Pred = getAL(CurDAG, dl);
3109  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3110  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3111  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3112  return;
3113  }
3114  case ARMISD::VUZP: {
3115  unsigned Opc = 0;
3116  EVT VT = N->getValueType(0);
3117  switch (VT.getSimpleVT().SimpleTy) {
3118  default: return;
3119  case MVT::v8i8: Opc = ARM::VUZPd8; break;
3120  case MVT::v4f16:
3121  case MVT::v4i16: Opc = ARM::VUZPd16; break;
3122  case MVT::v2f32:
3123  // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3124  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3125  case MVT::v16i8: Opc = ARM::VUZPq8; break;
3126  case MVT::v8f16:
3127  case MVT::v8i16: Opc = ARM::VUZPq16; break;
3128  case MVT::v4f32:
3129  case MVT::v4i32: Opc = ARM::VUZPq32; break;
3130  }
3131  SDValue Pred = getAL(CurDAG, dl);
3132  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3133  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3134  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3135  return;
3136  }
3137  case ARMISD::VTRN: {
3138  unsigned Opc = 0;
3139  EVT VT = N->getValueType(0);
3140  switch (VT.getSimpleVT().SimpleTy) {
3141  default: return;
3142  case MVT::v8i8: Opc = ARM::VTRNd8; break;
3143  case MVT::v4f16:
3144  case MVT::v4i16: Opc = ARM::VTRNd16; break;
3145  case MVT::v2f32:
3146  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3147  case MVT::v16i8: Opc = ARM::VTRNq8; break;
3148  case MVT::v8f16:
3149  case MVT::v8i16: Opc = ARM::VTRNq16; break;
3150  case MVT::v4f32:
3151  case MVT::v4i32: Opc = ARM::VTRNq32; break;
3152  }
3153  SDValue Pred = getAL(CurDAG, dl);
3154  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3155  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3156  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3157  return;
3158  }
3159  case ARMISD::BUILD_VECTOR: {
3160  EVT VecVT = N->getValueType(0);
3161  EVT EltVT = VecVT.getVectorElementType();
3162  unsigned NumElts = VecVT.getVectorNumElements();
3163  if (EltVT == MVT::f64) {
3164  assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
3165  ReplaceNode(
3166  N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3167  return;
3168  }
3169  assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
3170  if (NumElts == 2) {
3171  ReplaceNode(
3172  N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3173  return;
3174  }
3175  assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
3176  ReplaceNode(N,
3177  createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
3178  N->getOperand(2), N->getOperand(3)));
3179  return;
3180  }
3181 
3182  case ARMISD::VLD1DUP: {
3183  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
3184  ARM::VLD1DUPd32 };
3185  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
3186  ARM::VLD1DUPq32 };
3187  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes);
3188  return;
3189  }
3190 
3191  case ARMISD::VLD2DUP: {
3192  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3193  ARM::VLD2DUPd32 };
3194  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes);
3195  return;
3196  }
3197 
3198  case ARMISD::VLD3DUP: {
3199  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
3200  ARM::VLD3DUPd16Pseudo,
3201  ARM::VLD3DUPd32Pseudo };
3202  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes);
3203  return;
3204  }
3205 
3206  case ARMISD::VLD4DUP: {
3207  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
3208  ARM::VLD4DUPd16Pseudo,
3209  ARM::VLD4DUPd32Pseudo };
3210  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes);
3211  return;
3212  }
3213 
3214  case ARMISD::VLD1DUP_UPD: {
3215  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
3216  ARM::VLD1DUPd16wb_fixed,
3217  ARM::VLD1DUPd32wb_fixed };
3218  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
3219  ARM::VLD1DUPq16wb_fixed,
3220  ARM::VLD1DUPq32wb_fixed };
3221  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
3222  return;
3223  }
3224 
3225  case ARMISD::VLD2DUP_UPD: {
3226  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
3227  ARM::VLD2DUPd16wb_fixed,
3228  ARM::VLD2DUPd32wb_fixed };
3229  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, Opcodes);
3230  return;
3231  }
3232 
3233  case ARMISD::VLD3DUP_UPD: {
3234  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
3235  ARM::VLD3DUPd16Pseudo_UPD,
3236  ARM::VLD3DUPd32Pseudo_UPD };
3237  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, Opcodes);
3238  return;
3239  }
3240 
3241  case ARMISD::VLD4DUP_UPD: {
3242  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
3243  ARM::VLD4DUPd16Pseudo_UPD,
3244  ARM::VLD4DUPd32Pseudo_UPD };
3245  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, Opcodes);
3246  return;
3247  }
3248 
3249  case ARMISD::VLD1_UPD: {
3250  static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
3251  ARM::VLD1d16wb_fixed,
3252  ARM::VLD1d32wb_fixed,
3253  ARM::VLD1d64wb_fixed };
3254  static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
3255  ARM::VLD1q16wb_fixed,
3256  ARM::VLD1q32wb_fixed,
3257  ARM::VLD1q64wb_fixed };
3258  SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
3259  return;
3260  }
3261 
3262  case ARMISD::VLD2_UPD: {
3263  static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
3264  ARM::VLD2d16wb_fixed,
3265  ARM::VLD2d32wb_fixed,
3266  ARM::VLD1q64wb_fixed};
3267  static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
3268  ARM::VLD2q16PseudoWB_fixed,
3269  ARM::VLD2q32PseudoWB_fixed };
3270  SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
3271  return;
3272  }
3273 
3274  case ARMISD::VLD3_UPD: {
3275  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
3276  ARM::VLD3d16Pseudo_UPD,
3277  ARM::VLD3d32Pseudo_UPD,
3278  ARM::VLD1d64TPseudoWB_fixed};
3279  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3280  ARM::VLD3q16Pseudo_UPD,
3281  ARM::VLD3q32Pseudo_UPD };
3282  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
3283  ARM::VLD3q16oddPseudo_UPD,
3284  ARM::VLD3q32oddPseudo_UPD };
3285  SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3286  return;
3287  }
3288 
3289  case ARMISD::VLD4_UPD: {
3290  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
3291  ARM::VLD4d16Pseudo_UPD,
3292  ARM::VLD4d32Pseudo_UPD,
3293  ARM::VLD1d64QPseudoWB_fixed};
3294  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3295  ARM::VLD4q16Pseudo_UPD,
3296  ARM::VLD4q32Pseudo_UPD };
3297  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
3298  ARM::VLD4q16oddPseudo_UPD,
3299  ARM::VLD4q32oddPseudo_UPD };
3300  SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3301  return;
3302  }
3303 
3304  case ARMISD::VLD2LN_UPD: {
3305  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
3306  ARM::VLD2LNd16Pseudo_UPD,
3307  ARM::VLD2LNd32Pseudo_UPD };
3308  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
3309  ARM::VLD2LNq32Pseudo_UPD };
3310  SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
3311  return;
3312  }
3313 
3314  case ARMISD::VLD3LN_UPD: {
3315  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
3316  ARM::VLD3LNd16Pseudo_UPD,
3317  ARM::VLD3LNd32Pseudo_UPD };
3318  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
3319  ARM::VLD3LNq32Pseudo_UPD };
3320  SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
3321  return;
3322  }
3323 
3324  case ARMISD::VLD4LN_UPD: {
3325  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3326  ARM::VLD4LNd16Pseudo_UPD,
3327  ARM::VLD4LNd32Pseudo_UPD };
3328  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3329  ARM::VLD4LNq32Pseudo_UPD };
3330  SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3331  return;
3332  }
3333 
3334  case ARMISD::VST1_UPD: {
3335  static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3336  ARM::VST1d16wb_fixed,
3337  ARM::VST1d32wb_fixed,
3338  ARM::VST1d64wb_fixed };
3339  static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3340  ARM::VST1q16wb_fixed,
3341  ARM::VST1q32wb_fixed,
3342  ARM::VST1q64wb_fixed };
3343  SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
3344  return;
3345  }
3346 
3347  case ARMISD::VST2_UPD: {
3348  static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3349  ARM::VST2d16wb_fixed,
3350  ARM::VST2d32wb_fixed,
3351  ARM::VST1q64wb_fixed};
3352  static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3353  ARM::VST2q16PseudoWB_fixed,
3354  ARM::VST2q32PseudoWB_fixed };
3355  SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
3356  return;
3357  }
3358 
3359  case ARMISD::VST3_UPD: {
3360  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3361  ARM::VST3d16Pseudo_UPD,
3362  ARM::VST3d32Pseudo_UPD,
3363  ARM::VST1d64TPseudoWB_fixed};
3364  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3365  ARM::VST3q16Pseudo_UPD,
3366  ARM::VST3q32Pseudo_UPD };
3367  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3368  ARM::VST3q16oddPseudo_UPD,
3369  ARM::VST3q32oddPseudo_UPD };
3370  SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3371  return;
3372  }
3373 
3374  case ARMISD::VST4_UPD: {
3375  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3376  ARM::VST4d16Pseudo_UPD,
3377  ARM::VST4d32Pseudo_UPD,
3378  ARM::VST1d64QPseudoWB_fixed};
3379  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3380  ARM::VST4q16Pseudo_UPD,
3381  ARM::VST4q32Pseudo_UPD };
3382  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3383  ARM::VST4q16oddPseudo_UPD,
3384  ARM::VST4q32oddPseudo_UPD };
3385  SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3386  return;
3387  }
3388 
3389  case ARMISD::VST2LN_UPD: {
3390  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3391  ARM::VST2LNd16Pseudo_UPD,
3392  ARM::VST2LNd32Pseudo_UPD };
3393  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3394  ARM::VST2LNq32Pseudo_UPD };
3395  SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3396  return;
3397  }
3398 
3399  case ARMISD::VST3LN_UPD: {
3400  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3401  ARM::VST3LNd16Pseudo_UPD,
3402  ARM::VST3LNd32Pseudo_UPD };
3403  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3404  ARM::VST3LNq32Pseudo_UPD };
3405  SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3406  return;
3407  }
3408 
3409  case ARMISD::VST4LN_UPD: {
3410  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3411  ARM::VST4LNd16Pseudo_UPD,
3412  ARM::VST4LNd32Pseudo_UPD };
3413  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3414  ARM::VST4LNq32Pseudo_UPD };
3415  SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3416  return;
3417  }
3418 
3419  case ISD::INTRINSIC_VOID:
3420  case ISD::INTRINSIC_W_CHAIN: {
3421  unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3422  switch (IntNo) {
3423  default:
3424  break;
3425 
3426  case Intrinsic::arm_mrrc:
3427  case Intrinsic::arm_mrrc2: {
3428  SDLoc dl(N);
3429  SDValue Chain = N->getOperand(0);
3430  unsigned Opc;
3431 
3432  if (Subtarget->isThumb())
3433  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
3434  else
3435  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
3436 
3438  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
3439  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
3440  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
3441 
3442  // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
3443  // instruction will always be '1111' but it is possible in assembly language to specify
3444  // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
3445  if (Opc != ARM::MRRC2) {
3446  Ops.push_back(getAL(CurDAG, dl));
3447  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3448  }
3449 
3450  Ops.push_back(Chain);
3451 
3452  // Writes to two registers.
3453  const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
3454 
3455  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
3456  return;
3457  }
3458  case Intrinsic::arm_ldaexd:
3459  case Intrinsic::arm_ldrexd: {
3460  SDLoc dl(N);
3461  SDValue Chain = N->getOperand(0);
3462  SDValue MemAddr = N->getOperand(2);
3463  bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
3464 
3465  bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3466  unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3467  : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3468 
3469  // arm_ldrexd returns a i64 value in {i32, i32}
3470  std::vector<EVT> ResTys;
3471  if (isThumb) {
3472  ResTys.push_back(MVT::i32);
3473  ResTys.push_back(MVT::i32);
3474  } else
3475  ResTys.push_back(MVT::Untyped);
3476  ResTys.push_back(MVT::Other);
3477 
3478  // Place arguments in the right order.
3479  SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3480  CurDAG->getRegister(0, MVT::i32), Chain};
3481  SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3482  // Transfer memoperands.
3483  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3484  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3485 
3486  // Remap uses.
3487  SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3488  if (!SDValue(N, 0).use_empty()) {
3489  SDValue Result;
3490  if (isThumb)
3491  Result = SDValue(Ld, 0);
3492  else {
3493  SDValue SubRegIdx =
3494  CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
3495  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3496  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3497  Result = SDValue(ResNode,0);
3498  }
3499  ReplaceUses(SDValue(N, 0), Result);
3500  }
3501  if (!SDValue(N, 1).use_empty()) {
3502  SDValue Result;
3503  if (isThumb)
3504  Result = SDValue(Ld, 1);
3505  else {
3506  SDValue SubRegIdx =
3507  CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
3508  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3509  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3510  Result = SDValue(ResNode,0);
3511  }
3512  ReplaceUses(SDValue(N, 1), Result);
3513  }
3514  ReplaceUses(SDValue(N, 2), OutChain);
3515  CurDAG->RemoveDeadNode(N);
3516  return;
3517  }
3518  case Intrinsic::arm_stlexd:
3519  case Intrinsic::arm_strexd: {
3520  SDLoc dl(N);
3521  SDValue Chain = N->getOperand(0);
3522  SDValue Val0 = N->getOperand(2);
3523  SDValue Val1 = N->getOperand(3);
3524  SDValue MemAddr = N->getOperand(4);
3525 
3526  // Store exclusive double return a i32 value which is the return status
3527  // of the issued store.
3528  const EVT ResTys[] = {MVT::i32, MVT::Other};
3529 
3530  bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3531  // Place arguments in the right order.
3533  if (isThumb) {
3534  Ops.push_back(Val0);
3535  Ops.push_back(Val1);
3536  } else
3537  // arm_strexd uses GPRPair.
3538  Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3539  Ops.push_back(MemAddr);
3540  Ops.push_back(getAL(CurDAG, dl));
3541  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3542  Ops.push_back(Chain);
3543 
3544  bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3545  unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3546  : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3547 
3548  SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3549  // Transfer memoperands.
3550  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3551  CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3552 
3553  ReplaceNode(N, St);
3554  return;
3555  }
3556 
3557  case Intrinsic::arm_neon_vld1: {
3558  static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3559  ARM::VLD1d32, ARM::VLD1d64 };
3560  static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3561  ARM::VLD1q32, ARM::VLD1q64};
3562  SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3563  return;
3564  }
3565 
3566  case Intrinsic::arm_neon_vld1x2: {
3567  static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3568  ARM::VLD1q32, ARM::VLD1q64 };
3569  static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
3570  ARM::VLD1d16QPseudo,
3571  ARM::VLD1d32QPseudo,
3572  ARM::VLD1d64QPseudo };
3573  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3574  return;
3575  }
3576 
3577  case Intrinsic::arm_neon_vld1x3: {
3578  static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
3579  ARM::VLD1d16TPseudo,
3580  ARM::VLD1d32TPseudo,
3581  ARM::VLD1d64TPseudo };
3582  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
3583  ARM::VLD1q16LowTPseudo_UPD,
3584  ARM::VLD1q32LowTPseudo_UPD,
3585  ARM::VLD1q64LowTPseudo_UPD };
3586  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
3587  ARM::VLD1q16HighTPseudo,
3588  ARM::VLD1q32HighTPseudo,
3589  ARM::VLD1q64HighTPseudo };
3590  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3591  return;
3592  }
3593 
3594  case Intrinsic::arm_neon_vld1x4: {
3595  static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
3596  ARM::VLD1d16QPseudo,
3597  ARM::VLD1d32QPseudo,
3598  ARM::VLD1d64QPseudo };
3599  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
3600  ARM::VLD1q16LowQPseudo_UPD,
3601  ARM::VLD1q32LowQPseudo_UPD,
3602  ARM::VLD1q64LowQPseudo_UPD };
3603  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
3604  ARM::VLD1q16HighQPseudo,
3605  ARM::VLD1q32HighQPseudo,
3606  ARM::VLD1q64HighQPseudo };
3607  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3608  return;
3609  }
3610 
3611  case Intrinsic::arm_neon_vld2: {
3612  static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3613  ARM::VLD2d32, ARM::VLD1q64 };
3614  static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3615  ARM::VLD2q32Pseudo };
3616  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3617  return;
3618  }
3619 
3620  case Intrinsic::arm_neon_vld3: {
3621  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3622  ARM::VLD3d16Pseudo,
3623  ARM::VLD3d32Pseudo,
3624  ARM::VLD1d64TPseudo };
3625  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3626  ARM::VLD3q16Pseudo_UPD,
3627  ARM::VLD3q32Pseudo_UPD };
3628  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3629  ARM::VLD3q16oddPseudo,
3630  ARM::VLD3q32oddPseudo };
3631  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3632  return;
3633  }
3634 
3635  case Intrinsic::arm_neon_vld4: {
3636  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3637  ARM::VLD4d16Pseudo,
3638  ARM::VLD4d32Pseudo,
3639  ARM::VLD1d64QPseudo };
3640  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3641  ARM::VLD4q16Pseudo_UPD,
3642  ARM::VLD4q32Pseudo_UPD };
3643  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3644  ARM::VLD4q16oddPseudo,
3645  ARM::VLD4q32oddPseudo };
3646  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3647  return;
3648  }
3649 
3650  case Intrinsic::arm_neon_vld2dup: {
3651  static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3652  ARM::VLD2DUPd32, ARM::VLD1q64 };
3653  static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
3654  ARM::VLD2DUPq16EvenPseudo,
3655  ARM::VLD2DUPq32EvenPseudo };
3656  static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
3657  ARM::VLD2DUPq16OddPseudo,
3658  ARM::VLD2DUPq32OddPseudo };
3659  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2,
3660  DOpcodes, QOpcodes0, QOpcodes1);
3661  return;
3662  }
3663 
3664  case Intrinsic::arm_neon_vld3dup: {
3665  static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
3666  ARM::VLD3DUPd16Pseudo,
3667  ARM::VLD3DUPd32Pseudo,
3668  ARM::VLD1d64TPseudo };
3669  static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
3670  ARM::VLD3DUPq16EvenPseudo,
3671  ARM::VLD3DUPq32EvenPseudo };
3672  static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
3673  ARM::VLD3DUPq16OddPseudo,
3674  ARM::VLD3DUPq32OddPseudo };
3675  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3,
3676  DOpcodes, QOpcodes0, QOpcodes1);
3677  return;
3678  }
3679 
3680  case Intrinsic::arm_neon_vld4dup: {
3681  static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
3682  ARM::VLD4DUPd16Pseudo,
3683  ARM::VLD4DUPd32Pseudo,
3684  ARM::VLD1d64QPseudo };
3685  static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
3686  ARM::VLD4DUPq16EvenPseudo,
3687  ARM::VLD4DUPq32EvenPseudo };
3688  static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
3689  ARM::VLD4DUPq16OddPseudo,
3690  ARM::VLD4DUPq32OddPseudo };
3691  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4,
3692  DOpcodes, QOpcodes0, QOpcodes1);
3693  return;
3694  }
3695 
3696  case Intrinsic::arm_neon_vld2lane: {
3697  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3698  ARM::VLD2LNd16Pseudo,
3699  ARM::VLD2LNd32Pseudo };
3700  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3701  ARM::VLD2LNq32Pseudo };
3702  SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3703  return;
3704  }
3705 
3706  case Intrinsic::arm_neon_vld3lane: {
3707  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3708  ARM::VLD3LNd16Pseudo,
3709  ARM::VLD3LNd32Pseudo };
3710  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3711  ARM::VLD3LNq32Pseudo };
3712  SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3713  return;
3714  }
3715 
3716  case Intrinsic::arm_neon_vld4lane: {
3717  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3718  ARM::VLD4LNd16Pseudo,
3719  ARM::VLD4LNd32Pseudo };
3720  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3721  ARM::VLD4LNq32Pseudo };
3722  SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3723  return;
3724  }
3725 
3726  case Intrinsic::arm_neon_vst1: {
3727  static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3728  ARM::VST1d32, ARM::VST1d64 };
3729  static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3730  ARM::VST1q32, ARM::VST1q64 };
3731  SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3732  return;
3733  }
3734 
3735  case Intrinsic::arm_neon_vst1x2: {
3736  static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3737  ARM::VST1q32, ARM::VST1q64 };
3738  static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
3739  ARM::VST1d16QPseudo,
3740  ARM::VST1d32QPseudo,
3741  ARM::VST1d64QPseudo };
3742  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3743  return;
3744  }
3745 
3746  case Intrinsic::arm_neon_vst1x3: {
3747  static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
3748  ARM::VST1d16TPseudo,
3749  ARM::VST1d32TPseudo,
3750  ARM::VST1d64TPseudo };
3751  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
3752  ARM::VST1q16LowTPseudo_UPD,
3753  ARM::VST1q32LowTPseudo_UPD,
3754  ARM::VST1q64LowTPseudo_UPD };
3755  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
3756  ARM::VST1q16HighTPseudo,
3757  ARM::VST1q32HighTPseudo,
3758  ARM::VST1q64HighTPseudo };
3759  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3760  return;
3761  }
3762 
3763  case Intrinsic::arm_neon_vst1x4: {
3764  static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
3765  ARM::VST1d16QPseudo,
3766  ARM::VST1d32QPseudo,
3767  ARM::VST1d64QPseudo };
3768  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
3769  ARM::VST1q16LowQPseudo_UPD,
3770  ARM::VST1q32LowQPseudo_UPD,
3771  ARM::VST1q64LowQPseudo_UPD };
3772  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
3773  ARM::VST1q16HighQPseudo,
3774  ARM::VST1q32HighQPseudo,
3775  ARM::VST1q64HighQPseudo };
3776  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3777  return;
3778  }
3779 
3780  case Intrinsic::arm_neon_vst2: {
3781  static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3782  ARM::VST2d32, ARM::VST1q64 };
3783  static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3784  ARM::VST2q32Pseudo };
3785  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3786  return;
3787  }
3788 
3789  case Intrinsic::arm_neon_vst3: {
3790  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3791  ARM::VST3d16Pseudo,
3792  ARM::VST3d32Pseudo,
3793  ARM::VST1d64TPseudo };
3794  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3795  ARM::VST3q16Pseudo_UPD,
3796  ARM::VST3q32Pseudo_UPD };
3797  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3798  ARM::VST3q16oddPseudo,
3799  ARM::VST3q32oddPseudo };
3800  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3801  return;
3802  }
3803 
3804  case Intrinsic::arm_neon_vst4: {
3805  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3806  ARM::VST4d16Pseudo,
3807  ARM::VST4d32Pseudo,
3808  ARM::VST1d64QPseudo };
3809  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3810  ARM::VST4q16Pseudo_UPD,
3811  ARM::VST4q32Pseudo_UPD };
3812  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3813  ARM::VST4q16oddPseudo,
3814  ARM::VST4q32oddPseudo };
3815  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3816  return;
3817  }
3818 
3819  case Intrinsic::arm_neon_vst2lane: {
3820  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3821  ARM::VST2LNd16Pseudo,
3822  ARM::VST2LNd32Pseudo };
3823  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3824  ARM::VST2LNq32Pseudo };
3825  SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3826  return;
3827  }
3828 
3829  case Intrinsic::arm_neon_vst3lane: {
3830  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3831  ARM::VST3LNd16Pseudo,
3832  ARM::VST3LNd32Pseudo };
3833  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3834  ARM::VST3LNq32Pseudo };
3835  SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3836  return;
3837  }
3838 
3839  case Intrinsic::arm_neon_vst4lane: {
3840  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3841  ARM::VST4LNd16Pseudo,
3842  ARM::VST4LNd32Pseudo };
3843  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3844  ARM::VST4LNq32Pseudo };
3845  SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3846  return;
3847  }
3848  }
3849  break;
3850  }
3851 
3852  case ISD::ATOMIC_CMP_SWAP:
3853  SelectCMP_SWAP(N);
3854  return;
3855  }
3856 
3857  SelectCode(N);
3858 }
3859 
3860 // Inspect a register string of the form
3861 // cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
3862 // cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
3863 // and obtain the integer operands from them, adding these operands to the
3864 // provided vector.
3866  SelectionDAG *CurDAG,
3867  const SDLoc &DL,
3868  std::vector<SDValue> &Ops) {
3870  RegString.split(Fields, ':');
3871 
3872  if (Fields.size() > 1) {
3873  bool AllIntFields = true;
3874 
3875  for (StringRef Field : Fields) {
3876  // Need to trim out leading 'cp' characters and get the integer field.
3877  unsigned IntField;
3878  AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
3879  Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
3880  }
3881 
3882  assert(AllIntFields &&
3883  "Unexpected non-integer value in special register string.");
3884  }
3885 }
3886 
3887 // Maps a Banked Register string to its mask value. The mask value returned is
3888 // for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
3889 // mask operand, which expresses which register is to be used, e.g. r8, and in
3890 // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
3891 // was invalid.
3892 static inline int getBankedRegisterMask(StringRef RegString) {
3893  auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
3894  if (!TheReg)
3895  return -1;
3896  return TheReg->Encoding;
3897 }
3898 
3899 // The flags here are common to those allowed for apsr in the A class cores and
3900 // those allowed for the special registers in the M class cores. Returns a
3901 // value representing which flags were present, -1 if invalid.
3902 static inline int getMClassFlagsMask(StringRef Flags) {
3903  return StringSwitch<int>(Flags)
3904  .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
3905  // correct when flags are not permitted
3906  .Case("g", 0x1)
3907  .Case("nzcvq", 0x2)
3908  .Case("nzcvqg", 0x3)
3909  .Default(-1);
3910 }
3911 
3912 // Maps MClass special registers string to its value for use in the
3913 // t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
3914 // Returns -1 to signify that the string was invalid.
3915 static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
3916  auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
3917  const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
3918  if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
3919  return -1;
3920  return (int)(TheReg->Encoding & 0xFFF); // SYSm value
3921 }
3922 
3924  // The mask operand contains the special register (R Bit) in bit 4, whether
3925  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
3926  // bits 3-0 contains the fields to be accessed in the special register, set by
3927  // the flags provided with the register.
3928  int Mask = 0;
3929  if (Reg == "apsr") {
3930  // The flags permitted for apsr are the same flags that are allowed in
3931  // M class registers. We get the flag value and then shift the flags into
3932  // the correct place to combine with the mask.
3933  Mask = getMClassFlagsMask(Flags);
3934  if (Mask == -1)
3935  return -1;
3936  return Mask << 2;
3937  }
3938 
3939  if (Reg != "cpsr" && Reg != "spsr") {
3940  return -1;
3941  }
3942 
3943  // This is the same as if the flags were "fc"
3944  if (Flags.empty() || Flags == "all")
3945  return Mask | 0x9;
3946 
3947  // Inspect the supplied flags string and set the bits in the mask for
3948  // the relevant and valid flags allowed for cpsr and spsr.
3949  for (char Flag : Flags) {
3950  int FlagVal;
3951  switch (Flag) {
3952  case 'c':
3953  FlagVal = 0x1;
3954  break;
3955  case 'x':
3956  FlagVal = 0x2;
3957  break;
3958  case 's':
3959  FlagVal = 0x4;
3960  break;
3961  case 'f':
3962  FlagVal = 0x8;
3963  break;
3964  default:
3965  FlagVal = 0;
3966  }
3967 
3968  // This avoids allowing strings where the same flag bit appears twice.
3969  if (!FlagVal || (Mask & FlagVal))
3970  return -1;
3971  Mask |= FlagVal;
3972  }
3973 
3974  // If the register is spsr then we need to set the R bit.
3975  if (Reg == "spsr")
3976  Mask |= 0x10;
3977 
3978  return Mask;
3979 }
3980 
3981 // Lower the read_register intrinsic to ARM specific DAG nodes
3982 // using the supplied metadata string to select the instruction node to use
3983 // and the registers/masks to construct as operands for the node.
3984 bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
3985  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
3986  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
3987  bool IsThumb2 = Subtarget->isThumb2();
3988  SDLoc DL(N);
3989 
3990  std::vector<SDValue> Ops;
3991  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
3992 
3993  if (!Ops.empty()) {
3994  // If the special register string was constructed of fields (as defined
3995  // in the ACLE) then need to lower to MRC node (32 bit) or
3996  // MRRC node(64 bit), we can make the distinction based on the number of
3997  // operands we have.
3998  unsigned Opcode;
3999  SmallVector<EVT, 3> ResTypes;
4000  if (Ops.size() == 5){
4001  Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
4002  ResTypes.append({ MVT::i32, MVT::Other });
4003  } else {
4004  assert(Ops.size() == 3 &&
4005  "Invalid number of fields in special register string.");
4006  Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
4007  ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
4008  }
4009 
4010  Ops.push_back(getAL(CurDAG, DL));
4011  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4012  Ops.push_back(N->getOperand(0));
4013  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
4014  return true;
4015  }
4016 
4017  std::string SpecialReg = RegString->getString().lower();
4018 
4019  int BankedReg = getBankedRegisterMask(SpecialReg);
4020  if (BankedReg != -1) {
4021  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
4022  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4023  N->getOperand(0) };
4024  ReplaceNode(
4025  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
4026  DL, MVT::i32, MVT::Other, Ops));
4027  return true;
4028  }
4029 
4030  // The VFP registers are read by creating SelectionDAG nodes with opcodes
4031  // corresponding to the register that is being read from. So we switch on the
4032  // string to find which opcode we need to use.
4033  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4034  .Case("fpscr", ARM::VMRS)
4035  .Case("fpexc", ARM::VMRS_FPEXC)
4036  .Case("fpsid", ARM::VMRS_FPSID)
4037  .Case("mvfr0", ARM::VMRS_MVFR0)
4038  .Case("mvfr1", ARM::VMRS_MVFR1)
4039  .Case("mvfr2", ARM::VMRS_MVFR2)
4040  .Case("fpinst", ARM::VMRS_FPINST)
4041  .Case("fpinst2", ARM::VMRS_FPINST2)
4042  .Default(0);
4043 
4044  // If an opcode was found then we can lower the read to a VFP instruction.
4045  if (Opcode) {
4046  if (!Subtarget->hasVFP2Base())
4047  return false;
4048  if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8Base())
4049  return false;
4050 
4051  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4052  N->getOperand(0) };
4053  ReplaceNode(N,
4054  CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
4055  return true;
4056  }
4057 
4058  // If the target is M Class then need to validate that the register string
4059  // is an acceptable value, so check that a mask can be constructed from the
4060  // string.
4061  if (Subtarget->isMClass()) {
4062  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4063  if (SYSmValue == -1)
4064  return false;
4065 
4066  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4067  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4068  N->getOperand(0) };
4069  ReplaceNode(
4070  N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
4071  return true;
4072  }
4073 
4074  // Here we know the target is not M Class so we need to check if it is one
4075  // of the remaining possible values which are apsr, cpsr or spsr.
4076  if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
4077  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4078  N->getOperand(0) };
4079  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
4080  DL, MVT::i32, MVT::Other, Ops));
4081  return true;
4082  }
4083 
4084  if (SpecialReg == "spsr") {
4085  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4086  N->getOperand(0) };
4087  ReplaceNode(
4088  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
4089  MVT::i32, MVT::Other, Ops));
4090  return true;
4091  }
4092 
4093  return false;
4094 }
4095 
4096 // Lower the write_register intrinsic to ARM specific DAG nodes
4097 // using the supplied metadata string to select the instruction node to use
4098 // and the registers/masks to use in the nodes
4099 bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
4100  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
4101  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
4102  bool IsThumb2 = Subtarget->isThumb2();
4103  SDLoc DL(N);
4104 
4105  std::vector<SDValue> Ops;
4106  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
4107 
4108  if (!Ops.empty()) {
4109  // If the special register string was constructed of fields (as defined
4110  // in the ACLE) then need to lower to MCR node (32 bit) or
4111  // MCRR node(64 bit), we can make the distinction based on the number of
4112  // operands we have.
4113  unsigned Opcode;
4114  if (Ops.size() == 5) {
4115  Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
4116  Ops.insert(Ops.begin()+2, N->getOperand(2));
4117  } else {
4118  assert(Ops.size() == 3 &&
4119  "Invalid number of fields in special register string.");
4120  Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
4121  SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
4122  Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
4123  }
4124 
4125  Ops.push_back(getAL(CurDAG, DL));
4126  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4127  Ops.push_back(N->getOperand(0));
4128 
4129  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4130  return true;
4131  }
4132 
4133  std::string SpecialReg = RegString->getString().lower();
4134  int BankedReg = getBankedRegisterMask(SpecialReg);
4135  if (BankedReg != -1) {
4136  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
4137  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4138  N->getOperand(0) };
4139  ReplaceNode(
4140  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
4141  DL, MVT::Other, Ops));
4142  return true;
4143  }
4144 
4145  // The VFP registers are written to by creating SelectionDAG nodes with
4146  // opcodes corresponding to the register that is being written. So we switch
4147  // on the string to find which opcode we need to use.
4148  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4149  .Case("fpscr", ARM::VMSR)
4150  .Case("fpexc", ARM::VMSR_FPEXC)
4151  .Case("fpsid", ARM::VMSR_FPSID)
4152  .Case("fpinst", ARM::VMSR_FPINST)
4153  .Case("fpinst2", ARM::VMSR_FPINST2)
4154  .Default(0);
4155 
4156  if (Opcode) {
4157  if (!Subtarget->hasVFP2Base())
4158  return false;
4159  Ops = { N->getOperand(2), getAL(CurDAG, DL),
4160  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4161  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4162  return true;
4163  }
4164 
4165  std::pair<StringRef, StringRef> Fields;
4166  Fields = StringRef(SpecialReg).rsplit('_');
4167  std::string Reg = Fields.first.str();
4168  StringRef Flags = Fields.second;
4169 
4170  // If the target was M Class then need to validate the special register value
4171  // and retrieve the mask for use in the instruction node.
4172  if (Subtarget->isMClass()) {
4173  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4174  if (SYSmValue == -1)
4175  return false;
4176 
4177  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4178  N->getOperand(2), getAL(CurDAG, DL),
4179  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4180  ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
4181  return true;
4182  }
4183 
4184  // We then check to see if a valid mask can be constructed for one of the
4185  // register string values permitted for the A and R class cores. These values
4186  // are apsr, spsr and cpsr; these are also valid on older cores.
4187  int Mask = getARClassRegisterMask(Reg, Flags);
4188  if (Mask != -1) {
4189  Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
4190  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4191  N->getOperand(0) };
4192  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
4193  DL, MVT::Other, Ops));
4194  return true;
4195  }
4196 
4197  return false;
4198 }
4199 
4200 bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
4201  std::vector<SDValue> AsmNodeOperands;
4202  unsigned Flag, Kind;
4203  bool Changed = false;
4204  unsigned NumOps = N->getNumOperands();
4205 
4206  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
4207  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
4208  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
4209  // respectively. Since there is no constraint to explicitly specify a
4210  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
4211  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
4212  // them into a GPRPair.
4213 
4214  SDLoc dl(N);
4215  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
4216  : SDValue(nullptr,0);
4217 
4218  SmallVector<bool, 8> OpChanged;
4219  // Glue node will be appended late.
4220  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
4221  SDValue op = N->getOperand(i);
4222  AsmNodeOperands.push_back(op);
4223 
4225  continue;
4226 
4227  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
4228  Flag = C->getZExtValue();
4229  Kind = InlineAsm::getKind(Flag);
4230  }
4231  else
4232  continue;
4233 
4234  // Immediate operands to inline asm in the SelectionDAG are modeled with
4235  // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
4236  // the second is a constant with the value of the immediate. If we get here
4237  // and we have a Kind_Imm, skip the next operand, and continue.
4238  if (Kind == InlineAsm::Kind_Imm) {
4239  SDValue op = N->getOperand(++i);
4240  AsmNodeOperands.push_back(op);
4241  continue;
4242  }
4243 
4244  unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
4245  if (NumRegs)
4246  OpChanged.push_back(false);
4247 
4248  unsigned DefIdx = 0;
4249  bool IsTiedToChangedOp = false;
4250  // If it's a use that is tied with a previous def, it has no
4251  // reg class constraint.
4252  if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
4253  IsTiedToChangedOp = OpChanged[DefIdx];
4254 
4255  // Memory operands to inline asm in the SelectionDAG are modeled with two
4256  // operands: a constant of value InlineAsm::Kind_Mem followed by the input
4257  // operand. If we get here and we have a Kind_Mem, skip the next operand (so
4258  // it doesn't get misinterpreted), and continue. We do this here because
4259  // it's important to update the OpChanged array correctly before moving on.
4260  if (Kind == InlineAsm::Kind_Mem) {
4261  SDValue op = N->getOperand(++i);
4262  AsmNodeOperands.push_back(op);
4263  continue;
4264  }
4265 
4266  if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
4268  continue;
4269 
4270  unsigned RC;
4271  bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
4272  if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
4273  || NumRegs != 2)
4274  continue;
4275 
4276  assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
4277  SDValue V0 = N->getOperand(i+1);
4278  SDValue V1 = N->getOperand(i+2);
4279  unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
4280  unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
4281  SDValue PairedReg;
4282  MachineRegisterInfo &MRI = MF->getRegInfo();
4283 
4284  if (Kind == InlineAsm::Kind_RegDef ||
4286  // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
4287  // the original GPRs.
4288 
4289  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4290  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4291  SDValue Chain = SDValue(N,0);
4292 
4293  SDNode *GU = N->getGluedUser();
4294  SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4295  Chain.getValue(1));
4296 
4297  // Extract values from a GPRPair reg and copy to the original GPR reg.
4298  SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4299  RegCopy);
4300  SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4301  RegCopy);
4302  SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4303  RegCopy.getValue(1));
4304  SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4305 
4306  // Update the original glue user.
4307  std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4308  Ops.push_back(T1.getValue(1));
4309  CurDAG->UpdateNodeOperands(GU, Ops);
4310  }
4311  else {
4312  // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a
4313  // GPRPair and then pass the GPRPair to the inline asm.
4314  SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4315 
4316  // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
4317  SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4318  Chain.getValue(1));
4319  SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4320  T0.getValue(1));
4321  SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4322 
4323  // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
4324  // i32 VRs of inline asm with it.
4325  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4326  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4327  Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
4328 
4329  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
4330  Glue = Chain.getValue(1);
4331  }
4332 
4333  Changed = true;
4334 
4335  if(PairedReg.getNode()) {
4336  OpChanged[OpChanged.size() -1 ] = true;
4337  Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
4338  if (IsTiedToChangedOp)
4339  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
4340  else
4341  Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
4342  // Replace the current flag.
4343  AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
4344  Flag, dl, MVT::i32);
4345  // Add the new register node and skip the original two GPRs.
4346  AsmNodeOperands.push_back(PairedReg);
4347  // Skip the next two GPRs.
4348  i += 2;
4349  }
4350  }
4351 
4352  if (Glue.getNode())
4353  AsmNodeOperands.push_back(Glue);
4354  if (!Changed)
4355  return false;
4356 
4357  SDValue New = CurDAG->getNode(N->getOpcode(), SDLoc(N),
4358  CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
4359  New->setNodeId(-1);
4360  ReplaceNode(N, New.getNode());
4361  return true;
4362 }
4363 
4364 
4365 bool ARMDAGToDAGISel::
4366 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4367  std::vector<SDValue> &OutOps) {
4368  switch(ConstraintID) {
4369  default:
4370  llvm_unreachable("Unexpected asm memory constraint");
4372  // FIXME: It seems strange that 'i' is needed here since it's supposed to
4373  // be an immediate and not a memory constraint.
4385  // Require the address to be in a register. That is safe for all ARM
4386  // variants and it is hard to do anything much smarter without knowing
4387  // how the operand is used.
4388  OutOps.push_back(Op);
4389  return false;
4390  }
4391  return true;
4392 }
4393 
4394 /// createARMISelDag - This pass converts a legalized DAG into a
4395 /// ARM-specific DAG, ready for instruction scheduling.
4396 ///
4398  CodeGenOpt::Level OptLevel) {
4399  return new ARMDAGToDAGISel(TM, OptLevel);
4400 }
uint64_t CallInst * C
bool useMovt() const
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
EVT getValueType() const
Return the ValueType of the referenced return value.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const SDValue & getOffset() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isThumb() const
Definition: ARMSubtarget.h:749
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Various leaf nodes.
Definition: ISDOpcodes.h:59
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:268
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:164
const SDValue & getValue() const
SDVTList getVTList() const
static Optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
unsigned Reg
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:130
unsigned getAlignment() const
static bool isVSTfixed(unsigned Opc)
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:729
bool isSOImmTwoPartVal(unsigned V)
isSOImmTwoPartVal - Return true if the specified value can be obtained by or&#39;ing together two SOImmVa...
bool hasV6Ops() const
Definition: ARMSubtarget.h:566
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1068
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
bool isThumb1Only() const
Definition: ARMSubtarget.h:751
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
#define op(i)
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
static bool isThumb(const MCSubtargetInfo &STI)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:341
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1508
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:158
bool hasV8MBaselineOps() const
Definition: ARMSubtarget.h:577
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:119
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.h:1631
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isFpMLxInstruction(unsigned Opcode) const
isFpMLxInstruction - Return true if the specified opcode is a fp MLA / MLS instruction.
A description of a memory reference used in the backend.
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:299
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:434
Base class for LoadSDNode and StoreSDNode.
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:417
A Use represents the edge between a Value definition and its users.
Definition: Use.h:55
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:169
const MDNode * getMD() const
op_iterator op_end() const
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:695
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
bool hasDSP() const
Definition: ARMSubtarget.h:665
static bool isVLDfixed(unsigned Opc)
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
bool hasV6T2Ops() const
Definition: ARMSubtarget.h:569
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:582
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
This class is used to represent ISD::STORE nodes.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:117
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
const SDValue & getBasePtr() const
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:165
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:84
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
Container class for subtarget features.
static int getBankedRegisterMask(StringRef RegString)
unsigned countPopulation() const
Count the number of bits set.
Definition: APInt.h:1657
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
bool isMClass() const
Definition: ARMSubtarget.h:754
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:272
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
const SDValue & getOperand(unsigned Num) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:335
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
static unsigned getKind(unsigned Flags)
Definition: InlineAsm.h:324
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:284
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
self_iterator getIterator()
Definition: ilist_node.h:81
bool hasVFP2Base() const
Definition: ARMSubtarget.h:603
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:827
Extended Value Type.
Definition: ValueTypes.h:33
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:188
size_t size() const
Definition: SmallVector.h:52
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
bool isUnindexed() const
Return true if this is NOT a pre/post inc/dec load/store.
Iterator for intrusive lists based on ilist_node.
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
bool hasFPARMv8Base() const
Definition: ARMSubtarget.h:606
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:264
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:696
An SDNode that represents everything that will be needed to construct a MachineInstr.
This is an abstract virtual class for memory operations.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
Represents one node in the SelectionDAG.
#define NC
Definition: regutils.h:42
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:538
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:72
EVT getMemoryVT() const
Return the type of the in-memory value.
Class for arbitrary precision integers.
Definition: APInt.h:69
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:405
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:387
static bool shouldUseZeroOffsetLdSt(SDValue N)
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getNodeId() const
Return the unique node id.
static bool hasRegClassConstraint(unsigned Flag, unsigned &RC)
hasRegClassConstraint - Returns true if the flag contains a register class constraint.
Definition: InlineAsm.h:350
bool isThumb2() const
Definition: ARMSubtarget.h:752
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: ValueTypes.h:176
const SDValue & getValue() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:411
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:642
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:174
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOpt::Level OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
bool hasVMLxHazards() const
Definition: ARMSubtarget.h:648
LLVM_NODISCARD std::string lower() const
Definition: StringRef.cpp:107
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
INLINEASM_BR - Terminator version of inline asm. Used by asm-goto.
Definition: ISDOpcodes.h:698
bool isSwift() const
Definition: ARMSubtarget.h:594
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:477
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:204
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition: MathExtras.h:416
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
A single uniqued string.
Definition: Metadata.h:603
static int getMClassFlagsMask(StringRef Flags)
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1595
const SDValue & getOperand(unsigned i) const
uint64_t getZExtValue() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void setObjectAlignment(int ObjectIdx, unsigned Align)
setObjectAlignment - Change the alignment of the specified stack object.
bool hasThumb2() const
Definition: ARMSubtarget.h:753
bool isLikeA9() const
Definition: ARMSubtarget.h:596
#define T1
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:287
This file describes how to lower LLVM code to machine code.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:950
This class is used to represent ISD::LOAD nodes.