LLVM  8.0.0svn
ARMISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the ARM target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ARM.h"
15 #include "ARMBaseInstrInfo.h"
16 #include "ARMTargetMachine.h"
18 #include "Utils/ARMBaseInfo.h"
19 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/IR/CallingConv.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/IR/Intrinsics.h"
32 #include "llvm/IR/LLVMContext.h"
34 #include "llvm/Support/Debug.h"
37 
38 using namespace llvm;
39 
40 #define DEBUG_TYPE "arm-isel"
41 
42 static cl::opt<bool>
43 DisableShifterOp("disable-shifter-op", cl::Hidden,
44  cl::desc("Disable isel of shifter-op"),
45  cl::init(false));
46 
47 //===--------------------------------------------------------------------===//
48 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
49 /// instructions for SelectionDAG operations.
50 ///
51 namespace {
52 
53 class ARMDAGToDAGISel : public SelectionDAGISel {
54  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
55  /// make the right decision when generating code for different targets.
56  const ARMSubtarget *Subtarget;
57 
58 public:
59  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
60  : SelectionDAGISel(tm, OptLevel) {}
61 
62  bool runOnMachineFunction(MachineFunction &MF) override {
63  // Reset the subtarget each time through.
64  Subtarget = &MF.getSubtarget<ARMSubtarget>();
66  return true;
67  }
68 
69  StringRef getPassName() const override { return "ARM Instruction Selection"; }
70 
71  void PreprocessISelDAG() override;
72 
73  /// getI32Imm - Return a target constant of type i32 with the specified
74  /// value.
75  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
76  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
77  }
78 
79  void Select(SDNode *N) override;
80 
81  bool hasNoVMLxHazardUse(SDNode *N) const;
82  bool isShifterOpProfitable(const SDValue &Shift,
83  ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
84  bool SelectRegShifterOperand(SDValue N, SDValue &A,
85  SDValue &B, SDValue &C,
86  bool CheckProfitability = true);
87  bool SelectImmShifterOperand(SDValue N, SDValue &A,
88  SDValue &B, bool CheckProfitability = true);
89  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
90  SDValue &B, SDValue &C) {
91  // Don't apply the profitability check
92  return SelectRegShifterOperand(N, A, B, C, false);
93  }
94  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
95  SDValue &B) {
96  // Don't apply the profitability check
97  return SelectImmShifterOperand(N, A, B, false);
98  }
99 
100  bool SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out);
101 
102  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
103  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
104 
105  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
106  const ConstantSDNode *CN = cast<ConstantSDNode>(N);
107  Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
108  Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
109  return true;
110  }
111 
112  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
113  SDValue &Offset, SDValue &Opc);
114  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
115  SDValue &Offset, SDValue &Opc);
116  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
117  SDValue &Offset, SDValue &Opc);
118  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
119  bool SelectAddrMode3(SDValue N, SDValue &Base,
120  SDValue &Offset, SDValue &Opc);
121  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
122  SDValue &Offset, SDValue &Opc);
123  bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
124  int Lwb, int Upb, bool FP16);
125  bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
126  bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
127  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
128  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
129 
130  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
131 
132  // Thumb Addressing Modes:
133  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
134  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
135  SDValue &OffImm);
136  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
137  SDValue &OffImm);
138  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
139  SDValue &OffImm);
140  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
141  SDValue &OffImm);
142  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
143 
144  // Thumb 2 Addressing Modes:
145  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
146  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
147  SDValue &OffImm);
148  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
149  SDValue &OffImm);
150  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
151  SDValue &OffReg, SDValue &ShImm);
152  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
153 
154  inline bool is_so_imm(unsigned Imm) const {
155  return ARM_AM::getSOImmVal(Imm) != -1;
156  }
157 
158  inline bool is_so_imm_not(unsigned Imm) const {
159  return ARM_AM::getSOImmVal(~Imm) != -1;
160  }
161 
162  inline bool is_t2_so_imm(unsigned Imm) const {
163  return ARM_AM::getT2SOImmVal(Imm) != -1;
164  }
165 
166  inline bool is_t2_so_imm_not(unsigned Imm) const {
167  return ARM_AM::getT2SOImmVal(~Imm) != -1;
168  }
169 
170  // Include the pieces autogenerated from the target description.
171 #include "ARMGenDAGISel.inc"
172 
173 private:
174  void transferMemOperands(SDNode *Src, SDNode *Dst);
175 
176  /// Indexed (pre/post inc/dec) load matching code for ARM.
177  bool tryARMIndexedLoad(SDNode *N);
178  bool tryT1IndexedLoad(SDNode *N);
179  bool tryT2IndexedLoad(SDNode *N);
180 
181  /// SelectVLD - Select NEON load intrinsics. NumVecs should be
182  /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
183  /// loads of D registers and even subregs and odd subregs of Q registers.
184  /// For NumVecs <= 2, QOpcodes1 is not used.
185  void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
186  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
187  const uint16_t *QOpcodes1);
188 
189  /// SelectVST - Select NEON store intrinsics. NumVecs should
190  /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
191  /// stores of D registers and even subregs and odd subregs of Q registers.
192  /// For NumVecs <= 2, QOpcodes1 is not used.
193  void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
194  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
195  const uint16_t *QOpcodes1);
196 
197  /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
198  /// be 2, 3 or 4. The opcode arrays specify the instructions used for
199  /// load/store of D registers and Q registers.
200  void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
201  unsigned NumVecs, const uint16_t *DOpcodes,
202  const uint16_t *QOpcodes);
203 
204  /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
205  /// should be 1, 2, 3 or 4. The opcode array specifies the instructions used
206  /// for loading D registers.
207  void SelectVLDDup(SDNode *N, bool IsIntrinsic, bool isUpdating,
208  unsigned NumVecs, const uint16_t *DOpcodes,
209  const uint16_t *QOpcodes0 = nullptr,
210  const uint16_t *QOpcodes1 = nullptr);
211 
212  /// Try to select SBFX/UBFX instructions for ARM.
213  bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
214 
215  // Select special operations if node forms integer ABS pattern
216  bool tryABSOp(SDNode *N);
217 
218  bool tryReadRegister(SDNode *N);
219  bool tryWriteRegister(SDNode *N);
220 
221  bool tryInlineAsm(SDNode *N);
222 
223  void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
224 
225  void SelectCMP_SWAP(SDNode *N);
226 
227  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
228  /// inline asm expressions.
229  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
230  std::vector<SDValue> &OutOps) override;
231 
232  // Form pairs of consecutive R, S, D, or Q registers.
234  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
235  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
236  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
237 
238  // Form sequences of 4 consecutive S, D, or Q registers.
239  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
240  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
241  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
242 
243  // Get the alignment operand for a NEON VLD or VST instruction.
244  SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
245  bool is64BitVector);
246 
247  /// Returns the number of instructions required to materialize the given
248  /// constant in a register, or 3 if a literal pool load is needed.
249  unsigned ConstantMaterializationCost(unsigned Val) const;
250 
251  /// Checks if N is a multiplication by a constant where we can extract out a
252  /// power of two from the constant so that it can be used in a shift, but only
253  /// if it simplifies the materialization of the constant. Returns true if it
254  /// is, and assigns to PowerOfTwo the power of two that should be extracted
255  /// out and to NewMulConst the new constant to be multiplied by.
256  bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
257  unsigned &PowerOfTwo, SDValue &NewMulConst) const;
258 
259  /// Replace N with M in CurDAG, in a way that also ensures that M gets
260  /// selected when N would have been selected.
261  void replaceDAGValue(const SDValue &N, SDValue M);
262 };
263 }
264 
265 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
266 /// operand. If so Imm will receive the 32-bit value.
267 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
268  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
269  Imm = cast<ConstantSDNode>(N)->getZExtValue();
270  return true;
271  }
272  return false;
273 }
274 
275 // isInt32Immediate - This method tests to see if a constant operand.
276 // If so Imm will receive the 32 bit value.
277 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
278  return isInt32Immediate(N.getNode(), Imm);
279 }
280 
281 // isOpcWithIntImmediate - This method tests to see if the node is a specific
282 // opcode and that it has a immediate integer right operand.
283 // If so Imm will receive the 32 bit value.
284 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
285  return N->getOpcode() == Opc &&
286  isInt32Immediate(N->getOperand(1).getNode(), Imm);
287 }
288 
289 /// Check whether a particular node is a constant value representable as
290 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
291 ///
292 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
293 static bool isScaledConstantInRange(SDValue Node, int Scale,
294  int RangeMin, int RangeMax,
295  int &ScaledConstant) {
296  assert(Scale > 0 && "Invalid scale!");
297 
298  // Check that this is a constant.
299  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
300  if (!C)
301  return false;
302 
303  ScaledConstant = (int) C->getZExtValue();
304  if ((ScaledConstant % Scale) != 0)
305  return false;
306 
307  ScaledConstant /= Scale;
308  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
309 }
310 
311 void ARMDAGToDAGISel::PreprocessISelDAG() {
312  if (!Subtarget->hasV6T2Ops())
313  return;
314 
315  bool isThumb2 = Subtarget->isThumb();
316  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
317  E = CurDAG->allnodes_end(); I != E; ) {
318  SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
319 
320  if (N->getOpcode() != ISD::ADD)
321  continue;
322 
323  // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
324  // leading zeros, followed by consecutive set bits, followed by 1 or 2
325  // trailing zeros, e.g. 1020.
326  // Transform the expression to
327  // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
328  // of trailing zeros of c2. The left shift would be folded as an shifter
329  // operand of 'add' and the 'and' and 'srl' would become a bits extraction
330  // node (UBFX).
331 
332  SDValue N0 = N->getOperand(0);
333  SDValue N1 = N->getOperand(1);
334  unsigned And_imm = 0;
335  if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
336  if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
337  std::swap(N0, N1);
338  }
339  if (!And_imm)
340  continue;
341 
342  // Check if the AND mask is an immediate of the form: 000.....1111111100
343  unsigned TZ = countTrailingZeros(And_imm);
344  if (TZ != 1 && TZ != 2)
345  // Be conservative here. Shifter operands aren't always free. e.g. On
346  // Swift, left shifter operand of 1 / 2 for free but others are not.
347  // e.g.
348  // ubfx r3, r1, #16, #8
349  // ldr.w r3, [r0, r3, lsl #2]
350  // vs.
351  // mov.w r9, #1020
352  // and.w r2, r9, r1, lsr #14
353  // ldr r2, [r0, r2]
354  continue;
355  And_imm >>= TZ;
356  if (And_imm & (And_imm + 1))
357  continue;
358 
359  // Look for (and (srl X, c1), c2).
360  SDValue Srl = N1.getOperand(0);
361  unsigned Srl_imm = 0;
362  if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
363  (Srl_imm <= 2))
364  continue;
365 
366  // Make sure first operand is not a shifter operand which would prevent
367  // folding of the left shift.
368  SDValue CPTmp0;
369  SDValue CPTmp1;
370  SDValue CPTmp2;
371  if (isThumb2) {
372  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
373  continue;
374  } else {
375  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
376  SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
377  continue;
378  }
379 
380  // Now make the transformation.
381  Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
382  Srl.getOperand(0),
383  CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
384  MVT::i32));
385  N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
386  Srl,
387  CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
388  N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
389  N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
390  CurDAG->UpdateNodeOperands(N, N0, N1);
391  }
392 }
393 
394 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
395 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
396 /// least on current ARM implementations) which should be avoidded.
397 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
398  if (OptLevel == CodeGenOpt::None)
399  return true;
400 
401  if (!Subtarget->hasVMLxHazards())
402  return true;
403 
404  if (!N->hasOneUse())
405  return false;
406 
407  SDNode *Use = *N->use_begin();
408  if (Use->getOpcode() == ISD::CopyToReg)
409  return true;
410  if (Use->isMachineOpcode()) {
411  const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
412  CurDAG->getSubtarget().getInstrInfo());
413 
414  const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
415  if (MCID.mayStore())
416  return true;
417  unsigned Opcode = MCID.getOpcode();
418  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
419  return true;
420  // vmlx feeding into another vmlx. We actually want to unfold
421  // the use later in the MLxExpansion pass. e.g.
422  // vmla
423  // vmla (stall 8 cycles)
424  //
425  // vmul (5 cycles)
426  // vadd (5 cycles)
427  // vmla
428  // This adds up to about 18 - 19 cycles.
429  //
430  // vmla
431  // vmul (stall 4 cycles)
432  // vadd adds up to about 14 cycles.
433  return TII->isFpMLxInstruction(Opcode);
434  }
435 
436  return false;
437 }
438 
439 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
440  ARM_AM::ShiftOpc ShOpcVal,
441  unsigned ShAmt) {
442  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
443  return true;
444  if (Shift.hasOneUse())
445  return true;
446  // R << 2 is free.
447  return ShOpcVal == ARM_AM::lsl &&
448  (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
449 }
450 
451 unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
452  if (Subtarget->isThumb()) {
453  if (Val <= 255) return 1; // MOV
454  if (Subtarget->hasV6T2Ops() &&
455  (Val <= 0xffff || ARM_AM::getT2SOImmValSplatVal(Val) != -1))
456  return 1; // MOVW
457  if (Val <= 510) return 2; // MOV + ADDi8
458  if (~Val <= 255) return 2; // MOV + MVN
459  if (ARM_AM::isThumbImmShiftedVal(Val)) return 2; // MOV + LSL
460  } else {
461  if (ARM_AM::getSOImmVal(Val) != -1) return 1; // MOV
462  if (ARM_AM::getSOImmVal(~Val) != -1) return 1; // MVN
463  if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
464  if (ARM_AM::isSOImmTwoPartVal(Val)) return 2; // two instrs
465  }
466  if (Subtarget->useMovt(*MF)) return 2; // MOVW + MOVT
467  return 3; // Literal pool load
468 }
469 
470 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
471  unsigned MaxShift,
472  unsigned &PowerOfTwo,
473  SDValue &NewMulConst) const {
474  assert(N.getOpcode() == ISD::MUL);
475  assert(MaxShift > 0);
476 
477  // If the multiply is used in more than one place then changing the constant
478  // will make other uses incorrect, so don't.
479  if (!N.hasOneUse()) return false;
480  // Check if the multiply is by a constant
482  if (!MulConst) return false;
483  // If the constant is used in more than one place then modifying it will mean
484  // we need to materialize two constants instead of one, which is a bad idea.
485  if (!MulConst->hasOneUse()) return false;
486  unsigned MulConstVal = MulConst->getZExtValue();
487  if (MulConstVal == 0) return false;
488 
489  // Find the largest power of 2 that MulConstVal is a multiple of
490  PowerOfTwo = MaxShift;
491  while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
492  --PowerOfTwo;
493  if (PowerOfTwo == 0) return false;
494  }
495 
496  // Only optimise if the new cost is better
497  unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
498  NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
499  unsigned OldCost = ConstantMaterializationCost(MulConstVal);
500  unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
501  return NewCost < OldCost;
502 }
503 
504 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
505  CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
506  ReplaceUses(N, M);
507 }
508 
509 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
510  SDValue &BaseReg,
511  SDValue &Opc,
512  bool CheckProfitability) {
513  if (DisableShifterOp)
514  return false;
515 
516  // If N is a multiply-by-constant and it's profitable to extract a shift and
517  // use it in a shifted operand do so.
518  if (N.getOpcode() == ISD::MUL) {
519  unsigned PowerOfTwo = 0;
520  SDValue NewMulConst;
521  if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
522  HandleSDNode Handle(N);
523  SDLoc Loc(N);
524  replaceDAGValue(N.getOperand(1), NewMulConst);
525  BaseReg = Handle.getValue();
526  Opc = CurDAG->getTargetConstant(
527  ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
528  return true;
529  }
530  }
531 
533 
534  // Don't match base register only case. That is matched to a separate
535  // lower complexity pattern with explicit register operand.
536  if (ShOpcVal == ARM_AM::no_shift) return false;
537 
538  BaseReg = N.getOperand(0);
539  unsigned ShImmVal = 0;
541  if (!RHS) return false;
542  ShImmVal = RHS->getZExtValue() & 31;
543  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
544  SDLoc(N), MVT::i32);
545  return true;
546 }
547 
548 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
549  SDValue &BaseReg,
550  SDValue &ShReg,
551  SDValue &Opc,
552  bool CheckProfitability) {
553  if (DisableShifterOp)
554  return false;
555 
557 
558  // Don't match base register only case. That is matched to a separate
559  // lower complexity pattern with explicit register operand.
560  if (ShOpcVal == ARM_AM::no_shift) return false;
561 
562  BaseReg = N.getOperand(0);
563  unsigned ShImmVal = 0;
565  if (RHS) return false;
566 
567  ShReg = N.getOperand(1);
568  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
569  return false;
570  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
571  SDLoc(N), MVT::i32);
572  return true;
573 }
574 
575 // Determine whether an ISD::OR's operands are suitable to turn the operation
576 // into an addition, which often has more compact encodings.
577 bool ARMDAGToDAGISel::SelectAddLikeOr(SDNode *Parent, SDValue N, SDValue &Out) {
578  assert(Parent->getOpcode() == ISD::OR && "unexpected parent");
579  Out = N;
580  return CurDAG->haveNoCommonBitsSet(N, Parent->getOperand(1));
581 }
582 
583 
584 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
585  SDValue &Base,
586  SDValue &OffImm) {
587  // Match simple R + imm12 operands.
588 
589  // Base only.
590  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
591  !CurDAG->isBaseWithConstantOffset(N)) {
592  if (N.getOpcode() == ISD::FrameIndex) {
593  // Match frame index.
594  int FI = cast<FrameIndexSDNode>(N)->getIndex();
595  Base = CurDAG->getTargetFrameIndex(
596  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
597  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
598  return true;
599  }
600 
601  if (N.getOpcode() == ARMISD::Wrapper &&
605  Base = N.getOperand(0);
606  } else
607  Base = N;
608  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
609  return true;
610  }
611 
612  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
613  int RHSC = (int)RHS->getSExtValue();
614  if (N.getOpcode() == ISD::SUB)
615  RHSC = -RHSC;
616 
617  if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
618  Base = N.getOperand(0);
619  if (Base.getOpcode() == ISD::FrameIndex) {
620  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
621  Base = CurDAG->getTargetFrameIndex(
622  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
623  }
624  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
625  return true;
626  }
627  }
628 
629  // Base only.
630  Base = N;
631  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
632  return true;
633 }
634 
635 
636 
637 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
638  SDValue &Opc) {
639  if (N.getOpcode() == ISD::MUL &&
640  ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
641  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
642  // X * [3,5,9] -> X + X * [2,4,8] etc.
643  int RHSC = (int)RHS->getZExtValue();
644  if (RHSC & 1) {
645  RHSC = RHSC & ~1;
646  ARM_AM::AddrOpc AddSub = ARM_AM::add;
647  if (RHSC < 0) {
648  AddSub = ARM_AM::sub;
649  RHSC = - RHSC;
650  }
651  if (isPowerOf2_32(RHSC)) {
652  unsigned ShAmt = Log2_32(RHSC);
653  Base = Offset = N.getOperand(0);
654  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
655  ARM_AM::lsl),
656  SDLoc(N), MVT::i32);
657  return true;
658  }
659  }
660  }
661  }
662 
663  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
664  // ISD::OR that is equivalent to an ISD::ADD.
665  !CurDAG->isBaseWithConstantOffset(N))
666  return false;
667 
668  // Leave simple R +/- imm12 operands for LDRi12
669  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
670  int RHSC;
671  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
672  -0x1000+1, 0x1000, RHSC)) // 12 bits.
673  return false;
674  }
675 
676  // Otherwise this is R +/- [possibly shifted] R.
678  ARM_AM::ShiftOpc ShOpcVal =
680  unsigned ShAmt = 0;
681 
682  Base = N.getOperand(0);
683  Offset = N.getOperand(1);
684 
685  if (ShOpcVal != ARM_AM::no_shift) {
686  // Check to see if the RHS of the shift is a constant, if not, we can't fold
687  // it.
688  if (ConstantSDNode *Sh =
689  dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
690  ShAmt = Sh->getZExtValue();
691  if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
692  Offset = N.getOperand(1).getOperand(0);
693  else {
694  ShAmt = 0;
695  ShOpcVal = ARM_AM::no_shift;
696  }
697  } else {
698  ShOpcVal = ARM_AM::no_shift;
699  }
700  }
701 
702  // Try matching (R shl C) + (R).
703  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
704  !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
705  N.getOperand(0).hasOneUse())) {
707  if (ShOpcVal != ARM_AM::no_shift) {
708  // Check to see if the RHS of the shift is a constant, if not, we can't
709  // fold it.
710  if (ConstantSDNode *Sh =
711  dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
712  ShAmt = Sh->getZExtValue();
713  if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
714  Offset = N.getOperand(0).getOperand(0);
715  Base = N.getOperand(1);
716  } else {
717  ShAmt = 0;
718  ShOpcVal = ARM_AM::no_shift;
719  }
720  } else {
721  ShOpcVal = ARM_AM::no_shift;
722  }
723  }
724  }
725 
726  // If Offset is a multiply-by-constant and it's profitable to extract a shift
727  // and use it in a shifted operand do so.
728  if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
729  unsigned PowerOfTwo = 0;
730  SDValue NewMulConst;
731  if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
732  HandleSDNode Handle(Offset);
733  replaceDAGValue(Offset.getOperand(1), NewMulConst);
734  Offset = Handle.getValue();
735  ShAmt = PowerOfTwo;
736  ShOpcVal = ARM_AM::lsl;
737  }
738  }
739 
740  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
741  SDLoc(N), MVT::i32);
742  return true;
743 }
744 
745 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
746  SDValue &Offset, SDValue &Opc) {
747  unsigned Opcode = Op->getOpcode();
748  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
749  ? cast<LoadSDNode>(Op)->getAddressingMode()
750  : cast<StoreSDNode>(Op)->getAddressingMode();
751  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
753  int Val;
754  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
755  return false;
756 
757  Offset = N;
759  unsigned ShAmt = 0;
760  if (ShOpcVal != ARM_AM::no_shift) {
761  // Check to see if the RHS of the shift is a constant, if not, we can't fold
762  // it.
763  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
764  ShAmt = Sh->getZExtValue();
765  if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
766  Offset = N.getOperand(0);
767  else {
768  ShAmt = 0;
769  ShOpcVal = ARM_AM::no_shift;
770  }
771  } else {
772  ShOpcVal = ARM_AM::no_shift;
773  }
774  }
775 
776  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
777  SDLoc(N), MVT::i32);
778  return true;
779 }
780 
781 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
782  SDValue &Offset, SDValue &Opc) {
783  unsigned Opcode = Op->getOpcode();
784  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
785  ? cast<LoadSDNode>(Op)->getAddressingMode()
786  : cast<StoreSDNode>(Op)->getAddressingMode();
787  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
789  int Val;
790  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
791  if (AddSub == ARM_AM::sub) Val *= -1;
792  Offset = CurDAG->getRegister(0, MVT::i32);
793  Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
794  return true;
795  }
796 
797  return false;
798 }
799 
800 
801 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
802  SDValue &Offset, SDValue &Opc) {
803  unsigned Opcode = Op->getOpcode();
804  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
805  ? cast<LoadSDNode>(Op)->getAddressingMode()
806  : cast<StoreSDNode>(Op)->getAddressingMode();
807  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
809  int Val;
810  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
811  Offset = CurDAG->getRegister(0, MVT::i32);
812  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
814  SDLoc(Op), MVT::i32);
815  return true;
816  }
817 
818  return false;
819 }
820 
821 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
822  Base = N;
823  return true;
824 }
825 
826 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
827  SDValue &Base, SDValue &Offset,
828  SDValue &Opc) {
829  if (N.getOpcode() == ISD::SUB) {
830  // X - C is canonicalize to X + -C, no need to handle it here.
831  Base = N.getOperand(0);
832  Offset = N.getOperand(1);
833  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
834  MVT::i32);
835  return true;
836  }
837 
838  if (!CurDAG->isBaseWithConstantOffset(N)) {
839  Base = N;
840  if (N.getOpcode() == ISD::FrameIndex) {
841  int FI = cast<FrameIndexSDNode>(N)->getIndex();
842  Base = CurDAG->getTargetFrameIndex(
843  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
844  }
845  Offset = CurDAG->getRegister(0, MVT::i32);
846  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
847  MVT::i32);
848  return true;
849  }
850 
851  // If the RHS is +/- imm8, fold into addr mode.
852  int RHSC;
853  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
854  -256 + 1, 256, RHSC)) { // 8 bits.
855  Base = N.getOperand(0);
856  if (Base.getOpcode() == ISD::FrameIndex) {
857  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
858  Base = CurDAG->getTargetFrameIndex(
859  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
860  }
861  Offset = CurDAG->getRegister(0, MVT::i32);
862 
863  ARM_AM::AddrOpc AddSub = ARM_AM::add;
864  if (RHSC < 0) {
865  AddSub = ARM_AM::sub;
866  RHSC = -RHSC;
867  }
868  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
869  MVT::i32);
870  return true;
871  }
872 
873  Base = N.getOperand(0);
874  Offset = N.getOperand(1);
875  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
876  MVT::i32);
877  return true;
878 }
879 
880 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
881  SDValue &Offset, SDValue &Opc) {
882  unsigned Opcode = Op->getOpcode();
883  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
884  ? cast<LoadSDNode>(Op)->getAddressingMode()
885  : cast<StoreSDNode>(Op)->getAddressingMode();
886  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
888  int Val;
889  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
890  Offset = CurDAG->getRegister(0, MVT::i32);
891  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
892  MVT::i32);
893  return true;
894  }
895 
896  Offset = N;
897  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
898  MVT::i32);
899  return true;
900 }
901 
902 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
903  int Lwb, int Upb, bool FP16) {
904  if (!CurDAG->isBaseWithConstantOffset(N)) {
905  Base = N;
906  if (N.getOpcode() == ISD::FrameIndex) {
907  int FI = cast<FrameIndexSDNode>(N)->getIndex();
908  Base = CurDAG->getTargetFrameIndex(
909  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
910  } else if (N.getOpcode() == ARMISD::Wrapper &&
914  Base = N.getOperand(0);
915  }
916  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
917  SDLoc(N), MVT::i32);
918  return true;
919  }
920 
921  // If the RHS is +/- imm8, fold into addr mode.
922  int RHSC;
923  const int Scale = FP16 ? 2 : 4;
924 
925  if (isScaledConstantInRange(N.getOperand(1), Scale, Lwb, Upb, RHSC)) {
926  Base = N.getOperand(0);
927  if (Base.getOpcode() == ISD::FrameIndex) {
928  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
929  Base = CurDAG->getTargetFrameIndex(
930  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
931  }
932 
933  ARM_AM::AddrOpc AddSub = ARM_AM::add;
934  if (RHSC < 0) {
935  AddSub = ARM_AM::sub;
936  RHSC = -RHSC;
937  }
938 
939  if (FP16)
940  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
941  SDLoc(N), MVT::i32);
942  else
943  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
944  SDLoc(N), MVT::i32);
945 
946  return true;
947  }
948 
949  Base = N;
950 
951  if (FP16)
952  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
953  SDLoc(N), MVT::i32);
954  else
955  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
956  SDLoc(N), MVT::i32);
957 
958  return true;
959 }
960 
961 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
962  SDValue &Base, SDValue &Offset) {
963  int Lwb = -256 + 1;
964  int Upb = 256;
965  return IsAddressingMode5(N, Base, Offset, Lwb, Upb, /*FP16=*/ false);
966 }
967 
968 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
969  SDValue &Base, SDValue &Offset) {
970  int Lwb = -512 + 1;
971  int Upb = 512;
972  return IsAddressingMode5(N, Base, Offset, Lwb, Upb, /*FP16=*/ true);
973 }
974 
975 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
976  SDValue &Align) {
977  Addr = N;
978 
979  unsigned Alignment = 0;
980 
981  MemSDNode *MemN = cast<MemSDNode>(Parent);
982 
983  if (isa<LSBaseSDNode>(MemN) ||
984  ((MemN->getOpcode() == ARMISD::VST1_UPD ||
985  MemN->getOpcode() == ARMISD::VLD1_UPD) &&
986  MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
987  // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
988  // The maximum alignment is equal to the memory size being referenced.
989  unsigned MMOAlign = MemN->getAlignment();
990  unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
991  if (MMOAlign >= MemSize && MemSize > 1)
992  Alignment = MemSize;
993  } else {
994  // All other uses of addrmode6 are for intrinsics. For now just record
995  // the raw alignment value; it will be refined later based on the legal
996  // alignment operands for the intrinsic.
997  Alignment = MemN->getAlignment();
998  }
999 
1000  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
1001  return true;
1002 }
1003 
1004 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
1005  SDValue &Offset) {
1006  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
1008  if (AM != ISD::POST_INC)
1009  return false;
1010  Offset = N;
1011  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1012  if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1013  Offset = CurDAG->getRegister(0, MVT::i32);
1014  }
1015  return true;
1016 }
1017 
1018 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1019  SDValue &Offset, SDValue &Label) {
1020  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1021  Offset = N.getOperand(0);
1022  SDValue N1 = N.getOperand(1);
1023  Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1024  SDLoc(N), MVT::i32);
1025  return true;
1026  }
1027 
1028  return false;
1029 }
1030 
1031 
1032 //===----------------------------------------------------------------------===//
1033 // Thumb Addressing Modes
1034 //===----------------------------------------------------------------------===//
1035 
1036 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
1037  SDValue &Base, SDValue &Offset){
1038  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1040  if (!NC || !NC->isNullValue())
1041  return false;
1042 
1043  Base = Offset = N;
1044  return true;
1045  }
1046 
1047  Base = N.getOperand(0);
1048  Offset = N.getOperand(1);
1049  return true;
1050 }
1051 
1052 bool
1053 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1054  SDValue &Base, SDValue &OffImm) {
1055  if (!CurDAG->isBaseWithConstantOffset(N)) {
1056  if (N.getOpcode() == ISD::ADD) {
1057  return false; // We want to select register offset instead
1058  } else if (N.getOpcode() == ARMISD::Wrapper &&
1063  Base = N.getOperand(0);
1064  } else {
1065  Base = N;
1066  }
1067 
1068  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1069  return true;
1070  }
1071 
1072  // If the RHS is + imm5 * scale, fold into addr mode.
1073  int RHSC;
1074  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1075  Base = N.getOperand(0);
1076  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1077  return true;
1078  }
1079 
1080  // Offset is too large, so use register offset instead.
1081  return false;
1082 }
1083 
1084 bool
1085 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1086  SDValue &OffImm) {
1087  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1088 }
1089 
1090 bool
1091 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1092  SDValue &OffImm) {
1093  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1094 }
1095 
1096 bool
1097 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1098  SDValue &OffImm) {
1099  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1100 }
1101 
1102 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1103  SDValue &Base, SDValue &OffImm) {
1104  if (N.getOpcode() == ISD::FrameIndex) {
1105  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1106  // Only multiples of 4 are allowed for the offset, so the frame object
1107  // alignment must be at least 4.
1108  MachineFrameInfo &MFI = MF->getFrameInfo();
1109  if (MFI.getObjectAlignment(FI) < 4)
1110  MFI.setObjectAlignment(FI, 4);
1111  Base = CurDAG->getTargetFrameIndex(
1112  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1113  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1114  return true;
1115  }
1116 
1117  if (!CurDAG->isBaseWithConstantOffset(N))
1118  return false;
1119 
1121  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1122  (LHSR && LHSR->getReg() == ARM::SP)) {
1123  // If the RHS is + imm8 * scale, fold into addr mode.
1124  int RHSC;
1125  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1126  Base = N.getOperand(0);
1127  if (Base.getOpcode() == ISD::FrameIndex) {
1128  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1129  // For LHS+RHS to result in an offset that's a multiple of 4 the object
1130  // indexed by the LHS must be 4-byte aligned.
1131  MachineFrameInfo &MFI = MF->getFrameInfo();
1132  if (MFI.getObjectAlignment(FI) < 4)
1133  MFI.setObjectAlignment(FI, 4);
1134  Base = CurDAG->getTargetFrameIndex(
1135  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1136  }
1137  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1138  return true;
1139  }
1140  }
1141 
1142  return false;
1143 }
1144 
1145 
1146 //===----------------------------------------------------------------------===//
1147 // Thumb 2 Addressing Modes
1148 //===----------------------------------------------------------------------===//
1149 
1150 
1151 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1152  SDValue &Base, SDValue &OffImm) {
1153  // Match simple R + imm12 operands.
1154 
1155  // Base only.
1156  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1157  !CurDAG->isBaseWithConstantOffset(N)) {
1158  if (N.getOpcode() == ISD::FrameIndex) {
1159  // Match frame index.
1160  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1161  Base = CurDAG->getTargetFrameIndex(
1162  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1163  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1164  return true;
1165  }
1166 
1167  if (N.getOpcode() == ARMISD::Wrapper &&
1171  Base = N.getOperand(0);
1172  if (Base.getOpcode() == ISD::TargetConstantPool)
1173  return false; // We want to select t2LDRpci instead.
1174  } else
1175  Base = N;
1176  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1177  return true;
1178  }
1179 
1180  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1181  if (SelectT2AddrModeImm8(N, Base, OffImm))
1182  // Let t2LDRi8 handle (R - imm8).
1183  return false;
1184 
1185  int RHSC = (int)RHS->getZExtValue();
1186  if (N.getOpcode() == ISD::SUB)
1187  RHSC = -RHSC;
1188 
1189  if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1190  Base = N.getOperand(0);
1191  if (Base.getOpcode() == ISD::FrameIndex) {
1192  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1193  Base = CurDAG->getTargetFrameIndex(
1194  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1195  }
1196  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1197  return true;
1198  }
1199  }
1200 
1201  // Base only.
1202  Base = N;
1203  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1204  return true;
1205 }
1206 
1207 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1208  SDValue &Base, SDValue &OffImm) {
1209  // Match simple R - imm8 operands.
1210  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1211  !CurDAG->isBaseWithConstantOffset(N))
1212  return false;
1213 
1214  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1215  int RHSC = (int)RHS->getSExtValue();
1216  if (N.getOpcode() == ISD::SUB)
1217  RHSC = -RHSC;
1218 
1219  if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1220  Base = N.getOperand(0);
1221  if (Base.getOpcode() == ISD::FrameIndex) {
1222  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1223  Base = CurDAG->getTargetFrameIndex(
1224  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1225  }
1226  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1227  return true;
1228  }
1229  }
1230 
1231  return false;
1232 }
1233 
1234 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1235  SDValue &OffImm){
1236  unsigned Opcode = Op->getOpcode();
1237  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1238  ? cast<LoadSDNode>(Op)->getAddressingMode()
1239  : cast<StoreSDNode>(Op)->getAddressingMode();
1240  int RHSC;
1241  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1242  OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1243  ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
1244  : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1245  return true;
1246  }
1247 
1248  return false;
1249 }
1250 
1251 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1252  SDValue &Base,
1253  SDValue &OffReg, SDValue &ShImm) {
1254  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1255  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1256  return false;
1257 
1258  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1259  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1260  int RHSC = (int)RHS->getZExtValue();
1261  if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1262  return false;
1263  else if (RHSC < 0 && RHSC >= -255) // 8 bits
1264  return false;
1265  }
1266 
1267  // Look for (R + R) or (R + (R << [1,2,3])).
1268  unsigned ShAmt = 0;
1269  Base = N.getOperand(0);
1270  OffReg = N.getOperand(1);
1271 
1272  // Swap if it is ((R << c) + R).
1274  if (ShOpcVal != ARM_AM::lsl) {
1275  ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1276  if (ShOpcVal == ARM_AM::lsl)
1277  std::swap(Base, OffReg);
1278  }
1279 
1280  if (ShOpcVal == ARM_AM::lsl) {
1281  // Check to see if the RHS of the shift is a constant, if not, we can't fold
1282  // it.
1283  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1284  ShAmt = Sh->getZExtValue();
1285  if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1286  OffReg = OffReg.getOperand(0);
1287  else {
1288  ShAmt = 0;
1289  }
1290  }
1291  }
1292 
1293  // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1294  // and use it in a shifted operand do so.
1295  if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
1296  unsigned PowerOfTwo = 0;
1297  SDValue NewMulConst;
1298  if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1299  HandleSDNode Handle(OffReg);
1300  replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1301  OffReg = Handle.getValue();
1302  ShAmt = PowerOfTwo;
1303  }
1304  }
1305 
1306  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1307 
1308  return true;
1309 }
1310 
1311 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1312  SDValue &OffImm) {
1313  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1314  // instructions.
1315  Base = N;
1316  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1317 
1318  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1319  return true;
1320 
1322  if (!RHS)
1323  return true;
1324 
1325  uint32_t RHSC = (int)RHS->getZExtValue();
1326  if (RHSC > 1020 || RHSC % 4 != 0)
1327  return true;
1328 
1329  Base = N.getOperand(0);
1330  if (Base.getOpcode() == ISD::FrameIndex) {
1331  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1332  Base = CurDAG->getTargetFrameIndex(
1333  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1334  }
1335 
1336  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1337  return true;
1338 }
1339 
1340 //===--------------------------------------------------------------------===//
1341 
1342 /// getAL - Returns a ARMCC::AL immediate node.
1343 static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1344  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1345 }
1346 
1347 void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1348  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
1349  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp});
1350 }
1351 
1352 bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1353  LoadSDNode *LD = cast<LoadSDNode>(N);
1355  if (AM == ISD::UNINDEXED)
1356  return false;
1357 
1358  EVT LoadedVT = LD->getMemoryVT();
1359  SDValue Offset, AMOpc;
1360  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1361  unsigned Opcode = 0;
1362  bool Match = false;
1363  if (LoadedVT == MVT::i32 && isPre &&
1364  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1365  Opcode = ARM::LDR_PRE_IMM;
1366  Match = true;
1367  } else if (LoadedVT == MVT::i32 && !isPre &&
1368  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1369  Opcode = ARM::LDR_POST_IMM;
1370  Match = true;
1371  } else if (LoadedVT == MVT::i32 &&
1372  SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1373  Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1374  Match = true;
1375 
1376  } else if (LoadedVT == MVT::i16 &&
1377  SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1378  Match = true;
1379  Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1380  ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1381  : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1382  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1383  if (LD->getExtensionType() == ISD::SEXTLOAD) {
1384  if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1385  Match = true;
1386  Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1387  }
1388  } else {
1389  if (isPre &&
1390  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1391  Match = true;
1392  Opcode = ARM::LDRB_PRE_IMM;
1393  } else if (!isPre &&
1394  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1395  Match = true;
1396  Opcode = ARM::LDRB_POST_IMM;
1397  } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1398  Match = true;
1399  Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1400  }
1401  }
1402  }
1403 
1404  if (Match) {
1405  if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1406  SDValue Chain = LD->getChain();
1407  SDValue Base = LD->getBasePtr();
1408  SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1409  CurDAG->getRegister(0, MVT::i32), Chain };
1410  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1411  MVT::Other, Ops);
1412  transferMemOperands(N, New);
1413  ReplaceNode(N, New);
1414  return true;
1415  } else {
1416  SDValue Chain = LD->getChain();
1417  SDValue Base = LD->getBasePtr();
1418  SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1419  CurDAG->getRegister(0, MVT::i32), Chain };
1420  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1421  MVT::Other, Ops);
1422  transferMemOperands(N, New);
1423  ReplaceNode(N, New);
1424  return true;
1425  }
1426  }
1427 
1428  return false;
1429 }
1430 
1431 bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1432  LoadSDNode *LD = cast<LoadSDNode>(N);
1433  EVT LoadedVT = LD->getMemoryVT();
1435  if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
1436  LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
1437  return false;
1438 
1439  auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1440  if (!COffs || COffs->getZExtValue() != 4)
1441  return false;
1442 
1443  // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1444  // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1445  // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1446  // ISel.
1447  SDValue Chain = LD->getChain();
1448  SDValue Base = LD->getBasePtr();
1449  SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1450  CurDAG->getRegister(0, MVT::i32), Chain };
1451  SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1452  MVT::i32, MVT::Other, Ops);
1453  transferMemOperands(N, New);
1454  ReplaceNode(N, New);
1455  return true;
1456 }
1457 
1458 bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1459  LoadSDNode *LD = cast<LoadSDNode>(N);
1461  if (AM == ISD::UNINDEXED)
1462  return false;
1463 
1464  EVT LoadedVT = LD->getMemoryVT();
1465  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1466  SDValue Offset;
1467  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1468  unsigned Opcode = 0;
1469  bool Match = false;
1470  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1471  switch (LoadedVT.getSimpleVT().SimpleTy) {
1472  case MVT::i32:
1473  Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1474  break;
1475  case MVT::i16:
1476  if (isSExtLd)
1477  Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1478  else
1479  Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1480  break;
1481  case MVT::i8:
1482  case MVT::i1:
1483  if (isSExtLd)
1484  Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1485  else
1486  Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1487  break;
1488  default:
1489  return false;
1490  }
1491  Match = true;
1492  }
1493 
1494  if (Match) {
1495  SDValue Chain = LD->getChain();
1496  SDValue Base = LD->getBasePtr();
1497  SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1498  CurDAG->getRegister(0, MVT::i32), Chain };
1499  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1500  MVT::Other, Ops);
1501  transferMemOperands(N, New);
1502  ReplaceNode(N, New);
1503  return true;
1504  }
1505 
1506  return false;
1507 }
1508 
1509 /// Form a GPRPair pseudo register from a pair of GPR regs.
1511  SDLoc dl(V0.getNode());
1512  SDValue RegClass =
1513  CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1514  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1515  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1516  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1517  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1518 }
1519 
1520 /// Form a D register from a pair of S registers.
1521 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1522  SDLoc dl(V0.getNode());
1523  SDValue RegClass =
1524  CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1525  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1526  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1527  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1528  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1529 }
1530 
1531 /// Form a quad register from a pair of D registers.
1532 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1533  SDLoc dl(V0.getNode());
1534  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1535  MVT::i32);
1536  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1537  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1538  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1539  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1540 }
1541 
1542 /// Form 4 consecutive D registers from a pair of Q registers.
1543 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1544  SDLoc dl(V0.getNode());
1545  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1546  MVT::i32);
1547  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1548  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1549  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1550  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1551 }
1552 
1553 /// Form 4 consecutive S registers.
1554 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1555  SDValue V2, SDValue V3) {
1556  SDLoc dl(V0.getNode());
1557  SDValue RegClass =
1558  CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1559  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1560  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1561  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1562  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1563  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1564  V2, SubReg2, V3, SubReg3 };
1565  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1566 }
1567 
1568 /// Form 4 consecutive D registers.
1569 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1570  SDValue V2, SDValue V3) {
1571  SDLoc dl(V0.getNode());
1572  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1573  MVT::i32);
1574  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1575  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1576  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1577  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1578  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1579  V2, SubReg2, V3, SubReg3 };
1580  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1581 }
1582 
1583 /// Form 4 consecutive Q registers.
1584 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1585  SDValue V2, SDValue V3) {
1586  SDLoc dl(V0.getNode());
1587  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1588  MVT::i32);
1589  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1590  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1591  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1592  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1593  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1594  V2, SubReg2, V3, SubReg3 };
1595  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1596 }
1597 
1598 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1599 /// of a NEON VLD or VST instruction. The supported values depend on the
1600 /// number of registers being loaded.
1601 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1602  unsigned NumVecs, bool is64BitVector) {
1603  unsigned NumRegs = NumVecs;
1604  if (!is64BitVector && NumVecs < 3)
1605  NumRegs *= 2;
1606 
1607  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1608  if (Alignment >= 32 && NumRegs == 4)
1609  Alignment = 32;
1610  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1611  Alignment = 16;
1612  else if (Alignment >= 8)
1613  Alignment = 8;
1614  else
1615  Alignment = 0;
1616 
1617  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1618 }
1619 
1620 static bool isVLDfixed(unsigned Opc)
1621 {
1622  switch (Opc) {
1623  default: return false;
1624  case ARM::VLD1d8wb_fixed : return true;
1625  case ARM::VLD1d16wb_fixed : return true;
1626  case ARM::VLD1d64Qwb_fixed : return true;
1627  case ARM::VLD1d32wb_fixed : return true;
1628  case ARM::VLD1d64wb_fixed : return true;
1629  case ARM::VLD1d64TPseudoWB_fixed : return true;
1630  case ARM::VLD1d64QPseudoWB_fixed : return true;
1631  case ARM::VLD1q8wb_fixed : return true;
1632  case ARM::VLD1q16wb_fixed : return true;
1633  case ARM::VLD1q32wb_fixed : return true;
1634  case ARM::VLD1q64wb_fixed : return true;
1635  case ARM::VLD1DUPd8wb_fixed : return true;
1636  case ARM::VLD1DUPd16wb_fixed : return true;
1637  case ARM::VLD1DUPd32wb_fixed : return true;
1638  case ARM::VLD1DUPq8wb_fixed : return true;
1639  case ARM::VLD1DUPq16wb_fixed : return true;
1640  case ARM::VLD1DUPq32wb_fixed : return true;
1641  case ARM::VLD2d8wb_fixed : return true;
1642  case ARM::VLD2d16wb_fixed : return true;
1643  case ARM::VLD2d32wb_fixed : return true;
1644  case ARM::VLD2q8PseudoWB_fixed : return true;
1645  case ARM::VLD2q16PseudoWB_fixed : return true;
1646  case ARM::VLD2q32PseudoWB_fixed : return true;
1647  case ARM::VLD2DUPd8wb_fixed : return true;
1648  case ARM::VLD2DUPd16wb_fixed : return true;
1649  case ARM::VLD2DUPd32wb_fixed : return true;
1650  }
1651 }
1652 
1653 static bool isVSTfixed(unsigned Opc)
1654 {
1655  switch (Opc) {
1656  default: return false;
1657  case ARM::VST1d8wb_fixed : return true;
1658  case ARM::VST1d16wb_fixed : return true;
1659  case ARM::VST1d32wb_fixed : return true;
1660  case ARM::VST1d64wb_fixed : return true;
1661  case ARM::VST1q8wb_fixed : return true;
1662  case ARM::VST1q16wb_fixed : return true;
1663  case ARM::VST1q32wb_fixed : return true;
1664  case ARM::VST1q64wb_fixed : return true;
1665  case ARM::VST1d64TPseudoWB_fixed : return true;
1666  case ARM::VST1d64QPseudoWB_fixed : return true;
1667  case ARM::VST2d8wb_fixed : return true;
1668  case ARM::VST2d16wb_fixed : return true;
1669  case ARM::VST2d32wb_fixed : return true;
1670  case ARM::VST2q8PseudoWB_fixed : return true;
1671  case ARM::VST2q16PseudoWB_fixed : return true;
1672  case ARM::VST2q32PseudoWB_fixed : return true;
1673  }
1674 }
1675 
1676 // Get the register stride update opcode of a VLD/VST instruction that
1677 // is otherwise equivalent to the given fixed stride updating instruction.
1678 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1679  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1680  && "Incorrect fixed stride updating instruction.");
1681  switch (Opc) {
1682  default: break;
1683  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1684  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1685  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1686  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1687  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1688  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1689  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1690  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1691  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1692  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1693  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1694  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1695  case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
1696  case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
1697  case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
1698  case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
1699  case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
1700  case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
1701 
1702  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1703  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1704  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1705  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1706  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1707  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1708  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1709  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1710  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1711  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1712 
1713  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1714  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1715  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1716  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1717  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1718  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1719 
1720  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1721  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1722  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1723  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1724  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1725  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1726 
1727  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1728  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1729  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1730  }
1731  return Opc; // If not one we handle, return it unchanged.
1732 }
1733 
1734 /// Returns true if the given increment is a Constant known to be equal to the
1735 /// access size performed by a NEON load/store. This means the "[rN]!" form can
1736 /// be used.
1737 static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1738  auto C = dyn_cast<ConstantSDNode>(Inc);
1739  return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
1740 }
1741 
1742 void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1743  const uint16_t *DOpcodes,
1744  const uint16_t *QOpcodes0,
1745  const uint16_t *QOpcodes1) {
1746  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1747  SDLoc dl(N);
1748 
1749  SDValue MemAddr, Align;
1750  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1751  // nodes are not intrinsics.
1752  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1753  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1754  return;
1755 
1756  SDValue Chain = N->getOperand(0);
1757  EVT VT = N->getValueType(0);
1758  bool is64BitVector = VT.is64BitVector();
1759  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1760 
1761  unsigned OpcodeIndex;
1762  switch (VT.getSimpleVT().SimpleTy) {
1763  default: llvm_unreachable("unhandled vld type");
1764  // Double-register operations:
1765  case MVT::v8i8: OpcodeIndex = 0; break;
1766  case MVT::v4i16: OpcodeIndex = 1; break;
1767  case MVT::v2f32:
1768  case MVT::v2i32: OpcodeIndex = 2; break;
1769  case MVT::v1i64: OpcodeIndex = 3; break;
1770  // Quad-register operations:
1771  case MVT::v16i8: OpcodeIndex = 0; break;
1772  case MVT::v8i16: OpcodeIndex = 1; break;
1773  case MVT::v4f32:
1774  case MVT::v4i32: OpcodeIndex = 2; break;
1775  case MVT::v2f64:
1776  case MVT::v2i64: OpcodeIndex = 3; break;
1777  }
1778 
1779  EVT ResTy;
1780  if (NumVecs == 1)
1781  ResTy = VT;
1782  else {
1783  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1784  if (!is64BitVector)
1785  ResTyElts *= 2;
1786  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1787  }
1788  std::vector<EVT> ResTys;
1789  ResTys.push_back(ResTy);
1790  if (isUpdating)
1791  ResTys.push_back(MVT::i32);
1792  ResTys.push_back(MVT::Other);
1793 
1794  SDValue Pred = getAL(CurDAG, dl);
1795  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1796  SDNode *VLd;
1798 
1799  // Double registers and VLD1/VLD2 quad registers are directly supported.
1800  if (is64BitVector || NumVecs <= 2) {
1801  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1802  QOpcodes0[OpcodeIndex]);
1803  Ops.push_back(MemAddr);
1804  Ops.push_back(Align);
1805  if (isUpdating) {
1806  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1807  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1808  if (!IsImmUpdate) {
1809  // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1810  // check for the opcode rather than the number of vector elements.
1811  if (isVLDfixed(Opc))
1812  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1813  Ops.push_back(Inc);
1814  // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
1815  // the operands if not such an opcode.
1816  } else if (!isVLDfixed(Opc))
1817  Ops.push_back(Reg0);
1818  }
1819  Ops.push_back(Pred);
1820  Ops.push_back(Reg0);
1821  Ops.push_back(Chain);
1822  VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1823 
1824  } else {
1825  // Otherwise, quad registers are loaded with two separate instructions,
1826  // where one loads the even registers and the other loads the odd registers.
1827  EVT AddrTy = MemAddr.getValueType();
1828 
1829  // Load the even subregs. This is always an updating load, so that it
1830  // provides the address to the second load for the odd subregs.
1831  SDValue ImplDef =
1832  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1833  const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1834  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1835  ResTy, AddrTy, MVT::Other, OpsA);
1836  Chain = SDValue(VLdA, 2);
1837 
1838  // Load the odd subregs.
1839  Ops.push_back(SDValue(VLdA, 1));
1840  Ops.push_back(Align);
1841  if (isUpdating) {
1842  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1843  assert(isa<ConstantSDNode>(Inc.getNode()) &&
1844  "only constant post-increment update allowed for VLD3/4");
1845  (void)Inc;
1846  Ops.push_back(Reg0);
1847  }
1848  Ops.push_back(SDValue(VLdA, 0));
1849  Ops.push_back(Pred);
1850  Ops.push_back(Reg0);
1851  Ops.push_back(Chain);
1852  VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1853  }
1854 
1855  // Transfer memoperands.
1856  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1857  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp});
1858 
1859  if (NumVecs == 1) {
1860  ReplaceNode(N, VLd);
1861  return;
1862  }
1863 
1864  // Extract out the subregisters.
1865  SDValue SuperReg = SDValue(VLd, 0);
1866  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
1867  ARM::qsub_3 == ARM::qsub_0 + 3,
1868  "Unexpected subreg numbering");
1869  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1870  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1871  ReplaceUses(SDValue(N, Vec),
1872  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1873  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1874  if (isUpdating)
1875  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1876  CurDAG->RemoveDeadNode(N);
1877 }
1878 
1879 void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1880  const uint16_t *DOpcodes,
1881  const uint16_t *QOpcodes0,
1882  const uint16_t *QOpcodes1) {
1883  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1884  SDLoc dl(N);
1885 
1886  SDValue MemAddr, Align;
1887  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
1888  // nodes are not intrinsics.
1889  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
1890  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1891  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1892  return;
1893 
1894  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1895 
1896  SDValue Chain = N->getOperand(0);
1897  EVT VT = N->getOperand(Vec0Idx).getValueType();
1898  bool is64BitVector = VT.is64BitVector();
1899  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1900 
1901  unsigned OpcodeIndex;
1902  switch (VT.getSimpleVT().SimpleTy) {
1903  default: llvm_unreachable("unhandled vst type");
1904  // Double-register operations:
1905  case MVT::v8i8: OpcodeIndex = 0; break;
1906  case MVT::v4f16:
1907  case MVT::v4i16: OpcodeIndex = 1; break;
1908  case MVT::v2f32:
1909  case MVT::v2i32: OpcodeIndex = 2; break;
1910  case MVT::v1i64: OpcodeIndex = 3; break;
1911  // Quad-register operations:
1912  case MVT::v16i8: OpcodeIndex = 0; break;
1913  case MVT::v8f16:
1914  case MVT::v8i16: OpcodeIndex = 1; break;
1915  case MVT::v4f32:
1916  case MVT::v4i32: OpcodeIndex = 2; break;
1917  case MVT::v2f64:
1918  case MVT::v2i64: OpcodeIndex = 3; break;
1919  }
1920 
1921  std::vector<EVT> ResTys;
1922  if (isUpdating)
1923  ResTys.push_back(MVT::i32);
1924  ResTys.push_back(MVT::Other);
1925 
1926  SDValue Pred = getAL(CurDAG, dl);
1927  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1929 
1930  // Double registers and VST1/VST2 quad registers are directly supported.
1931  if (is64BitVector || NumVecs <= 2) {
1932  SDValue SrcReg;
1933  if (NumVecs == 1) {
1934  SrcReg = N->getOperand(Vec0Idx);
1935  } else if (is64BitVector) {
1936  // Form a REG_SEQUENCE to force register allocation.
1937  SDValue V0 = N->getOperand(Vec0Idx + 0);
1938  SDValue V1 = N->getOperand(Vec0Idx + 1);
1939  if (NumVecs == 2)
1940  SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1941  else {
1942  SDValue V2 = N->getOperand(Vec0Idx + 2);
1943  // If it's a vst3, form a quad D-register and leave the last part as
1944  // an undef.
1945  SDValue V3 = (NumVecs == 3)
1946  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1947  : N->getOperand(Vec0Idx + 3);
1948  SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1949  }
1950  } else {
1951  // Form a QQ register.
1952  SDValue Q0 = N->getOperand(Vec0Idx);
1953  SDValue Q1 = N->getOperand(Vec0Idx + 1);
1954  SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1955  }
1956 
1957  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1958  QOpcodes0[OpcodeIndex]);
1959  Ops.push_back(MemAddr);
1960  Ops.push_back(Align);
1961  if (isUpdating) {
1962  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1963  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1964  if (!IsImmUpdate) {
1965  // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
1966  // check for the opcode rather than the number of vector elements.
1967  if (isVSTfixed(Opc))
1968  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1969  Ops.push_back(Inc);
1970  }
1971  // VST1/VST2 fixed increment does not need Reg0 so only include it in
1972  // the operands if not such an opcode.
1973  else if (!isVSTfixed(Opc))
1974  Ops.push_back(Reg0);
1975  }
1976  Ops.push_back(SrcReg);
1977  Ops.push_back(Pred);
1978  Ops.push_back(Reg0);
1979  Ops.push_back(Chain);
1980  SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1981 
1982  // Transfer memoperands.
1983  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp});
1984 
1985  ReplaceNode(N, VSt);
1986  return;
1987  }
1988 
1989  // Otherwise, quad registers are stored with two separate instructions,
1990  // where one stores the even registers and the other stores the odd registers.
1991 
1992  // Form the QQQQ REG_SEQUENCE.
1993  SDValue V0 = N->getOperand(Vec0Idx + 0);
1994  SDValue V1 = N->getOperand(Vec0Idx + 1);
1995  SDValue V2 = N->getOperand(Vec0Idx + 2);
1996  SDValue V3 = (NumVecs == 3)
1997  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1998  : N->getOperand(Vec0Idx + 3);
1999  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2000 
2001  // Store the even D registers. This is always an updating store, so that it
2002  // provides the address to the second store for the odd subregs.
2003  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
2004  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
2005  MemAddr.getValueType(),
2006  MVT::Other, OpsA);
2007  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp});
2008  Chain = SDValue(VStA, 1);
2009 
2010  // Store the odd D registers.
2011  Ops.push_back(SDValue(VStA, 0));
2012  Ops.push_back(Align);
2013  if (isUpdating) {
2014  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2015  assert(isa<ConstantSDNode>(Inc.getNode()) &&
2016  "only constant post-increment update allowed for VST3/4");
2017  (void)Inc;
2018  Ops.push_back(Reg0);
2019  }
2020  Ops.push_back(RegSeq);
2021  Ops.push_back(Pred);
2022  Ops.push_back(Reg0);
2023  Ops.push_back(Chain);
2024  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2025  Ops);
2026  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp});
2027  ReplaceNode(N, VStB);
2028 }
2029 
2030 void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2031  unsigned NumVecs,
2032  const uint16_t *DOpcodes,
2033  const uint16_t *QOpcodes) {
2034  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2035  SDLoc dl(N);
2036 
2037  SDValue MemAddr, Align;
2038  bool IsIntrinsic = !isUpdating; // By coincidence, all supported updating
2039  // nodes are not intrinsics.
2040  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2041  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2042  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2043  return;
2044 
2045  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2046 
2047  SDValue Chain = N->getOperand(0);
2048  unsigned Lane =
2049  cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2050  EVT VT = N->getOperand(Vec0Idx).getValueType();
2051  bool is64BitVector = VT.is64BitVector();
2052 
2053  unsigned Alignment = 0;
2054  if (NumVecs != 3) {
2055  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2056  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2057  if (Alignment > NumBytes)
2058  Alignment = NumBytes;
2059  if (Alignment < 8 && Alignment < NumBytes)
2060  Alignment = 0;
2061  // Alignment must be a power of two; make sure of that.
2062  Alignment = (Alignment & -Alignment);
2063  if (Alignment == 1)
2064  Alignment = 0;
2065  }
2066  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2067 
2068  unsigned OpcodeIndex;
2069  switch (VT.getSimpleVT().SimpleTy) {
2070  default: llvm_unreachable("unhandled vld/vst lane type");
2071  // Double-register operations:
2072  case MVT::v8i8: OpcodeIndex = 0; break;
2073  case MVT::v4i16: OpcodeIndex = 1; break;
2074  case MVT::v2f32:
2075  case MVT::v2i32: OpcodeIndex = 2; break;
2076  // Quad-register operations:
2077  case MVT::v8i16: OpcodeIndex = 0; break;
2078  case MVT::v4f32:
2079  case MVT::v4i32: OpcodeIndex = 1; break;
2080  }
2081 
2082  std::vector<EVT> ResTys;
2083  if (IsLoad) {
2084  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2085  if (!is64BitVector)
2086  ResTyElts *= 2;
2087  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2088  MVT::i64, ResTyElts));
2089  }
2090  if (isUpdating)
2091  ResTys.push_back(MVT::i32);
2092  ResTys.push_back(MVT::Other);
2093 
2094  SDValue Pred = getAL(CurDAG, dl);
2095  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2096 
2098  Ops.push_back(MemAddr);
2099  Ops.push_back(Align);
2100  if (isUpdating) {
2101  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2102  bool IsImmUpdate =
2103  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2104  Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2105  }
2106 
2107  SDValue SuperReg;
2108  SDValue V0 = N->getOperand(Vec0Idx + 0);
2109  SDValue V1 = N->getOperand(Vec0Idx + 1);
2110  if (NumVecs == 2) {
2111  if (is64BitVector)
2112  SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2113  else
2114  SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2115  } else {
2116  SDValue V2 = N->getOperand(Vec0Idx + 2);
2117  SDValue V3 = (NumVecs == 3)
2118  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2119  : N->getOperand(Vec0Idx + 3);
2120  if (is64BitVector)
2121  SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2122  else
2123  SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2124  }
2125  Ops.push_back(SuperReg);
2126  Ops.push_back(getI32Imm(Lane, dl));
2127  Ops.push_back(Pred);
2128  Ops.push_back(Reg0);
2129  Ops.push_back(Chain);
2130 
2131  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2132  QOpcodes[OpcodeIndex]);
2133  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2134  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp});
2135  if (!IsLoad) {
2136  ReplaceNode(N, VLdLn);
2137  return;
2138  }
2139 
2140  // Extract the subregisters.
2141  SuperReg = SDValue(VLdLn, 0);
2142  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2143  ARM::qsub_3 == ARM::qsub_0 + 3,
2144  "Unexpected subreg numbering");
2145  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2146  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2147  ReplaceUses(SDValue(N, Vec),
2148  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2149  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2150  if (isUpdating)
2151  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2152  CurDAG->RemoveDeadNode(N);
2153 }
2154 
2155 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic,
2156  bool isUpdating, unsigned NumVecs,
2157  const uint16_t *DOpcodes,
2158  const uint16_t *QOpcodes0,
2159  const uint16_t *QOpcodes1) {
2160  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2161  SDLoc dl(N);
2162 
2163  SDValue MemAddr, Align;
2164  unsigned AddrOpIdx = IsIntrinsic ? 2 : 1;
2165  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2166  return;
2167 
2168  SDValue Chain = N->getOperand(0);
2169  EVT VT = N->getValueType(0);
2170  bool is64BitVector = VT.is64BitVector();
2171 
2172  unsigned Alignment = 0;
2173  if (NumVecs != 3) {
2174  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2175  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2176  if (Alignment > NumBytes)
2177  Alignment = NumBytes;
2178  if (Alignment < 8 && Alignment < NumBytes)
2179  Alignment = 0;
2180  // Alignment must be a power of two; make sure of that.
2181  Alignment = (Alignment & -Alignment);
2182  if (Alignment == 1)
2183  Alignment = 0;
2184  }
2185  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2186 
2187  unsigned OpcodeIndex;
2188  switch (VT.getSimpleVT().SimpleTy) {
2189  default: llvm_unreachable("unhandled vld-dup type");
2190  case MVT::v8i8:
2191  case MVT::v16i8: OpcodeIndex = 0; break;
2192  case MVT::v4i16:
2193  case MVT::v8i16: OpcodeIndex = 1; break;
2194  case MVT::v2f32:
2195  case MVT::v2i32:
2196  case MVT::v4f32:
2197  case MVT::v4i32: OpcodeIndex = 2; break;
2198  case MVT::v1f64:
2199  case MVT::v1i64: OpcodeIndex = 3; break;
2200  }
2201 
2202  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2203  if (!is64BitVector)
2204  ResTyElts *= 2;
2205  EVT ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
2206 
2207  std::vector<EVT> ResTys;
2208  ResTys.push_back(ResTy);
2209  if (isUpdating)
2210  ResTys.push_back(MVT::i32);
2211  ResTys.push_back(MVT::Other);
2212 
2213  SDValue Pred = getAL(CurDAG, dl);
2214  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2215 
2216  SDNode *VLdDup;
2217  if (is64BitVector || NumVecs == 1) {
2219  Ops.push_back(MemAddr);
2220  Ops.push_back(Align);
2221  unsigned Opc = is64BitVector ? DOpcodes[OpcodeIndex] :
2222  QOpcodes0[OpcodeIndex];
2223  if (isUpdating) {
2224  // fixed-stride update instructions don't have an explicit writeback
2225  // operand. It's implicit in the opcode itself.
2226  SDValue Inc = N->getOperand(2);
2227  bool IsImmUpdate =
2228  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2229  if (NumVecs <= 2 && !IsImmUpdate)
2230  Opc = getVLDSTRegisterUpdateOpcode(Opc);
2231  if (!IsImmUpdate)
2232  Ops.push_back(Inc);
2233  // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2234  else if (NumVecs > 2)
2235  Ops.push_back(Reg0);
2236  }
2237  Ops.push_back(Pred);
2238  Ops.push_back(Reg0);
2239  Ops.push_back(Chain);
2240  VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2241  } else if (NumVecs == 2) {
2242  const SDValue OpsA[] = { MemAddr, Align, Pred, Reg0, Chain };
2243  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2244  dl, ResTys, OpsA);
2245 
2246  Chain = SDValue(VLdA, 1);
2247  const SDValue OpsB[] = { MemAddr, Align, Pred, Reg0, Chain };
2248  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2249  } else {
2250  SDValue ImplDef =
2251  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
2252  const SDValue OpsA[] = { MemAddr, Align, ImplDef, Pred, Reg0, Chain };
2253  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex],
2254  dl, ResTys, OpsA);
2255 
2256  SDValue SuperReg = SDValue(VLdA, 0);
2257  Chain = SDValue(VLdA, 1);
2258  const SDValue OpsB[] = { MemAddr, Align, SuperReg, Pred, Reg0, Chain };
2259  VLdDup = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, OpsB);
2260  }
2261 
2262  // Transfer memoperands.
2263  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2264  CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp});
2265 
2266  // Extract the subregisters.
2267  if (NumVecs == 1) {
2268  ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
2269  } else {
2270  SDValue SuperReg = SDValue(VLdDup, 0);
2271  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
2272  unsigned SubIdx = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2273  for (unsigned Vec = 0; Vec != NumVecs; ++Vec) {
2274  ReplaceUses(SDValue(N, Vec),
2275  CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2276  }
2277  }
2278  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2279  if (isUpdating)
2280  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2281  CurDAG->RemoveDeadNode(N);
2282 }
2283 
2284 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
2285  if (!Subtarget->hasV6T2Ops())
2286  return false;
2287 
2288  unsigned Opc = isSigned
2289  ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2290  : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2291  SDLoc dl(N);
2292 
2293  // For unsigned extracts, check for a shift right and mask
2294  unsigned And_imm = 0;
2295  if (N->getOpcode() == ISD::AND) {
2296  if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2297 
2298  // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2299  if (And_imm & (And_imm + 1))
2300  return false;
2301 
2302  unsigned Srl_imm = 0;
2304  Srl_imm)) {
2305  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2306 
2307  // Mask off the unnecessary bits of the AND immediate; normally
2308  // DAGCombine will do this, but that might not happen if
2309  // targetShrinkDemandedConstant chooses a different immediate.
2310  And_imm &= -1U >> Srl_imm;
2311 
2312  // Note: The width operand is encoded as width-1.
2313  unsigned Width = countTrailingOnes(And_imm) - 1;
2314  unsigned LSB = Srl_imm;
2315 
2316  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2317 
2318  if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2319  // It's cheaper to use a right shift to extract the top bits.
2320  if (Subtarget->isThumb()) {
2321  Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2322  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2323  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2324  getAL(CurDAG, dl), Reg0, Reg0 };
2325  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2326  return true;
2327  }
2328 
2329  // ARM models shift instructions as MOVsi with shifter operand.
2331  SDValue ShOpc =
2332  CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
2333  MVT::i32);
2334  SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2335  getAL(CurDAG, dl), Reg0, Reg0 };
2336  CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2337  return true;
2338  }
2339 
2340  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2341  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2342  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2343  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2344  getAL(CurDAG, dl), Reg0 };
2345  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2346  return true;
2347  }
2348  }
2349  return false;
2350  }
2351 
2352  // Otherwise, we're looking for a shift of a shift
2353  unsigned Shl_imm = 0;
2354  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2355  assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2356  unsigned Srl_imm = 0;
2357  if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2358  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2359  // Note: The width operand is encoded as width-1.
2360  unsigned Width = 32 - Srl_imm - 1;
2361  int LSB = Srl_imm - Shl_imm;
2362  if (LSB < 0)
2363  return false;
2364  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2365  assert(LSB + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2366  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2367  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2368  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2369  getAL(CurDAG, dl), Reg0 };
2370  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2371  return true;
2372  }
2373  }
2374 
2375  // Or we are looking for a shift of an and, with a mask operand
2376  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
2377  isShiftedMask_32(And_imm)) {
2378  unsigned Srl_imm = 0;
2379  unsigned LSB = countTrailingZeros(And_imm);
2380  // Shift must be the same as the ands lsb
2381  if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
2382  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2383  unsigned MSB = 31 - countLeadingZeros(And_imm);
2384  // Note: The width operand is encoded as width-1.
2385  unsigned Width = MSB - LSB;
2386  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2387  assert(Srl_imm + Width + 1 <= 32 && "Shouldn't create an invalid ubfx");
2388  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2389  CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
2390  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2391  getAL(CurDAG, dl), Reg0 };
2392  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2393  return true;
2394  }
2395  }
2396 
2397  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2398  unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2399  unsigned LSB = 0;
2400  if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2402  return false;
2403 
2404  if (LSB + Width > 32)
2405  return false;
2406 
2407  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2408  assert(LSB + Width <= 32 && "Shouldn't create an invalid ubfx");
2409  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2410  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2411  CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
2412  getAL(CurDAG, dl), Reg0 };
2413  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2414  return true;
2415  }
2416 
2417  return false;
2418 }
2419 
2420 /// Target-specific DAG combining for ISD::XOR.
2421 /// Target-independent combining lowers SELECT_CC nodes of the form
2422 /// select_cc setg[ge] X, 0, X, -X
2423 /// select_cc setgt X, -1, X, -X
2424 /// select_cc setl[te] X, 0, -X, X
2425 /// select_cc setlt X, 1, -X, X
2426 /// which represent Integer ABS into:
2427 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2428 /// ARM instruction selection detects the latter and matches it to
2429 /// ARM::ABS or ARM::t2ABS machine node.
2430 bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
2431  SDValue XORSrc0 = N->getOperand(0);
2432  SDValue XORSrc1 = N->getOperand(1);
2433  EVT VT = N->getValueType(0);
2434 
2435  if (Subtarget->isThumb1Only())
2436  return false;
2437 
2438  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2439  return false;
2440 
2441  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2442  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2443  SDValue SRASrc0 = XORSrc1.getOperand(0);
2444  SDValue SRASrc1 = XORSrc1.getOperand(1);
2445  ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
2446  EVT XType = SRASrc0.getValueType();
2447  unsigned Size = XType.getSizeInBits() - 1;
2448 
2449  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2450  XType.isInteger() && SRAConstant != nullptr &&
2451  Size == SRAConstant->getZExtValue()) {
2452  unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2453  CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2454  return true;
2455  }
2456 
2457  return false;
2458 }
2459 
2460 /// We've got special pseudo-instructions for these
2461 void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2462  unsigned Opcode;
2463  EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2464  if (MemTy == MVT::i8)
2465  Opcode = ARM::CMP_SWAP_8;
2466  else if (MemTy == MVT::i16)
2467  Opcode = ARM::CMP_SWAP_16;
2468  else if (MemTy == MVT::i32)
2469  Opcode = ARM::CMP_SWAP_32;
2470  else
2471  llvm_unreachable("Unknown AtomicCmpSwap type");
2472 
2473  SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2474  N->getOperand(0)};
2475  SDNode *CmpSwap = CurDAG->getMachineNode(
2476  Opcode, SDLoc(N),
2477  CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
2478 
2479  MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
2480  CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
2481 
2482  ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2483  ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2484  CurDAG->RemoveDeadNode(N);
2485 }
2486 
2489  unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
2490  unsigned LastOne = A.countTrailingZeros();
2491  if (A.countPopulation() != (FirstOne - LastOne + 1))
2493  return std::make_pair(FirstOne, LastOne);
2494 }
2495 
2496 void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
2497  assert(N->getOpcode() == ARMISD::CMPZ);
2498  SwitchEQNEToPLMI = false;
2499 
2500  if (!Subtarget->isThumb())
2501  // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
2502  // LSR don't exist as standalone instructions - they need the barrel shifter.
2503  return;
2504 
2505  // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
2506  SDValue And = N->getOperand(0);
2507  if (!And->hasOneUse())
2508  return;
2509 
2510  SDValue Zero = N->getOperand(1);
2511  if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isNullValue() ||
2512  And->getOpcode() != ISD::AND)
2513  return;
2514  SDValue X = And.getOperand(0);
2515  auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
2516 
2517  if (!C)
2518  return;
2519  auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
2520  if (!Range)
2521  return;
2522 
2523  // There are several ways to lower this:
2524  SDNode *NewN;
2525  SDLoc dl(N);
2526 
2527  auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
2528  if (Subtarget->isThumb2()) {
2529  Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
2530  SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2531  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2532  CurDAG->getRegister(0, MVT::i32) };
2533  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2534  } else {
2535  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2536  CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2537  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2538  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2539  }
2540  };
2541 
2542  if (Range->second == 0) {
2543  // 1. Mask includes the LSB -> Simply shift the top N bits off
2544  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2545  ReplaceNode(And.getNode(), NewN);
2546  } else if (Range->first == 31) {
2547  // 2. Mask includes the MSB -> Simply shift the bottom N bits off
2548  NewN = EmitShift(ARM::tLSRri, X, Range->second);
2549  ReplaceNode(And.getNode(), NewN);
2550  } else if (Range->first == Range->second) {
2551  // 3. Only one bit is set. We can shift this into the sign bit and use a
2552  // PL/MI comparison.
2553  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2554  ReplaceNode(And.getNode(), NewN);
2555 
2556  SwitchEQNEToPLMI = true;
2557  } else if (!Subtarget->hasV6T2Ops()) {
2558  // 4. Do a double shift to clear bottom and top bits, but only in
2559  // thumb-1 mode as in thumb-2 we can use UBFX.
2560  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2561  NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
2562  Range->second + (31 - Range->first));
2563  ReplaceNode(And.getNode(), NewN);
2564  }
2565 
2566 }
2567 
2569  SDLoc dl(N);
2570 
2571  if (N->isMachineOpcode()) {
2572  N->setNodeId(-1);
2573  return; // Already selected.
2574  }
2575 
2576  switch (N->getOpcode()) {
2577  default: break;
2578  case ISD::WRITE_REGISTER:
2579  if (tryWriteRegister(N))
2580  return;
2581  break;
2582  case ISD::READ_REGISTER:
2583  if (tryReadRegister(N))
2584  return;
2585  break;
2586  case ISD::INLINEASM:
2587  if (tryInlineAsm(N))
2588  return;
2589  break;
2590  case ISD::XOR:
2591  // Select special operations if XOR node forms integer ABS pattern
2592  if (tryABSOp(N))
2593  return;
2594  // Other cases are autogenerated.
2595  break;
2596  case ISD::Constant: {
2597  unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2598  // If we can't materialize the constant we need to use a literal pool
2599  if (ConstantMaterializationCost(Val) > 2) {
2600  SDValue CPIdx = CurDAG->getTargetConstantPool(
2601  ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2602  TLI->getPointerTy(CurDAG->getDataLayout()));
2603 
2604  SDNode *ResNode;
2605  if (Subtarget->isThumb()) {
2606  SDValue Ops[] = {
2607  CPIdx,
2608  getAL(CurDAG, dl),
2609  CurDAG->getRegister(0, MVT::i32),
2610  CurDAG->getEntryNode()
2611  };
2612  ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2613  Ops);
2614  } else {
2615  SDValue Ops[] = {
2616  CPIdx,
2617  CurDAG->getTargetConstant(0, dl, MVT::i32),
2618  getAL(CurDAG, dl),
2619  CurDAG->getRegister(0, MVT::i32),
2620  CurDAG->getEntryNode()
2621  };
2622  ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2623  Ops);
2624  }
2625  // Annotate the Node with memory operand information so that MachineInstr
2626  // queries work properly. This e.g. gives the register allocation the
2627  // required information for rematerialization.
2628  MachineFunction& MF = CurDAG->getMachineFunction();
2629  MachineMemOperand *MemOp =
2632 
2633  CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp});
2634 
2635  ReplaceNode(N, ResNode);
2636  return;
2637  }
2638 
2639  // Other cases are autogenerated.
2640  break;
2641  }
2642  case ISD::FrameIndex: {
2643  // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2644  int FI = cast<FrameIndexSDNode>(N)->getIndex();
2645  SDValue TFI = CurDAG->getTargetFrameIndex(
2646  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2647  if (Subtarget->isThumb1Only()) {
2648  // Set the alignment of the frame object to 4, to avoid having to generate
2649  // more than one ADD
2650  MachineFrameInfo &MFI = MF->getFrameInfo();
2651  if (MFI.getObjectAlignment(FI) < 4)
2652  MFI.setObjectAlignment(FI, 4);
2653  CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2654  CurDAG->getTargetConstant(0, dl, MVT::i32));
2655  return;
2656  } else {
2657  unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2658  ARM::t2ADDri : ARM::ADDri);
2659  SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
2660  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2661  CurDAG->getRegister(0, MVT::i32) };
2662  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2663  return;
2664  }
2665  }
2666  case ISD::SRL:
2667  if (tryV6T2BitfieldExtractOp(N, false))
2668  return;
2669  break;
2671  case ISD::SRA:
2672  if (tryV6T2BitfieldExtractOp(N, true))
2673  return;
2674  break;
2675  case ISD::MUL:
2676  if (Subtarget->isThumb1Only())
2677  break;
2678  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2679  unsigned RHSV = C->getZExtValue();
2680  if (!RHSV) break;
2681  if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
2682  unsigned ShImm = Log2_32(RHSV-1);
2683  if (ShImm >= 32)
2684  break;
2685  SDValue V = N->getOperand(0);
2686  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2687  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2688  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2689  if (Subtarget->isThumb()) {
2690  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2691  CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2692  return;
2693  } else {
2694  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2695  Reg0 };
2696  CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2697  return;
2698  }
2699  }
2700  if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
2701  unsigned ShImm = Log2_32(RHSV+1);
2702  if (ShImm >= 32)
2703  break;
2704  SDValue V = N->getOperand(0);
2705  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2706  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2707  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2708  if (Subtarget->isThumb()) {
2709  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2710  CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2711  return;
2712  } else {
2713  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2714  Reg0 };
2715  CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2716  return;
2717  }
2718  }
2719  }
2720  break;
2721  case ISD::AND: {
2722  // Check for unsigned bitfield extract
2723  if (tryV6T2BitfieldExtractOp(N, false))
2724  return;
2725 
2726  // If an immediate is used in an AND node, it is possible that the immediate
2727  // can be more optimally materialized when negated. If this is the case we
2728  // can negate the immediate and use a BIC instead.
2729  auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2730  if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
2731  uint32_t Imm = (uint32_t) N1C->getZExtValue();
2732 
2733  // In Thumb2 mode, an AND can take a 12-bit immediate. If this
2734  // immediate can be negated and fit in the immediate operand of
2735  // a t2BIC, don't do any manual transform here as this can be
2736  // handled by the generic ISel machinery.
2737  bool PreferImmediateEncoding =
2738  Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
2739  if (!PreferImmediateEncoding &&
2740  ConstantMaterializationCost(Imm) >
2741  ConstantMaterializationCost(~Imm)) {
2742  // The current immediate costs more to materialize than a negated
2743  // immediate, so negate the immediate and use a BIC.
2744  SDValue NewImm =
2745  CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
2746  // If the new constant didn't exist before, reposition it in the topological
2747  // ordering so it is just before N. Otherwise, don't touch its location.
2748  if (NewImm->getNodeId() == -1)
2749  CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
2750 
2751  if (!Subtarget->hasThumb2()) {
2752  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
2753  N->getOperand(0), NewImm, getAL(CurDAG, dl),
2754  CurDAG->getRegister(0, MVT::i32)};
2755  ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
2756  return;
2757  } else {
2758  SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
2759  CurDAG->getRegister(0, MVT::i32),
2760  CurDAG->getRegister(0, MVT::i32)};
2761  ReplaceNode(N,
2762  CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
2763  return;
2764  }
2765  }
2766  }
2767 
2768  // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2769  // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2770  // are entirely contributed by c2 and lower 16-bits are entirely contributed
2771  // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2772  // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2773  EVT VT = N->getValueType(0);
2774  if (VT != MVT::i32)
2775  break;
2776  unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2777  ? ARM::t2MOVTi16
2778  : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2779  if (!Opc)
2780  break;
2781  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2782  N1C = dyn_cast<ConstantSDNode>(N1);
2783  if (!N1C)
2784  break;
2785  if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2786  SDValue N2 = N0.getOperand(1);
2788  if (!N2C)
2789  break;
2790  unsigned N1CVal = N1C->getZExtValue();
2791  unsigned N2CVal = N2C->getZExtValue();
2792  if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2793  (N1CVal & 0xffffU) == 0xffffU &&
2794  (N2CVal & 0xffffU) == 0x0U) {
2795  SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2796  dl, MVT::i32);
2797  SDValue Ops[] = { N0.getOperand(0), Imm16,
2798  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
2799  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
2800  return;
2801  }
2802  }
2803 
2804  break;
2805  }
2806  case ARMISD::UMAAL: {
2807  unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
2808  SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2809  N->getOperand(2), N->getOperand(3),
2810  getAL(CurDAG, dl),
2811  CurDAG->getRegister(0, MVT::i32) };
2812  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
2813  return;
2814  }
2815  case ARMISD::UMLAL:{
2816  if (Subtarget->isThumb()) {
2817  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2818  N->getOperand(3), getAL(CurDAG, dl),
2819  CurDAG->getRegister(0, MVT::i32)};
2820  ReplaceNode(
2821  N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
2822  return;
2823  }else{
2824  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2825  N->getOperand(3), getAL(CurDAG, dl),
2826  CurDAG->getRegister(0, MVT::i32),
2827  CurDAG->getRegister(0, MVT::i32) };
2828  ReplaceNode(N, CurDAG->getMachineNode(
2829  Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
2830  MVT::i32, MVT::i32, Ops));
2831  return;
2832  }
2833  }
2834  case ARMISD::SMLAL:{
2835  if (Subtarget->isThumb()) {
2836  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2837  N->getOperand(3), getAL(CurDAG, dl),
2838  CurDAG->getRegister(0, MVT::i32)};
2839  ReplaceNode(
2840  N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
2841  return;
2842  }else{
2843  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2844  N->getOperand(3), getAL(CurDAG, dl),
2845  CurDAG->getRegister(0, MVT::i32),
2846  CurDAG->getRegister(0, MVT::i32) };
2847  ReplaceNode(N, CurDAG->getMachineNode(
2848  Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
2849  MVT::i32, MVT::i32, Ops));
2850  return;
2851  }
2852  }
2853  case ARMISD::SUBE: {
2854  if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
2855  break;
2856  // Look for a pattern to match SMMLS
2857  // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
2858  if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
2859  N->getOperand(2).getOpcode() != ARMISD::SUBC ||
2860  !SDValue(N, 1).use_empty())
2861  break;
2862 
2863  if (Subtarget->isThumb())
2864  assert(Subtarget->hasThumb2() &&
2865  "This pattern should not be generated for Thumb");
2866 
2867  SDValue SmulLoHi = N->getOperand(1);
2868  SDValue Subc = N->getOperand(2);
2869  auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
2870 
2871  if (!Zero || Zero->getZExtValue() != 0 ||
2872  Subc.getOperand(1) != SmulLoHi.getValue(0) ||
2873  N->getOperand(1) != SmulLoHi.getValue(1) ||
2874  N->getOperand(2) != Subc.getValue(1))
2875  break;
2876 
2877  unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
2878  SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
2879  N->getOperand(0), getAL(CurDAG, dl),
2880  CurDAG->getRegister(0, MVT::i32) };
2881  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
2882  return;
2883  }
2884  case ISD::LOAD: {
2885  if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
2886  if (tryT2IndexedLoad(N))
2887  return;
2888  } else if (Subtarget->isThumb()) {
2889  if (tryT1IndexedLoad(N))
2890  return;
2891  } else if (tryARMIndexedLoad(N))
2892  return;
2893  // Other cases are autogenerated.
2894  break;
2895  }
2896  case ARMISD::BRCOND: {
2897  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2898  // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2899  // Pattern complexity = 6 cost = 1 size = 0
2900 
2901  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2902  // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2903  // Pattern complexity = 6 cost = 1 size = 0
2904 
2905  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2906  // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2907  // Pattern complexity = 6 cost = 1 size = 0
2908 
2909  unsigned Opc = Subtarget->isThumb() ?
2910  ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2911  SDValue Chain = N->getOperand(0);
2912  SDValue N1 = N->getOperand(1);
2913  SDValue N2 = N->getOperand(2);
2914  SDValue N3 = N->getOperand(3);
2915  SDValue InFlag = N->getOperand(4);
2916  assert(N1.getOpcode() == ISD::BasicBlock);
2917  assert(N2.getOpcode() == ISD::Constant);
2918  assert(N3.getOpcode() == ISD::Register);
2919 
2920  unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
2921 
2922  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2923  bool SwitchEQNEToPLMI;
2924  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2925  InFlag = N->getOperand(4);
2926 
2927  if (SwitchEQNEToPLMI) {
2928  switch ((ARMCC::CondCodes)CC) {
2929  default: llvm_unreachable("CMPZ must be either NE or EQ!");
2930  case ARMCC::NE:
2931  CC = (unsigned)ARMCC::MI;
2932  break;
2933  case ARMCC::EQ:
2934  CC = (unsigned)ARMCC::PL;
2935  break;
2936  }
2937  }
2938  }
2939 
2940  SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
2941  SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2942  SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2943  MVT::Glue, Ops);
2944  Chain = SDValue(ResNode, 0);
2945  if (N->getNumValues() == 2) {
2946  InFlag = SDValue(ResNode, 1);
2947  ReplaceUses(SDValue(N, 1), InFlag);
2948  }
2949  ReplaceUses(SDValue(N, 0),
2950  SDValue(Chain.getNode(), Chain.getResNo()));
2951  CurDAG->RemoveDeadNode(N);
2952  return;
2953  }
2954 
2955  case ARMISD::CMPZ: {
2956  // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
2957  // This allows us to avoid materializing the expensive negative constant.
2958  // The CMPZ #0 is useless and will be peepholed away but we need to keep it
2959  // for its glue output.
2960  SDValue X = N->getOperand(0);
2961  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
2962  if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
2963  int64_t Addend = -C->getSExtValue();
2964 
2965  SDNode *Add = nullptr;
2966  // ADDS can be better than CMN if the immediate fits in a
2967  // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
2968  // Outside that range we can just use a CMN which is 32-bit but has a
2969  // 12-bit immediate range.
2970  if (Addend < 1<<8) {
2971  if (Subtarget->isThumb2()) {
2972  SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
2973  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2974  CurDAG->getRegister(0, MVT::i32) };
2975  Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
2976  } else {
2977  unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
2978  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
2979  CurDAG->getTargetConstant(Addend, dl, MVT::i32),
2980  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2981  Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2982  }
2983  }
2984  if (Add) {
2985  SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
2986  CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
2987  }
2988  }
2989  // Other cases are autogenerated.
2990  break;
2991  }
2992 
2993  case ARMISD::CMOV: {
2994  SDValue InFlag = N->getOperand(4);
2995 
2996  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2997  bool SwitchEQNEToPLMI;
2998  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2999 
3000  if (SwitchEQNEToPLMI) {
3001  SDValue ARMcc = N->getOperand(2);
3002  ARMCC::CondCodes CC =
3003  (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
3004 
3005  switch (CC) {
3006  default: llvm_unreachable("CMPZ must be either NE or EQ!");
3007  case ARMCC::NE:
3008  CC = ARMCC::MI;
3009  break;
3010  case ARMCC::EQ:
3011  CC = ARMCC::PL;
3012  break;
3013  }
3014  SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
3015  SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
3016  N->getOperand(3), N->getOperand(4)};
3017  CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
3018  }
3019 
3020  }
3021  // Other cases are autogenerated.
3022  break;
3023  }
3024 
3025  case ARMISD::VZIP: {
3026  unsigned Opc = 0;
3027  EVT VT = N->getValueType(0);
3028  switch (VT.getSimpleVT().SimpleTy) {
3029  default: return;
3030  case MVT::v8i8: Opc = ARM::VZIPd8; break;
3031  case MVT::v4f16:
3032  case MVT::v4i16: Opc = ARM::VZIPd16; break;
3033  case MVT::v2f32:
3034  // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3035  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3036  case MVT::v16i8: Opc = ARM::VZIPq8; break;
3037  case MVT::v8f16:
3038  case MVT::v8i16: Opc = ARM::VZIPq16; break;
3039  case MVT::v4f32:
3040  case MVT::v4i32: Opc = ARM::VZIPq32; break;
3041  }
3042  SDValue Pred = getAL(CurDAG, dl);
3043  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3044  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3045  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3046  return;
3047  }
3048  case ARMISD::VUZP: {
3049  unsigned Opc = 0;
3050  EVT VT = N->getValueType(0);
3051  switch (VT.getSimpleVT().SimpleTy) {
3052  default: return;
3053  case MVT::v8i8: Opc = ARM::VUZPd8; break;
3054  case MVT::v4f16:
3055  case MVT::v4i16: Opc = ARM::VUZPd16; break;
3056  case MVT::v2f32:
3057  // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3058  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3059  case MVT::v16i8: Opc = ARM::VUZPq8; break;
3060  case MVT::v8f16:
3061  case MVT::v8i16: Opc = ARM::VUZPq16; break;
3062  case MVT::v4f32:
3063  case MVT::v4i32: Opc = ARM::VUZPq32; break;
3064  }
3065  SDValue Pred = getAL(CurDAG, dl);
3066  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3067  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3068  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3069  return;
3070  }
3071  case ARMISD::VTRN: {
3072  unsigned Opc = 0;
3073  EVT VT = N->getValueType(0);
3074  switch (VT.getSimpleVT().SimpleTy) {
3075  default: return;
3076  case MVT::v8i8: Opc = ARM::VTRNd8; break;
3077  case MVT::v4f16:
3078  case MVT::v4i16: Opc = ARM::VTRNd16; break;
3079  case MVT::v2f32:
3080  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3081  case MVT::v16i8: Opc = ARM::VTRNq8; break;
3082  case MVT::v8f16:
3083  case MVT::v8i16: Opc = ARM::VTRNq16; break;
3084  case MVT::v4f32:
3085  case MVT::v4i32: Opc = ARM::VTRNq32; break;
3086  }
3087  SDValue Pred = getAL(CurDAG, dl);
3088  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3089  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3090  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3091  return;
3092  }
3093  case ARMISD::BUILD_VECTOR: {
3094  EVT VecVT = N->getValueType(0);
3095  EVT EltVT = VecVT.getVectorElementType();
3096  unsigned NumElts = VecVT.getVectorNumElements();
3097  if (EltVT == MVT::f64) {
3098  assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
3099  ReplaceNode(
3100  N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3101  return;
3102  }
3103  assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
3104  if (NumElts == 2) {
3105  ReplaceNode(
3106  N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3107  return;
3108  }
3109  assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
3110  ReplaceNode(N,
3111  createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
3112  N->getOperand(2), N->getOperand(3)));
3113  return;
3114  }
3115 
3116  case ARMISD::VLD1DUP: {
3117  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
3118  ARM::VLD1DUPd32 };
3119  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
3120  ARM::VLD1DUPq32 };
3121  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 1, DOpcodes, QOpcodes);
3122  return;
3123  }
3124 
3125  case ARMISD::VLD2DUP: {
3126  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3127  ARM::VLD2DUPd32 };
3128  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 2, Opcodes);
3129  return;
3130  }
3131 
3132  case ARMISD::VLD3DUP: {
3133  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
3134  ARM::VLD3DUPd16Pseudo,
3135  ARM::VLD3DUPd32Pseudo };
3136  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 3, Opcodes);
3137  return;
3138  }
3139 
3140  case ARMISD::VLD4DUP: {
3141  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
3142  ARM::VLD4DUPd16Pseudo,
3143  ARM::VLD4DUPd32Pseudo };
3144  SelectVLDDup(N, /* IsIntrinsic= */ false, false, 4, Opcodes);
3145  return;
3146  }
3147 
3148  case ARMISD::VLD1DUP_UPD: {
3149  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
3150  ARM::VLD1DUPd16wb_fixed,
3151  ARM::VLD1DUPd32wb_fixed };
3152  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
3153  ARM::VLD1DUPq16wb_fixed,
3154  ARM::VLD1DUPq32wb_fixed };
3155  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 1, DOpcodes, QOpcodes);
3156  return;
3157  }
3158 
3159  case ARMISD::VLD2DUP_UPD: {
3160  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
3161  ARM::VLD2DUPd16wb_fixed,
3162  ARM::VLD2DUPd32wb_fixed };
3163  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 2, Opcodes);
3164  return;
3165  }
3166 
3167  case ARMISD::VLD3DUP_UPD: {
3168  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
3169  ARM::VLD3DUPd16Pseudo_UPD,
3170  ARM::VLD3DUPd32Pseudo_UPD };
3171  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 3, Opcodes);
3172  return;
3173  }
3174 
3175  case ARMISD::VLD4DUP_UPD: {
3176  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
3177  ARM::VLD4DUPd16Pseudo_UPD,
3178  ARM::VLD4DUPd32Pseudo_UPD };
3179  SelectVLDDup(N, /* IsIntrinsic= */ false, true, 4, Opcodes);
3180  return;
3181  }
3182 
3183  case ARMISD::VLD1_UPD: {
3184  static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
3185  ARM::VLD1d16wb_fixed,
3186  ARM::VLD1d32wb_fixed,
3187  ARM::VLD1d64wb_fixed };
3188  static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
3189  ARM::VLD1q16wb_fixed,
3190  ARM::VLD1q32wb_fixed,
3191  ARM::VLD1q64wb_fixed };
3192  SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
3193  return;
3194  }
3195 
3196  case ARMISD::VLD2_UPD: {
3197  static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
3198  ARM::VLD2d16wb_fixed,
3199  ARM::VLD2d32wb_fixed,
3200  ARM::VLD1q64wb_fixed};
3201  static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
3202  ARM::VLD2q16PseudoWB_fixed,
3203  ARM::VLD2q32PseudoWB_fixed };
3204  SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
3205  return;
3206  }
3207 
3208  case ARMISD::VLD3_UPD: {
3209  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
3210  ARM::VLD3d16Pseudo_UPD,
3211  ARM::VLD3d32Pseudo_UPD,
3212  ARM::VLD1d64TPseudoWB_fixed};
3213  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3214  ARM::VLD3q16Pseudo_UPD,
3215  ARM::VLD3q32Pseudo_UPD };
3216  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
3217  ARM::VLD3q16oddPseudo_UPD,
3218  ARM::VLD3q32oddPseudo_UPD };
3219  SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3220  return;
3221  }
3222 
3223  case ARMISD::VLD4_UPD: {
3224  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
3225  ARM::VLD4d16Pseudo_UPD,
3226  ARM::VLD4d32Pseudo_UPD,
3227  ARM::VLD1d64QPseudoWB_fixed};
3228  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3229  ARM::VLD4q16Pseudo_UPD,
3230  ARM::VLD4q32Pseudo_UPD };
3231  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
3232  ARM::VLD4q16oddPseudo_UPD,
3233  ARM::VLD4q32oddPseudo_UPD };
3234  SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3235  return;
3236  }
3237 
3238  case ARMISD::VLD2LN_UPD: {
3239  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
3240  ARM::VLD2LNd16Pseudo_UPD,
3241  ARM::VLD2LNd32Pseudo_UPD };
3242  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
3243  ARM::VLD2LNq32Pseudo_UPD };
3244  SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
3245  return;
3246  }
3247 
3248  case ARMISD::VLD3LN_UPD: {
3249  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
3250  ARM::VLD3LNd16Pseudo_UPD,
3251  ARM::VLD3LNd32Pseudo_UPD };
3252  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
3253  ARM::VLD3LNq32Pseudo_UPD };
3254  SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
3255  return;
3256  }
3257 
3258  case ARMISD::VLD4LN_UPD: {
3259  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3260  ARM::VLD4LNd16Pseudo_UPD,
3261  ARM::VLD4LNd32Pseudo_UPD };
3262  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3263  ARM::VLD4LNq32Pseudo_UPD };
3264  SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3265  return;
3266  }
3267 
3268  case ARMISD::VST1_UPD: {
3269  static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3270  ARM::VST1d16wb_fixed,
3271  ARM::VST1d32wb_fixed,
3272  ARM::VST1d64wb_fixed };
3273  static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3274  ARM::VST1q16wb_fixed,
3275  ARM::VST1q32wb_fixed,
3276  ARM::VST1q64wb_fixed };
3277  SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
3278  return;
3279  }
3280 
3281  case ARMISD::VST2_UPD: {
3282  static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3283  ARM::VST2d16wb_fixed,
3284  ARM::VST2d32wb_fixed,
3285  ARM::VST1q64wb_fixed};
3286  static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3287  ARM::VST2q16PseudoWB_fixed,
3288  ARM::VST2q32PseudoWB_fixed };
3289  SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
3290  return;
3291  }
3292 
3293  case ARMISD::VST3_UPD: {
3294  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3295  ARM::VST3d16Pseudo_UPD,
3296  ARM::VST3d32Pseudo_UPD,
3297  ARM::VST1d64TPseudoWB_fixed};
3298  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3299  ARM::VST3q16Pseudo_UPD,
3300  ARM::VST3q32Pseudo_UPD };
3301  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3302  ARM::VST3q16oddPseudo_UPD,
3303  ARM::VST3q32oddPseudo_UPD };
3304  SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3305  return;
3306  }
3307 
3308  case ARMISD::VST4_UPD: {
3309  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3310  ARM::VST4d16Pseudo_UPD,
3311  ARM::VST4d32Pseudo_UPD,
3312  ARM::VST1d64QPseudoWB_fixed};
3313  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3314  ARM::VST4q16Pseudo_UPD,
3315  ARM::VST4q32Pseudo_UPD };
3316  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3317  ARM::VST4q16oddPseudo_UPD,
3318  ARM::VST4q32oddPseudo_UPD };
3319  SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3320  return;
3321  }
3322 
3323  case ARMISD::VST2LN_UPD: {
3324  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3325  ARM::VST2LNd16Pseudo_UPD,
3326  ARM::VST2LNd32Pseudo_UPD };
3327  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3328  ARM::VST2LNq32Pseudo_UPD };
3329  SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3330  return;
3331  }
3332 
3333  case ARMISD::VST3LN_UPD: {
3334  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3335  ARM::VST3LNd16Pseudo_UPD,
3336  ARM::VST3LNd32Pseudo_UPD };
3337  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3338  ARM::VST3LNq32Pseudo_UPD };
3339  SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3340  return;
3341  }
3342 
3343  case ARMISD::VST4LN_UPD: {
3344  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3345  ARM::VST4LNd16Pseudo_UPD,
3346  ARM::VST4LNd32Pseudo_UPD };
3347  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3348  ARM::VST4LNq32Pseudo_UPD };
3349  SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3350  return;
3351  }
3352 
3353  case ISD::INTRINSIC_VOID:
3354  case ISD::INTRINSIC_W_CHAIN: {
3355  unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3356  switch (IntNo) {
3357  default:
3358  break;
3359 
3360  case Intrinsic::arm_mrrc:
3361  case Intrinsic::arm_mrrc2: {
3362  SDLoc dl(N);
3363  SDValue Chain = N->getOperand(0);
3364  unsigned Opc;
3365 
3366  if (Subtarget->isThumb())
3367  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
3368  else
3369  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
3370 
3372  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
3373  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
3374  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
3375 
3376  // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
3377  // instruction will always be '1111' but it is possible in assembly language to specify
3378  // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
3379  if (Opc != ARM::MRRC2) {
3380  Ops.push_back(getAL(CurDAG, dl));
3381  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3382  }
3383 
3384  Ops.push_back(Chain);
3385 
3386  // Writes to two registers.
3387  const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
3388 
3389  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
3390  return;
3391  }
3392  case Intrinsic::arm_ldaexd:
3393  case Intrinsic::arm_ldrexd: {
3394  SDLoc dl(N);
3395  SDValue Chain = N->getOperand(0);
3396  SDValue MemAddr = N->getOperand(2);
3397  bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
3398 
3399  bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3400  unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3401  : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3402 
3403  // arm_ldrexd returns a i64 value in {i32, i32}
3404  std::vector<EVT> ResTys;
3405  if (isThumb) {
3406  ResTys.push_back(MVT::i32);
3407  ResTys.push_back(MVT::i32);
3408  } else
3409  ResTys.push_back(MVT::Untyped);
3410  ResTys.push_back(MVT::Other);
3411 
3412  // Place arguments in the right order.
3413  SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3414  CurDAG->getRegister(0, MVT::i32), Chain};
3415  SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3416  // Transfer memoperands.
3417  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3418  CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp});
3419 
3420  // Remap uses.
3421  SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3422  if (!SDValue(N, 0).use_empty()) {
3423  SDValue Result;
3424  if (isThumb)
3425  Result = SDValue(Ld, 0);
3426  else {
3427  SDValue SubRegIdx =
3428  CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
3429  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3430  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3431  Result = SDValue(ResNode,0);
3432  }
3433  ReplaceUses(SDValue(N, 0), Result);
3434  }
3435  if (!SDValue(N, 1).use_empty()) {
3436  SDValue Result;
3437  if (isThumb)
3438  Result = SDValue(Ld, 1);
3439  else {
3440  SDValue SubRegIdx =
3441  CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
3442  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3443  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3444  Result = SDValue(ResNode,0);
3445  }
3446  ReplaceUses(SDValue(N, 1), Result);
3447  }
3448  ReplaceUses(SDValue(N, 2), OutChain);
3449  CurDAG->RemoveDeadNode(N);
3450  return;
3451  }
3452  case Intrinsic::arm_stlexd:
3453  case Intrinsic::arm_strexd: {
3454  SDLoc dl(N);
3455  SDValue Chain = N->getOperand(0);
3456  SDValue Val0 = N->getOperand(2);
3457  SDValue Val1 = N->getOperand(3);
3458  SDValue MemAddr = N->getOperand(4);
3459 
3460  // Store exclusive double return a i32 value which is the return status
3461  // of the issued store.
3462  const EVT ResTys[] = {MVT::i32, MVT::Other};
3463 
3464  bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3465  // Place arguments in the right order.
3467  if (isThumb) {
3468  Ops.push_back(Val0);
3469  Ops.push_back(Val1);
3470  } else
3471  // arm_strexd uses GPRPair.
3472  Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3473  Ops.push_back(MemAddr);
3474  Ops.push_back(getAL(CurDAG, dl));
3475  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3476  Ops.push_back(Chain);
3477 
3478  bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3479  unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3480  : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3481 
3482  SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3483  // Transfer memoperands.
3484  MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3485  CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp});
3486 
3487  ReplaceNode(N, St);
3488  return;
3489  }
3490 
3491  case Intrinsic::arm_neon_vld1: {
3492  static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3493  ARM::VLD1d32, ARM::VLD1d64 };
3494  static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3495  ARM::VLD1q32, ARM::VLD1q64};
3496  SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3497  return;
3498  }
3499 
3500  case Intrinsic::arm_neon_vld1x2: {
3501  static const uint16_t DOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3502  ARM::VLD1q32, ARM::VLD1q64 };
3503  static const uint16_t QOpcodes[] = { ARM::VLD1d8QPseudo,
3504  ARM::VLD1d16QPseudo,
3505  ARM::VLD1d32QPseudo,
3506  ARM::VLD1d64QPseudo };
3507  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3508  return;
3509  }
3510 
3511  case Intrinsic::arm_neon_vld1x3: {
3512  static const uint16_t DOpcodes[] = { ARM::VLD1d8TPseudo,
3513  ARM::VLD1d16TPseudo,
3514  ARM::VLD1d32TPseudo,
3515  ARM::VLD1d64TPseudo };
3516  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowTPseudo_UPD,
3517  ARM::VLD1q16LowTPseudo_UPD,
3518  ARM::VLD1q32LowTPseudo_UPD,
3519  ARM::VLD1q64LowTPseudo_UPD };
3520  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighTPseudo,
3521  ARM::VLD1q16HighTPseudo,
3522  ARM::VLD1q32HighTPseudo,
3523  ARM::VLD1q64HighTPseudo };
3524  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3525  return;
3526  }
3527 
3528  case Intrinsic::arm_neon_vld1x4: {
3529  static const uint16_t DOpcodes[] = { ARM::VLD1d8QPseudo,
3530  ARM::VLD1d16QPseudo,
3531  ARM::VLD1d32QPseudo,
3532  ARM::VLD1d64QPseudo };
3533  static const uint16_t QOpcodes0[] = { ARM::VLD1q8LowQPseudo_UPD,
3534  ARM::VLD1q16LowQPseudo_UPD,
3535  ARM::VLD1q32LowQPseudo_UPD,
3536  ARM::VLD1q64LowQPseudo_UPD };
3537  static const uint16_t QOpcodes1[] = { ARM::VLD1q8HighQPseudo,
3538  ARM::VLD1q16HighQPseudo,
3539  ARM::VLD1q32HighQPseudo,
3540  ARM::VLD1q64HighQPseudo };
3541  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3542  return;
3543  }
3544 
3545  case Intrinsic::arm_neon_vld2: {
3546  static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3547  ARM::VLD2d32, ARM::VLD1q64 };
3548  static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3549  ARM::VLD2q32Pseudo };
3550  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3551  return;
3552  }
3553 
3554  case Intrinsic::arm_neon_vld3: {
3555  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3556  ARM::VLD3d16Pseudo,
3557  ARM::VLD3d32Pseudo,
3558  ARM::VLD1d64TPseudo };
3559  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3560  ARM::VLD3q16Pseudo_UPD,
3561  ARM::VLD3q32Pseudo_UPD };
3562  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3563  ARM::VLD3q16oddPseudo,
3564  ARM::VLD3q32oddPseudo };
3565  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3566  return;
3567  }
3568 
3569  case Intrinsic::arm_neon_vld4: {
3570  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3571  ARM::VLD4d16Pseudo,
3572  ARM::VLD4d32Pseudo,
3573  ARM::VLD1d64QPseudo };
3574  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3575  ARM::VLD4q16Pseudo_UPD,
3576  ARM::VLD4q32Pseudo_UPD };
3577  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3578  ARM::VLD4q16oddPseudo,
3579  ARM::VLD4q32oddPseudo };
3580  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3581  return;
3582  }
3583 
3584  case Intrinsic::arm_neon_vld2dup: {
3585  static const uint16_t DOpcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3586  ARM::VLD2DUPd32, ARM::VLD1q64 };
3587  static const uint16_t QOpcodes0[] = { ARM::VLD2DUPq8EvenPseudo,
3588  ARM::VLD2DUPq16EvenPseudo,
3589  ARM::VLD2DUPq32EvenPseudo };
3590  static const uint16_t QOpcodes1[] = { ARM::VLD2DUPq8OddPseudo,
3591  ARM::VLD2DUPq16OddPseudo,
3592  ARM::VLD2DUPq32OddPseudo };
3593  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 2,
3594  DOpcodes, QOpcodes0, QOpcodes1);
3595  return;
3596  }
3597 
3598  case Intrinsic::arm_neon_vld3dup: {
3599  static const uint16_t DOpcodes[] = { ARM::VLD3DUPd8Pseudo,
3600  ARM::VLD3DUPd16Pseudo,
3601  ARM::VLD3DUPd32Pseudo,
3602  ARM::VLD1d64TPseudo };
3603  static const uint16_t QOpcodes0[] = { ARM::VLD3DUPq8EvenPseudo,
3604  ARM::VLD3DUPq16EvenPseudo,
3605  ARM::VLD3DUPq32EvenPseudo };
3606  static const uint16_t QOpcodes1[] = { ARM::VLD3DUPq8OddPseudo,
3607  ARM::VLD3DUPq16OddPseudo,
3608  ARM::VLD3DUPq32OddPseudo };
3609  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 3,
3610  DOpcodes, QOpcodes0, QOpcodes1);
3611  return;
3612  }
3613 
3614  case Intrinsic::arm_neon_vld4dup: {
3615  static const uint16_t DOpcodes[] = { ARM::VLD4DUPd8Pseudo,
3616  ARM::VLD4DUPd16Pseudo,
3617  ARM::VLD4DUPd32Pseudo,
3618  ARM::VLD1d64QPseudo };
3619  static const uint16_t QOpcodes0[] = { ARM::VLD4DUPq8EvenPseudo,
3620  ARM::VLD4DUPq16EvenPseudo,
3621  ARM::VLD4DUPq32EvenPseudo };
3622  static const uint16_t QOpcodes1[] = { ARM::VLD4DUPq8OddPseudo,
3623  ARM::VLD4DUPq16OddPseudo,
3624  ARM::VLD4DUPq32OddPseudo };
3625  SelectVLDDup(N, /* IsIntrinsic= */ true, false, 4,
3626  DOpcodes, QOpcodes0, QOpcodes1);
3627  return;
3628  }
3629 
3630  case Intrinsic::arm_neon_vld2lane: {
3631  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3632  ARM::VLD2LNd16Pseudo,
3633  ARM::VLD2LNd32Pseudo };
3634  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3635  ARM::VLD2LNq32Pseudo };
3636  SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3637  return;
3638  }
3639 
3640  case Intrinsic::arm_neon_vld3lane: {
3641  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3642  ARM::VLD3LNd16Pseudo,
3643  ARM::VLD3LNd32Pseudo };
3644  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3645  ARM::VLD3LNq32Pseudo };
3646  SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3647  return;
3648  }
3649 
3650  case Intrinsic::arm_neon_vld4lane: {
3651  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3652  ARM::VLD4LNd16Pseudo,
3653  ARM::VLD4LNd32Pseudo };
3654  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3655  ARM::VLD4LNq32Pseudo };
3656  SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3657  return;
3658  }
3659 
3660  case Intrinsic::arm_neon_vst1: {
3661  static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3662  ARM::VST1d32, ARM::VST1d64 };
3663  static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3664  ARM::VST1q32, ARM::VST1q64 };
3665  SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3666  return;
3667  }
3668 
3669  case Intrinsic::arm_neon_vst1x2: {
3670  static const uint16_t DOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3671  ARM::VST1q32, ARM::VST1q64 };
3672  static const uint16_t QOpcodes[] = { ARM::VST1d8QPseudo,
3673  ARM::VST1d16QPseudo,
3674  ARM::VST1d32QPseudo,
3675  ARM::VST1d64QPseudo };
3676  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3677  return;
3678  }
3679 
3680  case Intrinsic::arm_neon_vst1x3: {
3681  static const uint16_t DOpcodes[] = { ARM::VST1d8TPseudo,
3682  ARM::VST1d16TPseudo,
3683  ARM::VST1d32TPseudo,
3684  ARM::VST1d64TPseudo };
3685  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowTPseudo_UPD,
3686  ARM::VST1q16LowTPseudo_UPD,
3687  ARM::VST1q32LowTPseudo_UPD,
3688  ARM::VST1q64LowTPseudo_UPD };
3689  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighTPseudo,
3690  ARM::VST1q16HighTPseudo,
3691  ARM::VST1q32HighTPseudo,
3692  ARM::VST1q64HighTPseudo };
3693  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3694  return;
3695  }
3696 
3697  case Intrinsic::arm_neon_vst1x4: {
3698  static const uint16_t DOpcodes[] = { ARM::VST1d8QPseudo,
3699  ARM::VST1d16QPseudo,
3700  ARM::VST1d32QPseudo,
3701  ARM::VST1d64QPseudo };
3702  static const uint16_t QOpcodes0[] = { ARM::VST1q8LowQPseudo_UPD,
3703  ARM::VST1q16LowQPseudo_UPD,
3704  ARM::VST1q32LowQPseudo_UPD,
3705  ARM::VST1q64LowQPseudo_UPD };
3706  static const uint16_t QOpcodes1[] = { ARM::VST1q8HighQPseudo,
3707  ARM::VST1q16HighQPseudo,
3708  ARM::VST1q32HighQPseudo,
3709  ARM::VST1q64HighQPseudo };
3710  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3711  return;
3712  }
3713 
3714  case Intrinsic::arm_neon_vst2: {
3715  static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3716  ARM::VST2d32, ARM::VST1q64 };
3717  static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3718  ARM::VST2q32Pseudo };
3719  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3720  return;
3721  }
3722 
3723  case Intrinsic::arm_neon_vst3: {
3724  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3725  ARM::VST3d16Pseudo,
3726  ARM::VST3d32Pseudo,
3727  ARM::VST1d64TPseudo };
3728  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3729  ARM::VST3q16Pseudo_UPD,
3730  ARM::VST3q32Pseudo_UPD };
3731  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3732  ARM::VST3q16oddPseudo,
3733  ARM::VST3q32oddPseudo };
3734  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3735  return;
3736  }
3737 
3738  case Intrinsic::arm_neon_vst4: {
3739  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3740  ARM::VST4d16Pseudo,
3741  ARM::VST4d32Pseudo,
3742  ARM::VST1d64QPseudo };
3743  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3744  ARM::VST4q16Pseudo_UPD,
3745  ARM::VST4q32Pseudo_UPD };
3746  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3747  ARM::VST4q16oddPseudo,
3748  ARM::VST4q32oddPseudo };
3749  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3750  return;
3751  }
3752 
3753  case Intrinsic::arm_neon_vst2lane: {
3754  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3755  ARM::VST2LNd16Pseudo,
3756  ARM::VST2LNd32Pseudo };
3757  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3758  ARM::VST2LNq32Pseudo };
3759  SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3760  return;
3761  }
3762 
3763  case Intrinsic::arm_neon_vst3lane: {
3764  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3765  ARM::VST3LNd16Pseudo,
3766  ARM::VST3LNd32Pseudo };
3767  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3768  ARM::VST3LNq32Pseudo };
3769  SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3770  return;
3771  }
3772 
3773  case Intrinsic::arm_neon_vst4lane: {
3774  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3775  ARM::VST4LNd16Pseudo,
3776  ARM::VST4LNd32Pseudo };
3777  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3778  ARM::VST4LNq32Pseudo };
3779  SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3780  return;
3781  }
3782  }
3783  break;
3784  }
3785 
3786  case ISD::ATOMIC_CMP_SWAP:
3787  SelectCMP_SWAP(N);
3788  return;
3789  }
3790 
3791  SelectCode(N);
3792 }
3793 
3794 // Inspect a register string of the form
3795 // cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
3796 // cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
3797 // and obtain the integer operands from them, adding these operands to the
3798 // provided vector.
3800  SelectionDAG *CurDAG,
3801  const SDLoc &DL,
3802  std::vector<SDValue> &Ops) {
3804  RegString.split(Fields, ':');
3805 
3806  if (Fields.size() > 1) {
3807  bool AllIntFields = true;
3808 
3809  for (StringRef Field : Fields) {
3810  // Need to trim out leading 'cp' characters and get the integer field.
3811  unsigned IntField;
3812  AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
3813  Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
3814  }
3815 
3816  assert(AllIntFields &&
3817  "Unexpected non-integer value in special register string.");
3818  }
3819 }
3820 
3821 // Maps a Banked Register string to its mask value. The mask value returned is
3822 // for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
3823 // mask operand, which expresses which register is to be used, e.g. r8, and in
3824 // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
3825 // was invalid.
3826 static inline int getBankedRegisterMask(StringRef RegString) {
3827  auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
3828  if (!TheReg)
3829  return -1;
3830  return TheReg->Encoding;
3831 }
3832 
3833 // The flags here are common to those allowed for apsr in the A class cores and
3834 // those allowed for the special registers in the M class cores. Returns a
3835 // value representing which flags were present, -1 if invalid.
3836 static inline int getMClassFlagsMask(StringRef Flags) {
3837  return StringSwitch<int>(Flags)
3838  .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
3839  // correct when flags are not permitted
3840  .Case("g", 0x1)
3841  .Case("nzcvq", 0x2)
3842  .Case("nzcvqg", 0x3)
3843  .Default(-1);
3844 }
3845 
3846 // Maps MClass special registers string to its value for use in the
3847 // t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
3848 // Returns -1 to signify that the string was invalid.
3849 static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
3850  auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
3851  const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
3852  if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
3853  return -1;
3854  return (int)(TheReg->Encoding & 0xFFF); // SYSm value
3855 }
3856 
3858  // The mask operand contains the special register (R Bit) in bit 4, whether
3859  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
3860  // bits 3-0 contains the fields to be accessed in the special register, set by
3861  // the flags provided with the register.
3862  int Mask = 0;
3863  if (Reg == "apsr") {
3864  // The flags permitted for apsr are the same flags that are allowed in
3865  // M class registers. We get the flag value and then shift the flags into
3866  // the correct place to combine with the mask.
3867  Mask = getMClassFlagsMask(Flags);
3868  if (Mask == -1)
3869  return -1;
3870  return Mask << 2;
3871  }
3872 
3873  if (Reg != "cpsr" && Reg != "spsr") {
3874  return -1;
3875  }
3876 
3877  // This is the same as if the flags were "fc"
3878  if (Flags.empty() || Flags == "all")
3879  return Mask | 0x9;
3880 
3881  // Inspect the supplied flags string and set the bits in the mask for
3882  // the relevant and valid flags allowed for cpsr and spsr.
3883  for (char Flag : Flags) {
3884  int FlagVal;
3885  switch (Flag) {
3886  case 'c':
3887  FlagVal = 0x1;
3888  break;
3889  case 'x':
3890  FlagVal = 0x2;
3891  break;
3892  case 's':
3893  FlagVal = 0x4;
3894  break;
3895  case 'f':
3896  FlagVal = 0x8;
3897  break;
3898  default:
3899  FlagVal = 0;
3900  }
3901 
3902  // This avoids allowing strings where the same flag bit appears twice.
3903  if (!FlagVal || (Mask & FlagVal))
3904  return -1;
3905  Mask |= FlagVal;
3906  }
3907 
3908  // If the register is spsr then we need to set the R bit.
3909  if (Reg == "spsr")
3910  Mask |= 0x10;
3911 
3912  return Mask;
3913 }
3914 
3915 // Lower the read_register intrinsic to ARM specific DAG nodes
3916 // using the supplied metadata string to select the instruction node to use
3917 // and the registers/masks to construct as operands for the node.
3918 bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
3919  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
3920  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
3921  bool IsThumb2 = Subtarget->isThumb2();
3922  SDLoc DL(N);
3923 
3924  std::vector<SDValue> Ops;
3925  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
3926 
3927  if (!Ops.empty()) {
3928  // If the special register string was constructed of fields (as defined
3929  // in the ACLE) then need to lower to MRC node (32 bit) or
3930  // MRRC node(64 bit), we can make the distinction based on the number of
3931  // operands we have.
3932  unsigned Opcode;
3933  SmallVector<EVT, 3> ResTypes;
3934  if (Ops.size() == 5){
3935  Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
3936  ResTypes.append({ MVT::i32, MVT::Other });
3937  } else {
3938  assert(Ops.size() == 3 &&
3939  "Invalid number of fields in special register string.");
3940  Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
3941  ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
3942  }
3943 
3944  Ops.push_back(getAL(CurDAG, DL));
3945  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3946  Ops.push_back(N->getOperand(0));
3947  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
3948  return true;
3949  }
3950 
3951  std::string SpecialReg = RegString->getString().lower();
3952 
3953  int BankedReg = getBankedRegisterMask(SpecialReg);
3954  if (BankedReg != -1) {
3955  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
3956  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3957  N->getOperand(0) };
3958  ReplaceNode(
3959  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
3960  DL, MVT::i32, MVT::Other, Ops));
3961  return true;
3962  }
3963 
3964  // The VFP registers are read by creating SelectionDAG nodes with opcodes
3965  // corresponding to the register that is being read from. So we switch on the
3966  // string to find which opcode we need to use.
3967  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
3968  .Case("fpscr", ARM::VMRS)
3969  .Case("fpexc", ARM::VMRS_FPEXC)
3970  .Case("fpsid", ARM::VMRS_FPSID)
3971  .Case("mvfr0", ARM::VMRS_MVFR0)
3972  .Case("mvfr1", ARM::VMRS_MVFR1)
3973  .Case("mvfr2", ARM::VMRS_MVFR2)
3974  .Case("fpinst", ARM::VMRS_FPINST)
3975  .Case("fpinst2", ARM::VMRS_FPINST2)
3976  .Default(0);
3977 
3978  // If an opcode was found then we can lower the read to a VFP instruction.
3979  if (Opcode) {
3980  if (!Subtarget->hasVFP2())
3981  return false;
3982  if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8())
3983  return false;
3984 
3985  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3986  N->getOperand(0) };
3987  ReplaceNode(N,
3988  CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
3989  return true;
3990  }
3991 
3992  // If the target is M Class then need to validate that the register string
3993  // is an acceptable value, so check that a mask can be constructed from the
3994  // string.
3995  if (Subtarget->isMClass()) {
3996  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
3997  if (SYSmValue == -1)
3998  return false;
3999 
4000  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4001  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4002  N->getOperand(0) };
4003  ReplaceNode(
4004  N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
4005  return true;
4006  }
4007 
4008  // Here we know the target is not M Class so we need to check if it is one
4009  // of the remaining possible values which are apsr, cpsr or spsr.
4010  if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
4011  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4012  N->getOperand(0) };
4013  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
4014  DL, MVT::i32, MVT::Other, Ops));
4015  return true;
4016  }
4017 
4018  if (SpecialReg == "spsr") {
4019  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4020  N->getOperand(0) };
4021  ReplaceNode(
4022  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
4023  MVT::i32, MVT::Other, Ops));
4024  return true;
4025  }
4026 
4027  return false;
4028 }
4029 
4030 // Lower the write_register intrinsic to ARM specific DAG nodes
4031 // using the supplied metadata string to select the instruction node to use
4032 // and the registers/masks to use in the nodes
4033 bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
4034  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
4035  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
4036  bool IsThumb2 = Subtarget->isThumb2();
4037  SDLoc DL(N);
4038 
4039  std::vector<SDValue> Ops;
4040  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
4041 
4042  if (!Ops.empty()) {
4043  // If the special register string was constructed of fields (as defined
4044  // in the ACLE) then need to lower to MCR node (32 bit) or
4045  // MCRR node(64 bit), we can make the distinction based on the number of
4046  // operands we have.
4047  unsigned Opcode;
4048  if (Ops.size() == 5) {
4049  Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
4050  Ops.insert(Ops.begin()+2, N->getOperand(2));
4051  } else {
4052  assert(Ops.size() == 3 &&
4053  "Invalid number of fields in special register string.");
4054  Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
4055  SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
4056  Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
4057  }
4058 
4059  Ops.push_back(getAL(CurDAG, DL));
4060  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
4061  Ops.push_back(N->getOperand(0));
4062 
4063  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4064  return true;
4065  }
4066 
4067  std::string SpecialReg = RegString->getString().lower();
4068  int BankedReg = getBankedRegisterMask(SpecialReg);
4069  if (BankedReg != -1) {
4070  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
4071  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4072  N->getOperand(0) };
4073  ReplaceNode(
4074  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
4075  DL, MVT::Other, Ops));
4076  return true;
4077  }
4078 
4079  // The VFP registers are written to by creating SelectionDAG nodes with
4080  // opcodes corresponding to the register that is being written. So we switch
4081  // on the string to find which opcode we need to use.
4082  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
4083  .Case("fpscr", ARM::VMSR)
4084  .Case("fpexc", ARM::VMSR_FPEXC)
4085  .Case("fpsid", ARM::VMSR_FPSID)
4086  .Case("fpinst", ARM::VMSR_FPINST)
4087  .Case("fpinst2", ARM::VMSR_FPINST2)
4088  .Default(0);
4089 
4090  if (Opcode) {
4091  if (!Subtarget->hasVFP2())
4092  return false;
4093  Ops = { N->getOperand(2), getAL(CurDAG, DL),
4094  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4095  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
4096  return true;
4097  }
4098 
4099  std::pair<StringRef, StringRef> Fields;
4100  Fields = StringRef(SpecialReg).rsplit('_');
4101  std::string Reg = Fields.first.str();
4102  StringRef Flags = Fields.second;
4103 
4104  // If the target was M Class then need to validate the special register value
4105  // and retrieve the mask for use in the instruction node.
4106  if (Subtarget->isMClass()) {
4107  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
4108  if (SYSmValue == -1)
4109  return false;
4110 
4111  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
4112  N->getOperand(2), getAL(CurDAG, DL),
4113  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
4114  ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
4115  return true;
4116  }
4117 
4118  // We then check to see if a valid mask can be constructed for one of the
4119  // register string values permitted for the A and R class cores. These values
4120  // are apsr, spsr and cpsr; these are also valid on older cores.
4121  int Mask = getARClassRegisterMask(Reg, Flags);
4122  if (Mask != -1) {
4123  Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
4124  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
4125  N->getOperand(0) };
4126  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
4127  DL, MVT::Other, Ops));
4128  return true;
4129  }
4130 
4131  return false;
4132 }
4133 
4134 bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
4135  std::vector<SDValue> AsmNodeOperands;
4136  unsigned Flag, Kind;
4137  bool Changed = false;
4138  unsigned NumOps = N->getNumOperands();
4139 
4140  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
4141  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
4142  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
4143  // respectively. Since there is no constraint to explicitly specify a
4144  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
4145  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
4146  // them into a GPRPair.
4147 
4148  SDLoc dl(N);
4149  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
4150  : SDValue(nullptr,0);
4151 
4152  SmallVector<bool, 8> OpChanged;
4153  // Glue node will be appended late.
4154  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
4155  SDValue op = N->getOperand(i);
4156  AsmNodeOperands.push_back(op);
4157 
4159  continue;
4160 
4161  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
4162  Flag = C->getZExtValue();
4163  Kind = InlineAsm::getKind(Flag);
4164  }
4165  else
4166  continue;
4167 
4168  // Immediate operands to inline asm in the SelectionDAG are modeled with
4169  // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
4170  // the second is a constant with the value of the immediate. If we get here
4171  // and we have a Kind_Imm, skip the next operand, and continue.
4172  if (Kind == InlineAsm::Kind_Imm) {
4173  SDValue op = N->getOperand(++i);
4174  AsmNodeOperands.push_back(op);
4175  continue;
4176  }
4177 
4178  unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
4179  if (NumRegs)
4180  OpChanged.push_back(false);
4181 
4182  unsigned DefIdx = 0;
4183  bool IsTiedToChangedOp = false;
4184  // If it's a use that is tied with a previous def, it has no
4185  // reg class constraint.
4186  if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
4187  IsTiedToChangedOp = OpChanged[DefIdx];
4188 
4189  // Memory operands to inline asm in the SelectionDAG are modeled with two
4190  // operands: a constant of value InlineAsm::Kind_Mem followed by the input
4191  // operand. If we get here and we have a Kind_Mem, skip the next operand (so
4192  // it doesn't get misinterpreted), and continue. We do this here because
4193  // it's important to update the OpChanged array correctly before moving on.
4194  if (Kind == InlineAsm::Kind_Mem) {
4195  SDValue op = N->getOperand(++i);
4196  AsmNodeOperands.push_back(op);
4197  continue;
4198  }
4199 
4200  if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
4202  continue;
4203 
4204  unsigned RC;
4205  bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
4206  if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
4207  || NumRegs != 2)
4208  continue;
4209 
4210  assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
4211  SDValue V0 = N->getOperand(i+1);
4212  SDValue V1 = N->getOperand(i+2);
4213  unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
4214  unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
4215  SDValue PairedReg;
4216  MachineRegisterInfo &MRI = MF->getRegInfo();
4217 
4218  if (Kind == InlineAsm::Kind_RegDef ||
4220  // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
4221  // the original GPRs.
4222 
4223  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4224  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4225  SDValue Chain = SDValue(N,0);
4226 
4227  SDNode *GU = N->getGluedUser();
4228  SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4229  Chain.getValue(1));
4230 
4231  // Extract values from a GPRPair reg and copy to the original GPR reg.
4232  SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4233  RegCopy);
4234  SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4235  RegCopy);
4236  SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4237  RegCopy.getValue(1));
4238  SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4239 
4240  // Update the original glue user.
4241  std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4242  Ops.push_back(T1.getValue(1));
4243  CurDAG->UpdateNodeOperands(GU, Ops);
4244  }
4245  else {
4246  // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a
4247  // GPRPair and then pass the GPRPair to the inline asm.
4248  SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4249 
4250  // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
4251  SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4252  Chain.getValue(1));
4253  SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4254  T0.getValue(1));
4255  SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4256 
4257  // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
4258  // i32 VRs of inline asm with it.
4259  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4260  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4261  Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
4262 
4263  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
4264  Glue = Chain.getValue(1);
4265  }
4266 
4267  Changed = true;
4268 
4269  if(PairedReg.getNode()) {
4270  OpChanged[OpChanged.size() -1 ] = true;
4271  Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
4272  if (IsTiedToChangedOp)
4273  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
4274  else
4275  Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
4276  // Replace the current flag.
4277  AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
4278  Flag, dl, MVT::i32);
4279  // Add the new register node and skip the original two GPRs.
4280  AsmNodeOperands.push_back(PairedReg);
4281  // Skip the next two GPRs.
4282  i += 2;
4283  }
4284  }
4285 
4286  if (Glue.getNode())
4287  AsmNodeOperands.push_back(Glue);
4288  if (!Changed)
4289  return false;
4290 
4291  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
4292  CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
4293  New->setNodeId(-1);
4294  ReplaceNode(N, New.getNode());
4295  return true;
4296 }
4297 
4298 
4299 bool ARMDAGToDAGISel::
4300 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4301  std::vector<SDValue> &OutOps) {
4302  switch(ConstraintID) {
4303  default:
4304  llvm_unreachable("Unexpected asm memory constraint");
4306  // FIXME: It seems strange that 'i' is needed here since it's supposed to
4307  // be an immediate and not a memory constraint.
4319  // Require the address to be in a register. That is safe for all ARM
4320  // variants and it is hard to do anything much smarter without knowing
4321  // how the operand is used.
4322  OutOps.push_back(Op);
4323  return false;
4324  }
4325  return true;
4326 }
4327 
4328 /// createARMISelDag - This pass converts a legalized DAG into a
4329 /// ARM-specific DAG, ready for instruction scheduling.
4330 ///
4332  CodeGenOpt::Level OptLevel) {
4333  return new ARMDAGToDAGISel(TM, OptLevel);
4334 }
uint64_t CallInst * C
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
EVT getValueType() const
Return the ValueType of the referenced return value.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const SDValue & getOffset() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isThumb() const
Definition: ARMSubtarget.h:712
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
Various leaf nodes.
Definition: ISDOpcodes.h:60
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
#define LLVM_FALLTHROUGH
Definition: Compiler.h:86
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:269
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
SDVTList getVTList() const
static Optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
unsigned Reg
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:131
unsigned getAlignment() const
static bool isVSTfixed(unsigned Opc)
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(StringRef Separator) const
Split into two substrings around the last occurrence of a separator string.
Definition: StringRef.h:760
bool isSOImmTwoPartVal(unsigned V)
isSOImmTwoPartVal - Return true if the specified value can be obtained by or&#39;ing together two SOImmVa...
bool hasV6Ops() const
Definition: ARMSubtarget.h:536
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1069
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
bool isThumb1Only() const
Definition: ARMSubtarget.h:713
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
#define op(i)
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:189
static bool isThumb(const MCSubtargetInfo &STI)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:342
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1503
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
bool hasV8MBaselineOps() const
Definition: ARMSubtarget.h:547
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.h:1626
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isFpMLxInstruction(unsigned Opcode) const
isFpMLxInstruction - Return true if the specified opcode is a fp MLA / MLS instruction.
A description of a memory reference used in the backend.
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:300
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:390
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:478
Base class for LoadSDNode and StoreSDNode.
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:377
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:170
const MDNode * getMD() const
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE R Default(T Value)
Definition: StringSwitch.h:203
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:645
bool hasVFP2() const
Definition: ARMSubtarget.h:567
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
bool hasDSP() const
Definition: ARMSubtarget.h:628
bool useMovt(const MachineFunction &MF) const
static bool isVLDfixed(unsigned Opc)
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
bool hasV6T2Ops() const
Definition: ARMSubtarget.h:539
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:573
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:118
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:43
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
Container class for subtarget features.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:120
static int getBankedRegisterMask(StringRef RegString)
unsigned countPopulation() const
Count the number of bits set.
Definition: APInt.h:1652
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:429
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
bool isMClass() const
Definition: ARMSubtarget.h:716
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
const SDValue & getOperand(unsigned Num) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool hasFPARMv8() const
Definition: ARMSubtarget.h:570
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:336
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
static unsigned getKind(unsigned Flags)
Definition: InlineAsm.h:325
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
int getT2SOImmValSplatVal(unsigned V)
getT2SOImmValSplat - Return the 12-bit encoded representation if the specified value can be obtained ...
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
self_iterator getIterator()
Definition: ilist_node.h:82
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:774
Extended Value Type.
Definition: ValueTypes.h:34
size_t size() const
Definition: SmallVector.h:53
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Iterator for intrusive lists based on ilist_node.
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:222
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
This is an abstract virtual class for memory operations.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
Represents one node in the SelectionDAG.
#define NC
Definition: regutils.h:42
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:539
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:941
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
Class for arbitrary precision integers.
Definition: APInt.h:70
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:399
LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:70
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:394
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getNodeId() const
Return the unique node id.
static bool hasRegClassConstraint(unsigned Flag, unsigned &RC)
hasRegClassConstraint - Returns true if the flag contains a register class constraint.
Definition: InlineAsm.h:351
bool isThumb2() const
Definition: ARMSubtarget.h:714
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: ValueTypes.h:177
const SDValue & getValue() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:371
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:176
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:206
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:466
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:592
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
uint32_t Size
Definition: Profile.cpp:47
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOpt::Level OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
bool hasVMLxHazards() const
Definition: ARMSubtarget.h:611
const unsigned Kind
LLVM_NODISCARD std::string lower() const
Definition: StringRef.cpp:108
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getReg() const
bool isSwift() const
Definition: ARMSubtarget.h:558
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:203
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition: MathExtras.h:417
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
A single uniqued string.
Definition: Metadata.h:604
static int getMClassFlagsMask(StringRef Flags)
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1590
const SDValue & getOperand(unsigned i) const
uint64_t getZExtValue() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void setObjectAlignment(int ObjectIdx, unsigned Align)
setObjectAlignment - Change the alignment of the specified stack object.
bool hasThumb2() const
Definition: ARMSubtarget.h:715
bool isLikeA9() const
Definition: ARMSubtarget.h:560
#define T1
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:288
This file describes how to lower LLVM code to machine code.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:892
This class is used to represent ISD::LOAD nodes.