LLVM  7.0.0svn
ARMISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- ARMISelDAGToDAG.cpp - A dag to dag inst selector for ARM ----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines an instruction selector for the ARM target.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "ARM.h"
15 #include "ARMBaseInstrInfo.h"
16 #include "ARMTargetMachine.h"
18 #include "Utils/ARMBaseInfo.h"
19 #include "llvm/ADT/StringSwitch.h"
27 #include "llvm/IR/CallingConv.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Function.h"
31 #include "llvm/IR/Intrinsics.h"
32 #include "llvm/IR/LLVMContext.h"
34 #include "llvm/Support/Debug.h"
37 
38 using namespace llvm;
39 
40 #define DEBUG_TYPE "arm-isel"
41 
42 static cl::opt<bool>
43 DisableShifterOp("disable-shifter-op", cl::Hidden,
44  cl::desc("Disable isel of shifter-op"),
45  cl::init(false));
46 
47 //===--------------------------------------------------------------------===//
48 /// ARMDAGToDAGISel - ARM specific code to select ARM machine
49 /// instructions for SelectionDAG operations.
50 ///
51 namespace {
52 
53 class ARMDAGToDAGISel : public SelectionDAGISel {
54  /// Subtarget - Keep a pointer to the ARMSubtarget around so that we can
55  /// make the right decision when generating code for different targets.
56  const ARMSubtarget *Subtarget;
57 
58 public:
59  explicit ARMDAGToDAGISel(ARMBaseTargetMachine &tm, CodeGenOpt::Level OptLevel)
60  : SelectionDAGISel(tm, OptLevel) {}
61 
62  bool runOnMachineFunction(MachineFunction &MF) override {
63  // Reset the subtarget each time through.
64  Subtarget = &MF.getSubtarget<ARMSubtarget>();
66  return true;
67  }
68 
69  StringRef getPassName() const override { return "ARM Instruction Selection"; }
70 
71  void PreprocessISelDAG() override;
72 
73  /// getI32Imm - Return a target constant of type i32 with the specified
74  /// value.
75  inline SDValue getI32Imm(unsigned Imm, const SDLoc &dl) {
76  return CurDAG->getTargetConstant(Imm, dl, MVT::i32);
77  }
78 
79  void Select(SDNode *N) override;
80 
81  bool hasNoVMLxHazardUse(SDNode *N) const;
82  bool isShifterOpProfitable(const SDValue &Shift,
83  ARM_AM::ShiftOpc ShOpcVal, unsigned ShAmt);
84  bool SelectRegShifterOperand(SDValue N, SDValue &A,
85  SDValue &B, SDValue &C,
86  bool CheckProfitability = true);
87  bool SelectImmShifterOperand(SDValue N, SDValue &A,
88  SDValue &B, bool CheckProfitability = true);
89  bool SelectShiftRegShifterOperand(SDValue N, SDValue &A,
90  SDValue &B, SDValue &C) {
91  // Don't apply the profitability check
92  return SelectRegShifterOperand(N, A, B, C, false);
93  }
94  bool SelectShiftImmShifterOperand(SDValue N, SDValue &A,
95  SDValue &B) {
96  // Don't apply the profitability check
97  return SelectImmShifterOperand(N, A, B, false);
98  }
99 
100  bool SelectAddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
101  bool SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset, SDValue &Opc);
102 
103  bool SelectCMOVPred(SDValue N, SDValue &Pred, SDValue &Reg) {
104  const ConstantSDNode *CN = cast<ConstantSDNode>(N);
105  Pred = CurDAG->getTargetConstant(CN->getZExtValue(), SDLoc(N), MVT::i32);
106  Reg = CurDAG->getRegister(ARM::CPSR, MVT::i32);
107  return true;
108  }
109 
110  bool SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
111  SDValue &Offset, SDValue &Opc);
112  bool SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
113  SDValue &Offset, SDValue &Opc);
114  bool SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
115  SDValue &Offset, SDValue &Opc);
116  bool SelectAddrOffsetNone(SDValue N, SDValue &Base);
117  bool SelectAddrMode3(SDValue N, SDValue &Base,
118  SDValue &Offset, SDValue &Opc);
119  bool SelectAddrMode3Offset(SDNode *Op, SDValue N,
120  SDValue &Offset, SDValue &Opc);
121  bool IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
122  int Lwb, int Upb, bool FP16);
123  bool SelectAddrMode5(SDValue N, SDValue &Base, SDValue &Offset);
124  bool SelectAddrMode5FP16(SDValue N, SDValue &Base, SDValue &Offset);
125  bool SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,SDValue &Align);
126  bool SelectAddrMode6Offset(SDNode *Op, SDValue N, SDValue &Offset);
127 
128  bool SelectAddrModePC(SDValue N, SDValue &Offset, SDValue &Label);
129 
130  // Thumb Addressing Modes:
131  bool SelectThumbAddrModeRR(SDValue N, SDValue &Base, SDValue &Offset);
132  bool SelectThumbAddrModeImm5S(SDValue N, unsigned Scale, SDValue &Base,
133  SDValue &OffImm);
134  bool SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
135  SDValue &OffImm);
136  bool SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
137  SDValue &OffImm);
138  bool SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
139  SDValue &OffImm);
140  bool SelectThumbAddrModeSP(SDValue N, SDValue &Base, SDValue &OffImm);
141 
142  // Thumb 2 Addressing Modes:
143  bool SelectT2AddrModeImm12(SDValue N, SDValue &Base, SDValue &OffImm);
144  bool SelectT2AddrModeImm8(SDValue N, SDValue &Base,
145  SDValue &OffImm);
146  bool SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
147  SDValue &OffImm);
148  bool SelectT2AddrModeSoReg(SDValue N, SDValue &Base,
149  SDValue &OffReg, SDValue &ShImm);
150  bool SelectT2AddrModeExclusive(SDValue N, SDValue &Base, SDValue &OffImm);
151 
152  inline bool is_so_imm(unsigned Imm) const {
153  return ARM_AM::getSOImmVal(Imm) != -1;
154  }
155 
156  inline bool is_so_imm_not(unsigned Imm) const {
157  return ARM_AM::getSOImmVal(~Imm) != -1;
158  }
159 
160  inline bool is_t2_so_imm(unsigned Imm) const {
161  return ARM_AM::getT2SOImmVal(Imm) != -1;
162  }
163 
164  inline bool is_t2_so_imm_not(unsigned Imm) const {
165  return ARM_AM::getT2SOImmVal(~Imm) != -1;
166  }
167 
168  // Include the pieces autogenerated from the target description.
169 #include "ARMGenDAGISel.inc"
170 
171 private:
172  void transferMemOperands(SDNode *Src, SDNode *Dst);
173 
174  /// Indexed (pre/post inc/dec) load matching code for ARM.
175  bool tryARMIndexedLoad(SDNode *N);
176  bool tryT1IndexedLoad(SDNode *N);
177  bool tryT2IndexedLoad(SDNode *N);
178 
179  /// SelectVLD - Select NEON load intrinsics. NumVecs should be
180  /// 1, 2, 3 or 4. The opcode arrays specify the instructions used for
181  /// loads of D registers and even subregs and odd subregs of Q registers.
182  /// For NumVecs <= 2, QOpcodes1 is not used.
183  void SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
184  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
185  const uint16_t *QOpcodes1);
186 
187  /// SelectVST - Select NEON store intrinsics. NumVecs should
188  /// be 1, 2, 3 or 4. The opcode arrays specify the instructions used for
189  /// stores of D registers and even subregs and odd subregs of Q registers.
190  /// For NumVecs <= 2, QOpcodes1 is not used.
191  void SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
192  const uint16_t *DOpcodes, const uint16_t *QOpcodes0,
193  const uint16_t *QOpcodes1);
194 
195  /// SelectVLDSTLane - Select NEON load/store lane intrinsics. NumVecs should
196  /// be 2, 3 or 4. The opcode arrays specify the instructions used for
197  /// load/store of D registers and Q registers.
198  void SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
199  unsigned NumVecs, const uint16_t *DOpcodes,
200  const uint16_t *QOpcodes);
201 
202  /// SelectVLDDup - Select NEON load-duplicate intrinsics. NumVecs
203  /// should be 1, 2, 3 or 4. The opcode array specifies the instructions used
204  /// for loading D registers. (Q registers are not supported.)
205  void SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
206  const uint16_t *DOpcodes,
207  const uint16_t *QOpcodes = nullptr);
208 
209  /// Try to select SBFX/UBFX instructions for ARM.
210  bool tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned);
211 
212  // Select special operations if node forms integer ABS pattern
213  bool tryABSOp(SDNode *N);
214 
215  bool tryReadRegister(SDNode *N);
216  bool tryWriteRegister(SDNode *N);
217 
218  bool tryInlineAsm(SDNode *N);
219 
220  void SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI);
221 
222  void SelectCMP_SWAP(SDNode *N);
223 
224  /// SelectInlineAsmMemoryOperand - Implement addressing mode selection for
225  /// inline asm expressions.
226  bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
227  std::vector<SDValue> &OutOps) override;
228 
229  // Form pairs of consecutive R, S, D, or Q registers.
231  SDNode *createSRegPairNode(EVT VT, SDValue V0, SDValue V1);
232  SDNode *createDRegPairNode(EVT VT, SDValue V0, SDValue V1);
233  SDNode *createQRegPairNode(EVT VT, SDValue V0, SDValue V1);
234 
235  // Form sequences of 4 consecutive S, D, or Q registers.
236  SDNode *createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
237  SDNode *createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
238  SDNode *createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1, SDValue V2, SDValue V3);
239 
240  // Get the alignment operand for a NEON VLD or VST instruction.
241  SDValue GetVLDSTAlign(SDValue Align, const SDLoc &dl, unsigned NumVecs,
242  bool is64BitVector);
243 
244  /// Returns the number of instructions required to materialize the given
245  /// constant in a register, or 3 if a literal pool load is needed.
246  unsigned ConstantMaterializationCost(unsigned Val) const;
247 
248  /// Checks if N is a multiplication by a constant where we can extract out a
249  /// power of two from the constant so that it can be used in a shift, but only
250  /// if it simplifies the materialization of the constant. Returns true if it
251  /// is, and assigns to PowerOfTwo the power of two that should be extracted
252  /// out and to NewMulConst the new constant to be multiplied by.
253  bool canExtractShiftFromMul(const SDValue &N, unsigned MaxShift,
254  unsigned &PowerOfTwo, SDValue &NewMulConst) const;
255 
256  /// Replace N with M in CurDAG, in a way that also ensures that M gets
257  /// selected when N would have been selected.
258  void replaceDAGValue(const SDValue &N, SDValue M);
259 };
260 }
261 
262 /// isInt32Immediate - This method tests to see if the node is a 32-bit constant
263 /// operand. If so Imm will receive the 32-bit value.
264 static bool isInt32Immediate(SDNode *N, unsigned &Imm) {
265  if (N->getOpcode() == ISD::Constant && N->getValueType(0) == MVT::i32) {
266  Imm = cast<ConstantSDNode>(N)->getZExtValue();
267  return true;
268  }
269  return false;
270 }
271 
272 // isInt32Immediate - This method tests to see if a constant operand.
273 // If so Imm will receive the 32 bit value.
274 static bool isInt32Immediate(SDValue N, unsigned &Imm) {
275  return isInt32Immediate(N.getNode(), Imm);
276 }
277 
278 // isOpcWithIntImmediate - This method tests to see if the node is a specific
279 // opcode and that it has a immediate integer right operand.
280 // If so Imm will receive the 32 bit value.
281 static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
282  return N->getOpcode() == Opc &&
283  isInt32Immediate(N->getOperand(1).getNode(), Imm);
284 }
285 
286 /// \brief Check whether a particular node is a constant value representable as
287 /// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
288 ///
289 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
290 static bool isScaledConstantInRange(SDValue Node, int Scale,
291  int RangeMin, int RangeMax,
292  int &ScaledConstant) {
293  assert(Scale > 0 && "Invalid scale!");
294 
295  // Check that this is a constant.
296  const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Node);
297  if (!C)
298  return false;
299 
300  ScaledConstant = (int) C->getZExtValue();
301  if ((ScaledConstant % Scale) != 0)
302  return false;
303 
304  ScaledConstant /= Scale;
305  return ScaledConstant >= RangeMin && ScaledConstant < RangeMax;
306 }
307 
308 void ARMDAGToDAGISel::PreprocessISelDAG() {
309  if (!Subtarget->hasV6T2Ops())
310  return;
311 
312  bool isThumb2 = Subtarget->isThumb();
313  for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
314  E = CurDAG->allnodes_end(); I != E; ) {
315  SDNode *N = &*I++; // Preincrement iterator to avoid invalidation issues.
316 
317  if (N->getOpcode() != ISD::ADD)
318  continue;
319 
320  // Look for (add X1, (and (srl X2, c1), c2)) where c2 is constant with
321  // leading zeros, followed by consecutive set bits, followed by 1 or 2
322  // trailing zeros, e.g. 1020.
323  // Transform the expression to
324  // (add X1, (shl (and (srl X2, c1), (c2>>tz)), tz)) where tz is the number
325  // of trailing zeros of c2. The left shift would be folded as an shifter
326  // operand of 'add' and the 'and' and 'srl' would become a bits extraction
327  // node (UBFX).
328 
329  SDValue N0 = N->getOperand(0);
330  SDValue N1 = N->getOperand(1);
331  unsigned And_imm = 0;
332  if (!isOpcWithIntImmediate(N1.getNode(), ISD::AND, And_imm)) {
333  if (isOpcWithIntImmediate(N0.getNode(), ISD::AND, And_imm))
334  std::swap(N0, N1);
335  }
336  if (!And_imm)
337  continue;
338 
339  // Check if the AND mask is an immediate of the form: 000.....1111111100
340  unsigned TZ = countTrailingZeros(And_imm);
341  if (TZ != 1 && TZ != 2)
342  // Be conservative here. Shifter operands aren't always free. e.g. On
343  // Swift, left shifter operand of 1 / 2 for free but others are not.
344  // e.g.
345  // ubfx r3, r1, #16, #8
346  // ldr.w r3, [r0, r3, lsl #2]
347  // vs.
348  // mov.w r9, #1020
349  // and.w r2, r9, r1, lsr #14
350  // ldr r2, [r0, r2]
351  continue;
352  And_imm >>= TZ;
353  if (And_imm & (And_imm + 1))
354  continue;
355 
356  // Look for (and (srl X, c1), c2).
357  SDValue Srl = N1.getOperand(0);
358  unsigned Srl_imm = 0;
359  if (!isOpcWithIntImmediate(Srl.getNode(), ISD::SRL, Srl_imm) ||
360  (Srl_imm <= 2))
361  continue;
362 
363  // Make sure first operand is not a shifter operand which would prevent
364  // folding of the left shift.
365  SDValue CPTmp0;
366  SDValue CPTmp1;
367  SDValue CPTmp2;
368  if (isThumb2) {
369  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1))
370  continue;
371  } else {
372  if (SelectImmShifterOperand(N0, CPTmp0, CPTmp1) ||
373  SelectRegShifterOperand(N0, CPTmp0, CPTmp1, CPTmp2))
374  continue;
375  }
376 
377  // Now make the transformation.
378  Srl = CurDAG->getNode(ISD::SRL, SDLoc(Srl), MVT::i32,
379  Srl.getOperand(0),
380  CurDAG->getConstant(Srl_imm + TZ, SDLoc(Srl),
381  MVT::i32));
382  N1 = CurDAG->getNode(ISD::AND, SDLoc(N1), MVT::i32,
383  Srl,
384  CurDAG->getConstant(And_imm, SDLoc(Srl), MVT::i32));
385  N1 = CurDAG->getNode(ISD::SHL, SDLoc(N1), MVT::i32,
386  N1, CurDAG->getConstant(TZ, SDLoc(Srl), MVT::i32));
387  CurDAG->UpdateNodeOperands(N, N0, N1);
388  }
389 }
390 
391 /// hasNoVMLxHazardUse - Return true if it's desirable to select a FP MLA / MLS
392 /// node. VFP / NEON fp VMLA / VMLS instructions have special RAW hazards (at
393 /// least on current ARM implementations) which should be avoidded.
394 bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
395  if (OptLevel == CodeGenOpt::None)
396  return true;
397 
398  if (!Subtarget->hasVMLxHazards())
399  return true;
400 
401  if (!N->hasOneUse())
402  return false;
403 
404  SDNode *Use = *N->use_begin();
405  if (Use->getOpcode() == ISD::CopyToReg)
406  return true;
407  if (Use->isMachineOpcode()) {
408  const ARMBaseInstrInfo *TII = static_cast<const ARMBaseInstrInfo *>(
409  CurDAG->getSubtarget().getInstrInfo());
410 
411  const MCInstrDesc &MCID = TII->get(Use->getMachineOpcode());
412  if (MCID.mayStore())
413  return true;
414  unsigned Opcode = MCID.getOpcode();
415  if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD)
416  return true;
417  // vmlx feeding into another vmlx. We actually want to unfold
418  // the use later in the MLxExpansion pass. e.g.
419  // vmla
420  // vmla (stall 8 cycles)
421  //
422  // vmul (5 cycles)
423  // vadd (5 cycles)
424  // vmla
425  // This adds up to about 18 - 19 cycles.
426  //
427  // vmla
428  // vmul (stall 4 cycles)
429  // vadd adds up to about 14 cycles.
430  return TII->isFpMLxInstruction(Opcode);
431  }
432 
433  return false;
434 }
435 
436 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
437  ARM_AM::ShiftOpc ShOpcVal,
438  unsigned ShAmt) {
439  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
440  return true;
441  if (Shift.hasOneUse())
442  return true;
443  // R << 2 is free.
444  return ShOpcVal == ARM_AM::lsl &&
445  (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
446 }
447 
448 unsigned ARMDAGToDAGISel::ConstantMaterializationCost(unsigned Val) const {
449  if (Subtarget->isThumb()) {
450  if (Val <= 255) return 1; // MOV
451  if (Subtarget->hasV6T2Ops() &&
452  (Val <= 0xffff || ARM_AM::getT2SOImmValSplatVal(Val) != -1))
453  return 1; // MOVW
454  if (Val <= 510) return 2; // MOV + ADDi8
455  if (~Val <= 255) return 2; // MOV + MVN
456  if (ARM_AM::isThumbImmShiftedVal(Val)) return 2; // MOV + LSL
457  } else {
458  if (ARM_AM::getSOImmVal(Val) != -1) return 1; // MOV
459  if (ARM_AM::getSOImmVal(~Val) != -1) return 1; // MVN
460  if (Subtarget->hasV6T2Ops() && Val <= 0xffff) return 1; // MOVW
461  if (ARM_AM::isSOImmTwoPartVal(Val)) return 2; // two instrs
462  }
463  if (Subtarget->useMovt(*MF)) return 2; // MOVW + MOVT
464  return 3; // Literal pool load
465 }
466 
467 bool ARMDAGToDAGISel::canExtractShiftFromMul(const SDValue &N,
468  unsigned MaxShift,
469  unsigned &PowerOfTwo,
470  SDValue &NewMulConst) const {
471  assert(N.getOpcode() == ISD::MUL);
472  assert(MaxShift > 0);
473 
474  // If the multiply is used in more than one place then changing the constant
475  // will make other uses incorrect, so don't.
476  if (!N.hasOneUse()) return false;
477  // Check if the multiply is by a constant
479  if (!MulConst) return false;
480  // If the constant is used in more than one place then modifying it will mean
481  // we need to materialize two constants instead of one, which is a bad idea.
482  if (!MulConst->hasOneUse()) return false;
483  unsigned MulConstVal = MulConst->getZExtValue();
484  if (MulConstVal == 0) return false;
485 
486  // Find the largest power of 2 that MulConstVal is a multiple of
487  PowerOfTwo = MaxShift;
488  while ((MulConstVal % (1 << PowerOfTwo)) != 0) {
489  --PowerOfTwo;
490  if (PowerOfTwo == 0) return false;
491  }
492 
493  // Only optimise if the new cost is better
494  unsigned NewMulConstVal = MulConstVal / (1 << PowerOfTwo);
495  NewMulConst = CurDAG->getConstant(NewMulConstVal, SDLoc(N), MVT::i32);
496  unsigned OldCost = ConstantMaterializationCost(MulConstVal);
497  unsigned NewCost = ConstantMaterializationCost(NewMulConstVal);
498  return NewCost < OldCost;
499 }
500 
501 void ARMDAGToDAGISel::replaceDAGValue(const SDValue &N, SDValue M) {
502  CurDAG->RepositionNode(N.getNode()->getIterator(), M.getNode());
503  ReplaceUses(N, M);
504 }
505 
506 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
507  SDValue &BaseReg,
508  SDValue &Opc,
509  bool CheckProfitability) {
510  if (DisableShifterOp)
511  return false;
512 
513  // If N is a multiply-by-constant and it's profitable to extract a shift and
514  // use it in a shifted operand do so.
515  if (N.getOpcode() == ISD::MUL) {
516  unsigned PowerOfTwo = 0;
517  SDValue NewMulConst;
518  if (canExtractShiftFromMul(N, 31, PowerOfTwo, NewMulConst)) {
519  HandleSDNode Handle(N);
520  SDLoc Loc(N);
521  replaceDAGValue(N.getOperand(1), NewMulConst);
522  BaseReg = Handle.getValue();
523  Opc = CurDAG->getTargetConstant(
524  ARM_AM::getSORegOpc(ARM_AM::lsl, PowerOfTwo), Loc, MVT::i32);
525  return true;
526  }
527  }
528 
530 
531  // Don't match base register only case. That is matched to a separate
532  // lower complexity pattern with explicit register operand.
533  if (ShOpcVal == ARM_AM::no_shift) return false;
534 
535  BaseReg = N.getOperand(0);
536  unsigned ShImmVal = 0;
538  if (!RHS) return false;
539  ShImmVal = RHS->getZExtValue() & 31;
540  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
541  SDLoc(N), MVT::i32);
542  return true;
543 }
544 
545 bool ARMDAGToDAGISel::SelectRegShifterOperand(SDValue N,
546  SDValue &BaseReg,
547  SDValue &ShReg,
548  SDValue &Opc,
549  bool CheckProfitability) {
550  if (DisableShifterOp)
551  return false;
552 
554 
555  // Don't match base register only case. That is matched to a separate
556  // lower complexity pattern with explicit register operand.
557  if (ShOpcVal == ARM_AM::no_shift) return false;
558 
559  BaseReg = N.getOperand(0);
560  unsigned ShImmVal = 0;
562  if (RHS) return false;
563 
564  ShReg = N.getOperand(1);
565  if (CheckProfitability && !isShifterOpProfitable(N, ShOpcVal, ShImmVal))
566  return false;
567  Opc = CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, ShImmVal),
568  SDLoc(N), MVT::i32);
569  return true;
570 }
571 
572 
573 bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
574  SDValue &Base,
575  SDValue &OffImm) {
576  // Match simple R + imm12 operands.
577 
578  // Base only.
579  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
580  !CurDAG->isBaseWithConstantOffset(N)) {
581  if (N.getOpcode() == ISD::FrameIndex) {
582  // Match frame index.
583  int FI = cast<FrameIndexSDNode>(N)->getIndex();
584  Base = CurDAG->getTargetFrameIndex(
585  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
586  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
587  return true;
588  }
589 
590  if (N.getOpcode() == ARMISD::Wrapper &&
594  Base = N.getOperand(0);
595  } else
596  Base = N;
597  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
598  return true;
599  }
600 
601  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
602  int RHSC = (int)RHS->getSExtValue();
603  if (N.getOpcode() == ISD::SUB)
604  RHSC = -RHSC;
605 
606  if (RHSC > -0x1000 && RHSC < 0x1000) { // 12 bits
607  Base = N.getOperand(0);
608  if (Base.getOpcode() == ISD::FrameIndex) {
609  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
610  Base = CurDAG->getTargetFrameIndex(
611  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
612  }
613  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
614  return true;
615  }
616  }
617 
618  // Base only.
619  Base = N;
620  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
621  return true;
622 }
623 
624 
625 
626 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
627  SDValue &Opc) {
628  if (N.getOpcode() == ISD::MUL &&
629  ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
630  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
631  // X * [3,5,9] -> X + X * [2,4,8] etc.
632  int RHSC = (int)RHS->getZExtValue();
633  if (RHSC & 1) {
634  RHSC = RHSC & ~1;
635  ARM_AM::AddrOpc AddSub = ARM_AM::add;
636  if (RHSC < 0) {
637  AddSub = ARM_AM::sub;
638  RHSC = - RHSC;
639  }
640  if (isPowerOf2_32(RHSC)) {
641  unsigned ShAmt = Log2_32(RHSC);
642  Base = Offset = N.getOperand(0);
643  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt,
644  ARM_AM::lsl),
645  SDLoc(N), MVT::i32);
646  return true;
647  }
648  }
649  }
650  }
651 
652  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
653  // ISD::OR that is equivalent to an ISD::ADD.
654  !CurDAG->isBaseWithConstantOffset(N))
655  return false;
656 
657  // Leave simple R +/- imm12 operands for LDRi12
658  if (N.getOpcode() == ISD::ADD || N.getOpcode() == ISD::OR) {
659  int RHSC;
660  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
661  -0x1000+1, 0x1000, RHSC)) // 12 bits.
662  return false;
663  }
664 
665  // Otherwise this is R +/- [possibly shifted] R.
667  ARM_AM::ShiftOpc ShOpcVal =
669  unsigned ShAmt = 0;
670 
671  Base = N.getOperand(0);
672  Offset = N.getOperand(1);
673 
674  if (ShOpcVal != ARM_AM::no_shift) {
675  // Check to see if the RHS of the shift is a constant, if not, we can't fold
676  // it.
677  if (ConstantSDNode *Sh =
678  dyn_cast<ConstantSDNode>(N.getOperand(1).getOperand(1))) {
679  ShAmt = Sh->getZExtValue();
680  if (isShifterOpProfitable(Offset, ShOpcVal, ShAmt))
681  Offset = N.getOperand(1).getOperand(0);
682  else {
683  ShAmt = 0;
684  ShOpcVal = ARM_AM::no_shift;
685  }
686  } else {
687  ShOpcVal = ARM_AM::no_shift;
688  }
689  }
690 
691  // Try matching (R shl C) + (R).
692  if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
693  !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
694  N.getOperand(0).hasOneUse())) {
696  if (ShOpcVal != ARM_AM::no_shift) {
697  // Check to see if the RHS of the shift is a constant, if not, we can't
698  // fold it.
699  if (ConstantSDNode *Sh =
700  dyn_cast<ConstantSDNode>(N.getOperand(0).getOperand(1))) {
701  ShAmt = Sh->getZExtValue();
702  if (isShifterOpProfitable(N.getOperand(0), ShOpcVal, ShAmt)) {
703  Offset = N.getOperand(0).getOperand(0);
704  Base = N.getOperand(1);
705  } else {
706  ShAmt = 0;
707  ShOpcVal = ARM_AM::no_shift;
708  }
709  } else {
710  ShOpcVal = ARM_AM::no_shift;
711  }
712  }
713  }
714 
715  // If Offset is a multiply-by-constant and it's profitable to extract a shift
716  // and use it in a shifted operand do so.
717  if (Offset.getOpcode() == ISD::MUL && N.hasOneUse()) {
718  unsigned PowerOfTwo = 0;
719  SDValue NewMulConst;
720  if (canExtractShiftFromMul(Offset, 31, PowerOfTwo, NewMulConst)) {
721  HandleSDNode Handle(Offset);
722  replaceDAGValue(Offset.getOperand(1), NewMulConst);
723  Offset = Handle.getValue();
724  ShAmt = PowerOfTwo;
725  ShOpcVal = ARM_AM::lsl;
726  }
727  }
728 
729  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
730  SDLoc(N), MVT::i32);
731  return true;
732 }
733 
734 bool ARMDAGToDAGISel::SelectAddrMode2OffsetReg(SDNode *Op, SDValue N,
735  SDValue &Offset, SDValue &Opc) {
736  unsigned Opcode = Op->getOpcode();
737  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
738  ? cast<LoadSDNode>(Op)->getAddressingMode()
739  : cast<StoreSDNode>(Op)->getAddressingMode();
740  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
742  int Val;
743  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val))
744  return false;
745 
746  Offset = N;
748  unsigned ShAmt = 0;
749  if (ShOpcVal != ARM_AM::no_shift) {
750  // Check to see if the RHS of the shift is a constant, if not, we can't fold
751  // it.
752  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
753  ShAmt = Sh->getZExtValue();
754  if (isShifterOpProfitable(N, ShOpcVal, ShAmt))
755  Offset = N.getOperand(0);
756  else {
757  ShAmt = 0;
758  ShOpcVal = ARM_AM::no_shift;
759  }
760  } else {
761  ShOpcVal = ARM_AM::no_shift;
762  }
763  }
764 
765  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, ShAmt, ShOpcVal),
766  SDLoc(N), MVT::i32);
767  return true;
768 }
769 
770 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImmPre(SDNode *Op, SDValue N,
771  SDValue &Offset, SDValue &Opc) {
772  unsigned Opcode = Op->getOpcode();
773  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
774  ? cast<LoadSDNode>(Op)->getAddressingMode()
775  : cast<StoreSDNode>(Op)->getAddressingMode();
776  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
778  int Val;
779  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
780  if (AddSub == ARM_AM::sub) Val *= -1;
781  Offset = CurDAG->getRegister(0, MVT::i32);
782  Opc = CurDAG->getTargetConstant(Val, SDLoc(Op), MVT::i32);
783  return true;
784  }
785 
786  return false;
787 }
788 
789 
790 bool ARMDAGToDAGISel::SelectAddrMode2OffsetImm(SDNode *Op, SDValue N,
791  SDValue &Offset, SDValue &Opc) {
792  unsigned Opcode = Op->getOpcode();
793  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
794  ? cast<LoadSDNode>(Op)->getAddressingMode()
795  : cast<StoreSDNode>(Op)->getAddressingMode();
796  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
798  int Val;
799  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x1000, Val)) { // 12 bits.
800  Offset = CurDAG->getRegister(0, MVT::i32);
801  Opc = CurDAG->getTargetConstant(ARM_AM::getAM2Opc(AddSub, Val,
803  SDLoc(Op), MVT::i32);
804  return true;
805  }
806 
807  return false;
808 }
809 
810 bool ARMDAGToDAGISel::SelectAddrOffsetNone(SDValue N, SDValue &Base) {
811  Base = N;
812  return true;
813 }
814 
815 bool ARMDAGToDAGISel::SelectAddrMode3(SDValue N,
816  SDValue &Base, SDValue &Offset,
817  SDValue &Opc) {
818  if (N.getOpcode() == ISD::SUB) {
819  // X - C is canonicalize to X + -C, no need to handle it here.
820  Base = N.getOperand(0);
821  Offset = N.getOperand(1);
822  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::sub, 0), SDLoc(N),
823  MVT::i32);
824  return true;
825  }
826 
827  if (!CurDAG->isBaseWithConstantOffset(N)) {
828  Base = N;
829  if (N.getOpcode() == ISD::FrameIndex) {
830  int FI = cast<FrameIndexSDNode>(N)->getIndex();
831  Base = CurDAG->getTargetFrameIndex(
832  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
833  }
834  Offset = CurDAG->getRegister(0, MVT::i32);
835  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
836  MVT::i32);
837  return true;
838  }
839 
840  // If the RHS is +/- imm8, fold into addr mode.
841  int RHSC;
842  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/1,
843  -256 + 1, 256, RHSC)) { // 8 bits.
844  Base = N.getOperand(0);
845  if (Base.getOpcode() == ISD::FrameIndex) {
846  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
847  Base = CurDAG->getTargetFrameIndex(
848  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
849  }
850  Offset = CurDAG->getRegister(0, MVT::i32);
851 
852  ARM_AM::AddrOpc AddSub = ARM_AM::add;
853  if (RHSC < 0) {
854  AddSub = ARM_AM::sub;
855  RHSC = -RHSC;
856  }
857  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, RHSC), SDLoc(N),
858  MVT::i32);
859  return true;
860  }
861 
862  Base = N.getOperand(0);
863  Offset = N.getOperand(1);
864  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(ARM_AM::add, 0), SDLoc(N),
865  MVT::i32);
866  return true;
867 }
868 
869 bool ARMDAGToDAGISel::SelectAddrMode3Offset(SDNode *Op, SDValue N,
870  SDValue &Offset, SDValue &Opc) {
871  unsigned Opcode = Op->getOpcode();
872  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
873  ? cast<LoadSDNode>(Op)->getAddressingMode()
874  : cast<StoreSDNode>(Op)->getAddressingMode();
875  ARM_AM::AddrOpc AddSub = (AM == ISD::PRE_INC || AM == ISD::POST_INC)
877  int Val;
878  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 256, Val)) { // 12 bits.
879  Offset = CurDAG->getRegister(0, MVT::i32);
880  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, Val), SDLoc(Op),
881  MVT::i32);
882  return true;
883  }
884 
885  Offset = N;
886  Opc = CurDAG->getTargetConstant(ARM_AM::getAM3Opc(AddSub, 0), SDLoc(Op),
887  MVT::i32);
888  return true;
889 }
890 
891 bool ARMDAGToDAGISel::IsAddressingMode5(SDValue N, SDValue &Base, SDValue &Offset,
892  int Lwb, int Upb, bool FP16) {
893  if (!CurDAG->isBaseWithConstantOffset(N)) {
894  Base = N;
895  if (N.getOpcode() == ISD::FrameIndex) {
896  int FI = cast<FrameIndexSDNode>(N)->getIndex();
897  Base = CurDAG->getTargetFrameIndex(
898  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
899  } else if (N.getOpcode() == ARMISD::Wrapper &&
903  Base = N.getOperand(0);
904  }
905  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
906  SDLoc(N), MVT::i32);
907  return true;
908  }
909 
910  // If the RHS is +/- imm8, fold into addr mode.
911  int RHSC;
912  const int Scale = FP16 ? 2 : 4;
913 
914  if (isScaledConstantInRange(N.getOperand(1), Scale, Lwb, Upb, RHSC)) {
915  Base = N.getOperand(0);
916  if (Base.getOpcode() == ISD::FrameIndex) {
917  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
918  Base = CurDAG->getTargetFrameIndex(
919  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
920  }
921 
922  ARM_AM::AddrOpc AddSub = ARM_AM::add;
923  if (RHSC < 0) {
924  AddSub = ARM_AM::sub;
925  RHSC = -RHSC;
926  }
927 
928  if (FP16)
929  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(AddSub, RHSC),
930  SDLoc(N), MVT::i32);
931  else
932  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(AddSub, RHSC),
933  SDLoc(N), MVT::i32);
934 
935  return true;
936  }
937 
938  Base = N;
939 
940  if (FP16)
941  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5FP16Opc(ARM_AM::add, 0),
942  SDLoc(N), MVT::i32);
943  else
944  Offset = CurDAG->getTargetConstant(ARM_AM::getAM5Opc(ARM_AM::add, 0),
945  SDLoc(N), MVT::i32);
946 
947  return true;
948 }
949 
950 bool ARMDAGToDAGISel::SelectAddrMode5(SDValue N,
951  SDValue &Base, SDValue &Offset) {
952  int Lwb = -256 + 1;
953  int Upb = 256;
954  return IsAddressingMode5(N, Base, Offset, Lwb, Upb, /*FP16=*/ false);
955 }
956 
957 bool ARMDAGToDAGISel::SelectAddrMode5FP16(SDValue N,
958  SDValue &Base, SDValue &Offset) {
959  int Lwb = -512 + 1;
960  int Upb = 512;
961  return IsAddressingMode5(N, Base, Offset, Lwb, Upb, /*FP16=*/ true);
962 }
963 
964 bool ARMDAGToDAGISel::SelectAddrMode6(SDNode *Parent, SDValue N, SDValue &Addr,
965  SDValue &Align) {
966  Addr = N;
967 
968  unsigned Alignment = 0;
969 
970  MemSDNode *MemN = cast<MemSDNode>(Parent);
971 
972  if (isa<LSBaseSDNode>(MemN) ||
973  ((MemN->getOpcode() == ARMISD::VST1_UPD ||
974  MemN->getOpcode() == ARMISD::VLD1_UPD) &&
975  MemN->getConstantOperandVal(MemN->getNumOperands() - 1) == 1)) {
976  // This case occurs only for VLD1-lane/dup and VST1-lane instructions.
977  // The maximum alignment is equal to the memory size being referenced.
978  unsigned MMOAlign = MemN->getAlignment();
979  unsigned MemSize = MemN->getMemoryVT().getSizeInBits() / 8;
980  if (MMOAlign >= MemSize && MemSize > 1)
981  Alignment = MemSize;
982  } else {
983  // All other uses of addrmode6 are for intrinsics. For now just record
984  // the raw alignment value; it will be refined later based on the legal
985  // alignment operands for the intrinsic.
986  Alignment = MemN->getAlignment();
987  }
988 
989  Align = CurDAG->getTargetConstant(Alignment, SDLoc(N), MVT::i32);
990  return true;
991 }
992 
993 bool ARMDAGToDAGISel::SelectAddrMode6Offset(SDNode *Op, SDValue N,
994  SDValue &Offset) {
995  LSBaseSDNode *LdSt = cast<LSBaseSDNode>(Op);
997  if (AM != ISD::POST_INC)
998  return false;
999  Offset = N;
1000  if (ConstantSDNode *NC = dyn_cast<ConstantSDNode>(N)) {
1001  if (NC->getZExtValue() * 8 == LdSt->getMemoryVT().getSizeInBits())
1002  Offset = CurDAG->getRegister(0, MVT::i32);
1003  }
1004  return true;
1005 }
1006 
1007 bool ARMDAGToDAGISel::SelectAddrModePC(SDValue N,
1008  SDValue &Offset, SDValue &Label) {
1009  if (N.getOpcode() == ARMISD::PIC_ADD && N.hasOneUse()) {
1010  Offset = N.getOperand(0);
1011  SDValue N1 = N.getOperand(1);
1012  Label = CurDAG->getTargetConstant(cast<ConstantSDNode>(N1)->getZExtValue(),
1013  SDLoc(N), MVT::i32);
1014  return true;
1015  }
1016 
1017  return false;
1018 }
1019 
1020 
1021 //===----------------------------------------------------------------------===//
1022 // Thumb Addressing Modes
1023 //===----------------------------------------------------------------------===//
1024 
1025 bool ARMDAGToDAGISel::SelectThumbAddrModeRR(SDValue N,
1026  SDValue &Base, SDValue &Offset){
1027  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N)) {
1029  if (!NC || !NC->isNullValue())
1030  return false;
1031 
1032  Base = Offset = N;
1033  return true;
1034  }
1035 
1036  Base = N.getOperand(0);
1037  Offset = N.getOperand(1);
1038  return true;
1039 }
1040 
1041 bool
1042 ARMDAGToDAGISel::SelectThumbAddrModeImm5S(SDValue N, unsigned Scale,
1043  SDValue &Base, SDValue &OffImm) {
1044  if (!CurDAG->isBaseWithConstantOffset(N)) {
1045  if (N.getOpcode() == ISD::ADD) {
1046  return false; // We want to select register offset instead
1047  } else if (N.getOpcode() == ARMISD::Wrapper &&
1052  Base = N.getOperand(0);
1053  } else {
1054  Base = N;
1055  }
1056 
1057  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1058  return true;
1059  }
1060 
1061  // If the RHS is + imm5 * scale, fold into addr mode.
1062  int RHSC;
1063  if (isScaledConstantInRange(N.getOperand(1), Scale, 0, 32, RHSC)) {
1064  Base = N.getOperand(0);
1065  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1066  return true;
1067  }
1068 
1069  // Offset is too large, so use register offset instead.
1070  return false;
1071 }
1072 
1073 bool
1074 ARMDAGToDAGISel::SelectThumbAddrModeImm5S4(SDValue N, SDValue &Base,
1075  SDValue &OffImm) {
1076  return SelectThumbAddrModeImm5S(N, 4, Base, OffImm);
1077 }
1078 
1079 bool
1080 ARMDAGToDAGISel::SelectThumbAddrModeImm5S2(SDValue N, SDValue &Base,
1081  SDValue &OffImm) {
1082  return SelectThumbAddrModeImm5S(N, 2, Base, OffImm);
1083 }
1084 
1085 bool
1086 ARMDAGToDAGISel::SelectThumbAddrModeImm5S1(SDValue N, SDValue &Base,
1087  SDValue &OffImm) {
1088  return SelectThumbAddrModeImm5S(N, 1, Base, OffImm);
1089 }
1090 
1091 bool ARMDAGToDAGISel::SelectThumbAddrModeSP(SDValue N,
1092  SDValue &Base, SDValue &OffImm) {
1093  if (N.getOpcode() == ISD::FrameIndex) {
1094  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1095  // Only multiples of 4 are allowed for the offset, so the frame object
1096  // alignment must be at least 4.
1097  MachineFrameInfo &MFI = MF->getFrameInfo();
1098  if (MFI.getObjectAlignment(FI) < 4)
1099  MFI.setObjectAlignment(FI, 4);
1100  Base = CurDAG->getTargetFrameIndex(
1101  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1102  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1103  return true;
1104  }
1105 
1106  if (!CurDAG->isBaseWithConstantOffset(N))
1107  return false;
1108 
1110  if (N.getOperand(0).getOpcode() == ISD::FrameIndex ||
1111  (LHSR && LHSR->getReg() == ARM::SP)) {
1112  // If the RHS is + imm8 * scale, fold into addr mode.
1113  int RHSC;
1114  if (isScaledConstantInRange(N.getOperand(1), /*Scale=*/4, 0, 256, RHSC)) {
1115  Base = N.getOperand(0);
1116  if (Base.getOpcode() == ISD::FrameIndex) {
1117  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1118  // For LHS+RHS to result in an offset that's a multiple of 4 the object
1119  // indexed by the LHS must be 4-byte aligned.
1120  MachineFrameInfo &MFI = MF->getFrameInfo();
1121  if (MFI.getObjectAlignment(FI) < 4)
1122  MFI.setObjectAlignment(FI, 4);
1123  Base = CurDAG->getTargetFrameIndex(
1124  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1125  }
1126  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1127  return true;
1128  }
1129  }
1130 
1131  return false;
1132 }
1133 
1134 
1135 //===----------------------------------------------------------------------===//
1136 // Thumb 2 Addressing Modes
1137 //===----------------------------------------------------------------------===//
1138 
1139 
1140 bool ARMDAGToDAGISel::SelectT2AddrModeImm12(SDValue N,
1141  SDValue &Base, SDValue &OffImm) {
1142  // Match simple R + imm12 operands.
1143 
1144  // Base only.
1145  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1146  !CurDAG->isBaseWithConstantOffset(N)) {
1147  if (N.getOpcode() == ISD::FrameIndex) {
1148  // Match frame index.
1149  int FI = cast<FrameIndexSDNode>(N)->getIndex();
1150  Base = CurDAG->getTargetFrameIndex(
1151  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1152  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1153  return true;
1154  }
1155 
1156  if (N.getOpcode() == ARMISD::Wrapper &&
1160  Base = N.getOperand(0);
1161  if (Base.getOpcode() == ISD::TargetConstantPool)
1162  return false; // We want to select t2LDRpci instead.
1163  } else
1164  Base = N;
1165  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1166  return true;
1167  }
1168 
1169  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1170  if (SelectT2AddrModeImm8(N, Base, OffImm))
1171  // Let t2LDRi8 handle (R - imm8).
1172  return false;
1173 
1174  int RHSC = (int)RHS->getZExtValue();
1175  if (N.getOpcode() == ISD::SUB)
1176  RHSC = -RHSC;
1177 
1178  if (RHSC >= 0 && RHSC < 0x1000) { // 12 bits (unsigned)
1179  Base = N.getOperand(0);
1180  if (Base.getOpcode() == ISD::FrameIndex) {
1181  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1182  Base = CurDAG->getTargetFrameIndex(
1183  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1184  }
1185  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1186  return true;
1187  }
1188  }
1189 
1190  // Base only.
1191  Base = N;
1192  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1193  return true;
1194 }
1195 
1196 bool ARMDAGToDAGISel::SelectT2AddrModeImm8(SDValue N,
1197  SDValue &Base, SDValue &OffImm) {
1198  // Match simple R - imm8 operands.
1199  if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::SUB &&
1200  !CurDAG->isBaseWithConstantOffset(N))
1201  return false;
1202 
1203  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1204  int RHSC = (int)RHS->getSExtValue();
1205  if (N.getOpcode() == ISD::SUB)
1206  RHSC = -RHSC;
1207 
1208  if ((RHSC >= -255) && (RHSC < 0)) { // 8 bits (always negative)
1209  Base = N.getOperand(0);
1210  if (Base.getOpcode() == ISD::FrameIndex) {
1211  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1212  Base = CurDAG->getTargetFrameIndex(
1213  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1214  }
1215  OffImm = CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32);
1216  return true;
1217  }
1218  }
1219 
1220  return false;
1221 }
1222 
1223 bool ARMDAGToDAGISel::SelectT2AddrModeImm8Offset(SDNode *Op, SDValue N,
1224  SDValue &OffImm){
1225  unsigned Opcode = Op->getOpcode();
1226  ISD::MemIndexedMode AM = (Opcode == ISD::LOAD)
1227  ? cast<LoadSDNode>(Op)->getAddressingMode()
1228  : cast<StoreSDNode>(Op)->getAddressingMode();
1229  int RHSC;
1230  if (isScaledConstantInRange(N, /*Scale=*/1, 0, 0x100, RHSC)) { // 8 bits.
1231  OffImm = ((AM == ISD::PRE_INC) || (AM == ISD::POST_INC))
1232  ? CurDAG->getTargetConstant(RHSC, SDLoc(N), MVT::i32)
1233  : CurDAG->getTargetConstant(-RHSC, SDLoc(N), MVT::i32);
1234  return true;
1235  }
1236 
1237  return false;
1238 }
1239 
1240 bool ARMDAGToDAGISel::SelectT2AddrModeSoReg(SDValue N,
1241  SDValue &Base,
1242  SDValue &OffReg, SDValue &ShImm) {
1243  // (R - imm8) should be handled by t2LDRi8. The rest are handled by t2LDRi12.
1244  if (N.getOpcode() != ISD::ADD && !CurDAG->isBaseWithConstantOffset(N))
1245  return false;
1246 
1247  // Leave (R + imm12) for t2LDRi12, (R - imm8) for t2LDRi8.
1248  if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
1249  int RHSC = (int)RHS->getZExtValue();
1250  if (RHSC >= 0 && RHSC < 0x1000) // 12 bits (unsigned)
1251  return false;
1252  else if (RHSC < 0 && RHSC >= -255) // 8 bits
1253  return false;
1254  }
1255 
1256  // Look for (R + R) or (R + (R << [1,2,3])).
1257  unsigned ShAmt = 0;
1258  Base = N.getOperand(0);
1259  OffReg = N.getOperand(1);
1260 
1261  // Swap if it is ((R << c) + R).
1263  if (ShOpcVal != ARM_AM::lsl) {
1264  ShOpcVal = ARM_AM::getShiftOpcForNode(Base.getOpcode());
1265  if (ShOpcVal == ARM_AM::lsl)
1266  std::swap(Base, OffReg);
1267  }
1268 
1269  if (ShOpcVal == ARM_AM::lsl) {
1270  // Check to see if the RHS of the shift is a constant, if not, we can't fold
1271  // it.
1272  if (ConstantSDNode *Sh = dyn_cast<ConstantSDNode>(OffReg.getOperand(1))) {
1273  ShAmt = Sh->getZExtValue();
1274  if (ShAmt < 4 && isShifterOpProfitable(OffReg, ShOpcVal, ShAmt))
1275  OffReg = OffReg.getOperand(0);
1276  else {
1277  ShAmt = 0;
1278  }
1279  }
1280  }
1281 
1282  // If OffReg is a multiply-by-constant and it's profitable to extract a shift
1283  // and use it in a shifted operand do so.
1284  if (OffReg.getOpcode() == ISD::MUL && N.hasOneUse()) {
1285  unsigned PowerOfTwo = 0;
1286  SDValue NewMulConst;
1287  if (canExtractShiftFromMul(OffReg, 3, PowerOfTwo, NewMulConst)) {
1288  HandleSDNode Handle(OffReg);
1289  replaceDAGValue(OffReg.getOperand(1), NewMulConst);
1290  OffReg = Handle.getValue();
1291  ShAmt = PowerOfTwo;
1292  }
1293  }
1294 
1295  ShImm = CurDAG->getTargetConstant(ShAmt, SDLoc(N), MVT::i32);
1296 
1297  return true;
1298 }
1299 
1300 bool ARMDAGToDAGISel::SelectT2AddrModeExclusive(SDValue N, SDValue &Base,
1301  SDValue &OffImm) {
1302  // This *must* succeed since it's used for the irreplaceable ldrex and strex
1303  // instructions.
1304  Base = N;
1305  OffImm = CurDAG->getTargetConstant(0, SDLoc(N), MVT::i32);
1306 
1307  if (N.getOpcode() != ISD::ADD || !CurDAG->isBaseWithConstantOffset(N))
1308  return true;
1309 
1311  if (!RHS)
1312  return true;
1313 
1314  uint32_t RHSC = (int)RHS->getZExtValue();
1315  if (RHSC > 1020 || RHSC % 4 != 0)
1316  return true;
1317 
1318  Base = N.getOperand(0);
1319  if (Base.getOpcode() == ISD::FrameIndex) {
1320  int FI = cast<FrameIndexSDNode>(Base)->getIndex();
1321  Base = CurDAG->getTargetFrameIndex(
1322  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
1323  }
1324 
1325  OffImm = CurDAG->getTargetConstant(RHSC/4, SDLoc(N), MVT::i32);
1326  return true;
1327 }
1328 
1329 //===--------------------------------------------------------------------===//
1330 
1331 /// getAL - Returns a ARMCC::AL immediate node.
1332 static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) {
1333  return CurDAG->getTargetConstant((uint64_t)ARMCC::AL, dl, MVT::i32);
1334 }
1335 
1336 void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) {
1337  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1338  MemOp[0] = cast<MemSDNode>(N)->getMemOperand();
1339  cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1);
1340 }
1341 
1342 bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) {
1343  LoadSDNode *LD = cast<LoadSDNode>(N);
1345  if (AM == ISD::UNINDEXED)
1346  return false;
1347 
1348  EVT LoadedVT = LD->getMemoryVT();
1349  SDValue Offset, AMOpc;
1350  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1351  unsigned Opcode = 0;
1352  bool Match = false;
1353  if (LoadedVT == MVT::i32 && isPre &&
1354  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1355  Opcode = ARM::LDR_PRE_IMM;
1356  Match = true;
1357  } else if (LoadedVT == MVT::i32 && !isPre &&
1358  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1359  Opcode = ARM::LDR_POST_IMM;
1360  Match = true;
1361  } else if (LoadedVT == MVT::i32 &&
1362  SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1363  Opcode = isPre ? ARM::LDR_PRE_REG : ARM::LDR_POST_REG;
1364  Match = true;
1365 
1366  } else if (LoadedVT == MVT::i16 &&
1367  SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1368  Match = true;
1369  Opcode = (LD->getExtensionType() == ISD::SEXTLOAD)
1370  ? (isPre ? ARM::LDRSH_PRE : ARM::LDRSH_POST)
1371  : (isPre ? ARM::LDRH_PRE : ARM::LDRH_POST);
1372  } else if (LoadedVT == MVT::i8 || LoadedVT == MVT::i1) {
1373  if (LD->getExtensionType() == ISD::SEXTLOAD) {
1374  if (SelectAddrMode3Offset(N, LD->getOffset(), Offset, AMOpc)) {
1375  Match = true;
1376  Opcode = isPre ? ARM::LDRSB_PRE : ARM::LDRSB_POST;
1377  }
1378  } else {
1379  if (isPre &&
1380  SelectAddrMode2OffsetImmPre(N, LD->getOffset(), Offset, AMOpc)) {
1381  Match = true;
1382  Opcode = ARM::LDRB_PRE_IMM;
1383  } else if (!isPre &&
1384  SelectAddrMode2OffsetImm(N, LD->getOffset(), Offset, AMOpc)) {
1385  Match = true;
1386  Opcode = ARM::LDRB_POST_IMM;
1387  } else if (SelectAddrMode2OffsetReg(N, LD->getOffset(), Offset, AMOpc)) {
1388  Match = true;
1389  Opcode = isPre ? ARM::LDRB_PRE_REG : ARM::LDRB_POST_REG;
1390  }
1391  }
1392  }
1393 
1394  if (Match) {
1395  if (Opcode == ARM::LDR_PRE_IMM || Opcode == ARM::LDRB_PRE_IMM) {
1396  SDValue Chain = LD->getChain();
1397  SDValue Base = LD->getBasePtr();
1398  SDValue Ops[]= { Base, AMOpc, getAL(CurDAG, SDLoc(N)),
1399  CurDAG->getRegister(0, MVT::i32), Chain };
1400  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1401  MVT::Other, Ops);
1402  transferMemOperands(N, New);
1403  ReplaceNode(N, New);
1404  return true;
1405  } else {
1406  SDValue Chain = LD->getChain();
1407  SDValue Base = LD->getBasePtr();
1408  SDValue Ops[]= { Base, Offset, AMOpc, getAL(CurDAG, SDLoc(N)),
1409  CurDAG->getRegister(0, MVT::i32), Chain };
1410  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1411  MVT::Other, Ops);
1412  transferMemOperands(N, New);
1413  ReplaceNode(N, New);
1414  return true;
1415  }
1416  }
1417 
1418  return false;
1419 }
1420 
1421 bool ARMDAGToDAGISel::tryT1IndexedLoad(SDNode *N) {
1422  LoadSDNode *LD = cast<LoadSDNode>(N);
1423  EVT LoadedVT = LD->getMemoryVT();
1425  if (AM != ISD::POST_INC || LD->getExtensionType() != ISD::NON_EXTLOAD ||
1426  LoadedVT.getSimpleVT().SimpleTy != MVT::i32)
1427  return false;
1428 
1429  auto *COffs = dyn_cast<ConstantSDNode>(LD->getOffset());
1430  if (!COffs || COffs->getZExtValue() != 4)
1431  return false;
1432 
1433  // A T1 post-indexed load is just a single register LDM: LDM r0!, {r1}.
1434  // The encoding of LDM is not how the rest of ISel expects a post-inc load to
1435  // look however, so we use a pseudo here and switch it for a tLDMIA_UPD after
1436  // ISel.
1437  SDValue Chain = LD->getChain();
1438  SDValue Base = LD->getBasePtr();
1439  SDValue Ops[]= { Base, getAL(CurDAG, SDLoc(N)),
1440  CurDAG->getRegister(0, MVT::i32), Chain };
1441  SDNode *New = CurDAG->getMachineNode(ARM::tLDR_postidx, SDLoc(N), MVT::i32,
1442  MVT::i32, MVT::Other, Ops);
1443  transferMemOperands(N, New);
1444  ReplaceNode(N, New);
1445  return true;
1446 }
1447 
1448 bool ARMDAGToDAGISel::tryT2IndexedLoad(SDNode *N) {
1449  LoadSDNode *LD = cast<LoadSDNode>(N);
1451  if (AM == ISD::UNINDEXED)
1452  return false;
1453 
1454  EVT LoadedVT = LD->getMemoryVT();
1455  bool isSExtLd = LD->getExtensionType() == ISD::SEXTLOAD;
1456  SDValue Offset;
1457  bool isPre = (AM == ISD::PRE_INC) || (AM == ISD::PRE_DEC);
1458  unsigned Opcode = 0;
1459  bool Match = false;
1460  if (SelectT2AddrModeImm8Offset(N, LD->getOffset(), Offset)) {
1461  switch (LoadedVT.getSimpleVT().SimpleTy) {
1462  case MVT::i32:
1463  Opcode = isPre ? ARM::t2LDR_PRE : ARM::t2LDR_POST;
1464  break;
1465  case MVT::i16:
1466  if (isSExtLd)
1467  Opcode = isPre ? ARM::t2LDRSH_PRE : ARM::t2LDRSH_POST;
1468  else
1469  Opcode = isPre ? ARM::t2LDRH_PRE : ARM::t2LDRH_POST;
1470  break;
1471  case MVT::i8:
1472  case MVT::i1:
1473  if (isSExtLd)
1474  Opcode = isPre ? ARM::t2LDRSB_PRE : ARM::t2LDRSB_POST;
1475  else
1476  Opcode = isPre ? ARM::t2LDRB_PRE : ARM::t2LDRB_POST;
1477  break;
1478  default:
1479  return false;
1480  }
1481  Match = true;
1482  }
1483 
1484  if (Match) {
1485  SDValue Chain = LD->getChain();
1486  SDValue Base = LD->getBasePtr();
1487  SDValue Ops[]= { Base, Offset, getAL(CurDAG, SDLoc(N)),
1488  CurDAG->getRegister(0, MVT::i32), Chain };
1489  SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(N), MVT::i32, MVT::i32,
1490  MVT::Other, Ops);
1491  transferMemOperands(N, New);
1492  ReplaceNode(N, New);
1493  return true;
1494  }
1495 
1496  return false;
1497 }
1498 
1499 /// \brief Form a GPRPair pseudo register from a pair of GPR regs.
1501  SDLoc dl(V0.getNode());
1502  SDValue RegClass =
1503  CurDAG->getTargetConstant(ARM::GPRPairRegClassID, dl, MVT::i32);
1504  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
1505  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
1506  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1507  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1508 }
1509 
1510 /// \brief Form a D register from a pair of S registers.
1511 SDNode *ARMDAGToDAGISel::createSRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1512  SDLoc dl(V0.getNode());
1513  SDValue RegClass =
1514  CurDAG->getTargetConstant(ARM::DPR_VFP2RegClassID, dl, MVT::i32);
1515  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1516  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1517  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1518  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1519 }
1520 
1521 /// \brief Form a quad register from a pair of D registers.
1522 SDNode *ARMDAGToDAGISel::createDRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1523  SDLoc dl(V0.getNode());
1524  SDValue RegClass = CurDAG->getTargetConstant(ARM::QPRRegClassID, dl,
1525  MVT::i32);
1526  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1527  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1528  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1529  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1530 }
1531 
1532 /// \brief Form 4 consecutive D registers from a pair of Q registers.
1533 SDNode *ARMDAGToDAGISel::createQRegPairNode(EVT VT, SDValue V0, SDValue V1) {
1534  SDLoc dl(V0.getNode());
1535  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1536  MVT::i32);
1537  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1538  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1539  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1 };
1540  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1541 }
1542 
1543 /// \brief Form 4 consecutive S registers.
1544 SDNode *ARMDAGToDAGISel::createQuadSRegsNode(EVT VT, SDValue V0, SDValue V1,
1545  SDValue V2, SDValue V3) {
1546  SDLoc dl(V0.getNode());
1547  SDValue RegClass =
1548  CurDAG->getTargetConstant(ARM::QPR_VFP2RegClassID, dl, MVT::i32);
1549  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::ssub_0, dl, MVT::i32);
1550  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::ssub_1, dl, MVT::i32);
1551  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::ssub_2, dl, MVT::i32);
1552  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::ssub_3, dl, MVT::i32);
1553  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1554  V2, SubReg2, V3, SubReg3 };
1555  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1556 }
1557 
1558 /// \brief Form 4 consecutive D registers.
1559 SDNode *ARMDAGToDAGISel::createQuadDRegsNode(EVT VT, SDValue V0, SDValue V1,
1560  SDValue V2, SDValue V3) {
1561  SDLoc dl(V0.getNode());
1562  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQPRRegClassID, dl,
1563  MVT::i32);
1564  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::dsub_0, dl, MVT::i32);
1565  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::dsub_1, dl, MVT::i32);
1566  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::dsub_2, dl, MVT::i32);
1567  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::dsub_3, dl, MVT::i32);
1568  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1569  V2, SubReg2, V3, SubReg3 };
1570  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1571 }
1572 
1573 /// \brief Form 4 consecutive Q registers.
1574 SDNode *ARMDAGToDAGISel::createQuadQRegsNode(EVT VT, SDValue V0, SDValue V1,
1575  SDValue V2, SDValue V3) {
1576  SDLoc dl(V0.getNode());
1577  SDValue RegClass = CurDAG->getTargetConstant(ARM::QQQQPRRegClassID, dl,
1578  MVT::i32);
1579  SDValue SubReg0 = CurDAG->getTargetConstant(ARM::qsub_0, dl, MVT::i32);
1580  SDValue SubReg1 = CurDAG->getTargetConstant(ARM::qsub_1, dl, MVT::i32);
1581  SDValue SubReg2 = CurDAG->getTargetConstant(ARM::qsub_2, dl, MVT::i32);
1582  SDValue SubReg3 = CurDAG->getTargetConstant(ARM::qsub_3, dl, MVT::i32);
1583  const SDValue Ops[] = { RegClass, V0, SubReg0, V1, SubReg1,
1584  V2, SubReg2, V3, SubReg3 };
1585  return CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, dl, VT, Ops);
1586 }
1587 
1588 /// GetVLDSTAlign - Get the alignment (in bytes) for the alignment operand
1589 /// of a NEON VLD or VST instruction. The supported values depend on the
1590 /// number of registers being loaded.
1591 SDValue ARMDAGToDAGISel::GetVLDSTAlign(SDValue Align, const SDLoc &dl,
1592  unsigned NumVecs, bool is64BitVector) {
1593  unsigned NumRegs = NumVecs;
1594  if (!is64BitVector && NumVecs < 3)
1595  NumRegs *= 2;
1596 
1597  unsigned Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
1598  if (Alignment >= 32 && NumRegs == 4)
1599  Alignment = 32;
1600  else if (Alignment >= 16 && (NumRegs == 2 || NumRegs == 4))
1601  Alignment = 16;
1602  else if (Alignment >= 8)
1603  Alignment = 8;
1604  else
1605  Alignment = 0;
1606 
1607  return CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
1608 }
1609 
1610 static bool isVLDfixed(unsigned Opc)
1611 {
1612  switch (Opc) {
1613  default: return false;
1614  case ARM::VLD1d8wb_fixed : return true;
1615  case ARM::VLD1d16wb_fixed : return true;
1616  case ARM::VLD1d64Qwb_fixed : return true;
1617  case ARM::VLD1d32wb_fixed : return true;
1618  case ARM::VLD1d64wb_fixed : return true;
1619  case ARM::VLD1d64TPseudoWB_fixed : return true;
1620  case ARM::VLD1d64QPseudoWB_fixed : return true;
1621  case ARM::VLD1q8wb_fixed : return true;
1622  case ARM::VLD1q16wb_fixed : return true;
1623  case ARM::VLD1q32wb_fixed : return true;
1624  case ARM::VLD1q64wb_fixed : return true;
1625  case ARM::VLD1DUPd8wb_fixed : return true;
1626  case ARM::VLD1DUPd16wb_fixed : return true;
1627  case ARM::VLD1DUPd32wb_fixed : return true;
1628  case ARM::VLD1DUPq8wb_fixed : return true;
1629  case ARM::VLD1DUPq16wb_fixed : return true;
1630  case ARM::VLD1DUPq32wb_fixed : return true;
1631  case ARM::VLD2d8wb_fixed : return true;
1632  case ARM::VLD2d16wb_fixed : return true;
1633  case ARM::VLD2d32wb_fixed : return true;
1634  case ARM::VLD2q8PseudoWB_fixed : return true;
1635  case ARM::VLD2q16PseudoWB_fixed : return true;
1636  case ARM::VLD2q32PseudoWB_fixed : return true;
1637  case ARM::VLD2DUPd8wb_fixed : return true;
1638  case ARM::VLD2DUPd16wb_fixed : return true;
1639  case ARM::VLD2DUPd32wb_fixed : return true;
1640  }
1641 }
1642 
1643 static bool isVSTfixed(unsigned Opc)
1644 {
1645  switch (Opc) {
1646  default: return false;
1647  case ARM::VST1d8wb_fixed : return true;
1648  case ARM::VST1d16wb_fixed : return true;
1649  case ARM::VST1d32wb_fixed : return true;
1650  case ARM::VST1d64wb_fixed : return true;
1651  case ARM::VST1q8wb_fixed : return true;
1652  case ARM::VST1q16wb_fixed : return true;
1653  case ARM::VST1q32wb_fixed : return true;
1654  case ARM::VST1q64wb_fixed : return true;
1655  case ARM::VST1d64TPseudoWB_fixed : return true;
1656  case ARM::VST1d64QPseudoWB_fixed : return true;
1657  case ARM::VST2d8wb_fixed : return true;
1658  case ARM::VST2d16wb_fixed : return true;
1659  case ARM::VST2d32wb_fixed : return true;
1660  case ARM::VST2q8PseudoWB_fixed : return true;
1661  case ARM::VST2q16PseudoWB_fixed : return true;
1662  case ARM::VST2q32PseudoWB_fixed : return true;
1663  }
1664 }
1665 
1666 // Get the register stride update opcode of a VLD/VST instruction that
1667 // is otherwise equivalent to the given fixed stride updating instruction.
1668 static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc) {
1669  assert((isVLDfixed(Opc) || isVSTfixed(Opc))
1670  && "Incorrect fixed stride updating instruction.");
1671  switch (Opc) {
1672  default: break;
1673  case ARM::VLD1d8wb_fixed: return ARM::VLD1d8wb_register;
1674  case ARM::VLD1d16wb_fixed: return ARM::VLD1d16wb_register;
1675  case ARM::VLD1d32wb_fixed: return ARM::VLD1d32wb_register;
1676  case ARM::VLD1d64wb_fixed: return ARM::VLD1d64wb_register;
1677  case ARM::VLD1q8wb_fixed: return ARM::VLD1q8wb_register;
1678  case ARM::VLD1q16wb_fixed: return ARM::VLD1q16wb_register;
1679  case ARM::VLD1q32wb_fixed: return ARM::VLD1q32wb_register;
1680  case ARM::VLD1q64wb_fixed: return ARM::VLD1q64wb_register;
1681  case ARM::VLD1d64Twb_fixed: return ARM::VLD1d64Twb_register;
1682  case ARM::VLD1d64Qwb_fixed: return ARM::VLD1d64Qwb_register;
1683  case ARM::VLD1d64TPseudoWB_fixed: return ARM::VLD1d64TPseudoWB_register;
1684  case ARM::VLD1d64QPseudoWB_fixed: return ARM::VLD1d64QPseudoWB_register;
1685  case ARM::VLD1DUPd8wb_fixed : return ARM::VLD1DUPd8wb_register;
1686  case ARM::VLD1DUPd16wb_fixed : return ARM::VLD1DUPd16wb_register;
1687  case ARM::VLD1DUPd32wb_fixed : return ARM::VLD1DUPd32wb_register;
1688  case ARM::VLD1DUPq8wb_fixed : return ARM::VLD1DUPq8wb_register;
1689  case ARM::VLD1DUPq16wb_fixed : return ARM::VLD1DUPq16wb_register;
1690  case ARM::VLD1DUPq32wb_fixed : return ARM::VLD1DUPq32wb_register;
1691 
1692  case ARM::VST1d8wb_fixed: return ARM::VST1d8wb_register;
1693  case ARM::VST1d16wb_fixed: return ARM::VST1d16wb_register;
1694  case ARM::VST1d32wb_fixed: return ARM::VST1d32wb_register;
1695  case ARM::VST1d64wb_fixed: return ARM::VST1d64wb_register;
1696  case ARM::VST1q8wb_fixed: return ARM::VST1q8wb_register;
1697  case ARM::VST1q16wb_fixed: return ARM::VST1q16wb_register;
1698  case ARM::VST1q32wb_fixed: return ARM::VST1q32wb_register;
1699  case ARM::VST1q64wb_fixed: return ARM::VST1q64wb_register;
1700  case ARM::VST1d64TPseudoWB_fixed: return ARM::VST1d64TPseudoWB_register;
1701  case ARM::VST1d64QPseudoWB_fixed: return ARM::VST1d64QPseudoWB_register;
1702 
1703  case ARM::VLD2d8wb_fixed: return ARM::VLD2d8wb_register;
1704  case ARM::VLD2d16wb_fixed: return ARM::VLD2d16wb_register;
1705  case ARM::VLD2d32wb_fixed: return ARM::VLD2d32wb_register;
1706  case ARM::VLD2q8PseudoWB_fixed: return ARM::VLD2q8PseudoWB_register;
1707  case ARM::VLD2q16PseudoWB_fixed: return ARM::VLD2q16PseudoWB_register;
1708  case ARM::VLD2q32PseudoWB_fixed: return ARM::VLD2q32PseudoWB_register;
1709 
1710  case ARM::VST2d8wb_fixed: return ARM::VST2d8wb_register;
1711  case ARM::VST2d16wb_fixed: return ARM::VST2d16wb_register;
1712  case ARM::VST2d32wb_fixed: return ARM::VST2d32wb_register;
1713  case ARM::VST2q8PseudoWB_fixed: return ARM::VST2q8PseudoWB_register;
1714  case ARM::VST2q16PseudoWB_fixed: return ARM::VST2q16PseudoWB_register;
1715  case ARM::VST2q32PseudoWB_fixed: return ARM::VST2q32PseudoWB_register;
1716 
1717  case ARM::VLD2DUPd8wb_fixed: return ARM::VLD2DUPd8wb_register;
1718  case ARM::VLD2DUPd16wb_fixed: return ARM::VLD2DUPd16wb_register;
1719  case ARM::VLD2DUPd32wb_fixed: return ARM::VLD2DUPd32wb_register;
1720  }
1721  return Opc; // If not one we handle, return it unchanged.
1722 }
1723 
1724 /// Returns true if the given increment is a Constant known to be equal to the
1725 /// access size performed by a NEON load/store. This means the "[rN]!" form can
1726 /// be used.
1727 static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs) {
1728  auto C = dyn_cast<ConstantSDNode>(Inc);
1729  return C && C->getZExtValue() == VecTy.getSizeInBits() / 8 * NumVecs;
1730 }
1731 
1732 void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs,
1733  const uint16_t *DOpcodes,
1734  const uint16_t *QOpcodes0,
1735  const uint16_t *QOpcodes1) {
1736  assert(NumVecs >= 1 && NumVecs <= 4 && "VLD NumVecs out-of-range");
1737  SDLoc dl(N);
1738 
1739  SDValue MemAddr, Align;
1740  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1741  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1742  return;
1743 
1744  SDValue Chain = N->getOperand(0);
1745  EVT VT = N->getValueType(0);
1746  bool is64BitVector = VT.is64BitVector();
1747  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1748 
1749  unsigned OpcodeIndex;
1750  switch (VT.getSimpleVT().SimpleTy) {
1751  default: llvm_unreachable("unhandled vld type");
1752  // Double-register operations:
1753  case MVT::v8i8: OpcodeIndex = 0; break;
1754  case MVT::v4i16: OpcodeIndex = 1; break;
1755  case MVT::v2f32:
1756  case MVT::v2i32: OpcodeIndex = 2; break;
1757  case MVT::v1i64: OpcodeIndex = 3; break;
1758  // Quad-register operations:
1759  case MVT::v16i8: OpcodeIndex = 0; break;
1760  case MVT::v8i16: OpcodeIndex = 1; break;
1761  case MVT::v4f32:
1762  case MVT::v4i32: OpcodeIndex = 2; break;
1763  case MVT::v2f64:
1764  case MVT::v2i64: OpcodeIndex = 3;
1765  assert(NumVecs == 1 && "v2i64 type only supported for VLD1");
1766  break;
1767  }
1768 
1769  EVT ResTy;
1770  if (NumVecs == 1)
1771  ResTy = VT;
1772  else {
1773  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
1774  if (!is64BitVector)
1775  ResTyElts *= 2;
1776  ResTy = EVT::getVectorVT(*CurDAG->getContext(), MVT::i64, ResTyElts);
1777  }
1778  std::vector<EVT> ResTys;
1779  ResTys.push_back(ResTy);
1780  if (isUpdating)
1781  ResTys.push_back(MVT::i32);
1782  ResTys.push_back(MVT::Other);
1783 
1784  SDValue Pred = getAL(CurDAG, dl);
1785  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1786  SDNode *VLd;
1788 
1789  // Double registers and VLD1/VLD2 quad registers are directly supported.
1790  if (is64BitVector || NumVecs <= 2) {
1791  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1792  QOpcodes0[OpcodeIndex]);
1793  Ops.push_back(MemAddr);
1794  Ops.push_back(Align);
1795  if (isUpdating) {
1796  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1797  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1798  if (!IsImmUpdate) {
1799  // We use a VLD1 for v1i64 even if the pseudo says vld2/3/4, so
1800  // check for the opcode rather than the number of vector elements.
1801  if (isVLDfixed(Opc))
1802  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1803  Ops.push_back(Inc);
1804  // VLD1/VLD2 fixed increment does not need Reg0 so only include it in
1805  // the operands if not such an opcode.
1806  } else if (!isVLDfixed(Opc))
1807  Ops.push_back(Reg0);
1808  }
1809  Ops.push_back(Pred);
1810  Ops.push_back(Reg0);
1811  Ops.push_back(Chain);
1812  VLd = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1813 
1814  } else {
1815  // Otherwise, quad registers are loaded with two separate instructions,
1816  // where one loads the even registers and the other loads the odd registers.
1817  EVT AddrTy = MemAddr.getValueType();
1818 
1819  // Load the even subregs. This is always an updating load, so that it
1820  // provides the address to the second load for the odd subregs.
1821  SDValue ImplDef =
1822  SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, ResTy), 0);
1823  const SDValue OpsA[] = { MemAddr, Align, Reg0, ImplDef, Pred, Reg0, Chain };
1824  SDNode *VLdA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1825  ResTy, AddrTy, MVT::Other, OpsA);
1826  Chain = SDValue(VLdA, 2);
1827 
1828  // Load the odd subregs.
1829  Ops.push_back(SDValue(VLdA, 1));
1830  Ops.push_back(Align);
1831  if (isUpdating) {
1832  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1833  assert(isa<ConstantSDNode>(Inc.getNode()) &&
1834  "only constant post-increment update allowed for VLD3/4");
1835  (void)Inc;
1836  Ops.push_back(Reg0);
1837  }
1838  Ops.push_back(SDValue(VLdA, 0));
1839  Ops.push_back(Pred);
1840  Ops.push_back(Reg0);
1841  Ops.push_back(Chain);
1842  VLd = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops);
1843  }
1844 
1845  // Transfer memoperands.
1846  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1847  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1848  cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1);
1849 
1850  if (NumVecs == 1) {
1851  ReplaceNode(N, VLd);
1852  return;
1853  }
1854 
1855  // Extract out the subregisters.
1856  SDValue SuperReg = SDValue(VLd, 0);
1857  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
1858  ARM::qsub_3 == ARM::qsub_0 + 3,
1859  "Unexpected subreg numbering");
1860  unsigned Sub0 = (is64BitVector ? ARM::dsub_0 : ARM::qsub_0);
1861  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
1862  ReplaceUses(SDValue(N, Vec),
1863  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
1864  ReplaceUses(SDValue(N, NumVecs), SDValue(VLd, 1));
1865  if (isUpdating)
1866  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLd, 2));
1867  CurDAG->RemoveDeadNode(N);
1868 }
1869 
1870 void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs,
1871  const uint16_t *DOpcodes,
1872  const uint16_t *QOpcodes0,
1873  const uint16_t *QOpcodes1) {
1874  assert(NumVecs >= 1 && NumVecs <= 4 && "VST NumVecs out-of-range");
1875  SDLoc dl(N);
1876 
1877  SDValue MemAddr, Align;
1878  unsigned AddrOpIdx = isUpdating ? 1 : 2;
1879  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
1880  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
1881  return;
1882 
1883  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
1884  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
1885 
1886  SDValue Chain = N->getOperand(0);
1887  EVT VT = N->getOperand(Vec0Idx).getValueType();
1888  bool is64BitVector = VT.is64BitVector();
1889  Align = GetVLDSTAlign(Align, dl, NumVecs, is64BitVector);
1890 
1891  unsigned OpcodeIndex;
1892  switch (VT.getSimpleVT().SimpleTy) {
1893  default: llvm_unreachable("unhandled vst type");
1894  // Double-register operations:
1895  case MVT::v8i8: OpcodeIndex = 0; break;
1896  case MVT::v4f16:
1897  case MVT::v4i16: OpcodeIndex = 1; break;
1898  case MVT::v2f32:
1899  case MVT::v2i32: OpcodeIndex = 2; break;
1900  case MVT::v1i64: OpcodeIndex = 3; break;
1901  // Quad-register operations:
1902  case MVT::v16i8: OpcodeIndex = 0; break;
1903  case MVT::v8f16:
1904  case MVT::v8i16: OpcodeIndex = 1; break;
1905  case MVT::v4f32:
1906  case MVT::v4i32: OpcodeIndex = 2; break;
1907  case MVT::v2f64:
1908  case MVT::v2i64: OpcodeIndex = 3;
1909  assert(NumVecs == 1 && "v2i64 type only supported for VST1");
1910  break;
1911  }
1912 
1913  std::vector<EVT> ResTys;
1914  if (isUpdating)
1915  ResTys.push_back(MVT::i32);
1916  ResTys.push_back(MVT::Other);
1917 
1918  SDValue Pred = getAL(CurDAG, dl);
1919  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
1921 
1922  // Double registers and VST1/VST2 quad registers are directly supported.
1923  if (is64BitVector || NumVecs <= 2) {
1924  SDValue SrcReg;
1925  if (NumVecs == 1) {
1926  SrcReg = N->getOperand(Vec0Idx);
1927  } else if (is64BitVector) {
1928  // Form a REG_SEQUENCE to force register allocation.
1929  SDValue V0 = N->getOperand(Vec0Idx + 0);
1930  SDValue V1 = N->getOperand(Vec0Idx + 1);
1931  if (NumVecs == 2)
1932  SrcReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
1933  else {
1934  SDValue V2 = N->getOperand(Vec0Idx + 2);
1935  // If it's a vst3, form a quad D-register and leave the last part as
1936  // an undef.
1937  SDValue V3 = (NumVecs == 3)
1938  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,dl,VT), 0)
1939  : N->getOperand(Vec0Idx + 3);
1940  SrcReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
1941  }
1942  } else {
1943  // Form a QQ register.
1944  SDValue Q0 = N->getOperand(Vec0Idx);
1945  SDValue Q1 = N->getOperand(Vec0Idx + 1);
1946  SrcReg = SDValue(createQRegPairNode(MVT::v4i64, Q0, Q1), 0);
1947  }
1948 
1949  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
1950  QOpcodes0[OpcodeIndex]);
1951  Ops.push_back(MemAddr);
1952  Ops.push_back(Align);
1953  if (isUpdating) {
1954  SDValue Inc = N->getOperand(AddrOpIdx + 1);
1955  bool IsImmUpdate = isPerfectIncrement(Inc, VT, NumVecs);
1956  if (!IsImmUpdate) {
1957  // We use a VST1 for v1i64 even if the pseudo says VST2/3/4, so
1958  // check for the opcode rather than the number of vector elements.
1959  if (isVSTfixed(Opc))
1960  Opc = getVLDSTRegisterUpdateOpcode(Opc);
1961  Ops.push_back(Inc);
1962  }
1963  // VST1/VST2 fixed increment does not need Reg0 so only include it in
1964  // the operands if not such an opcode.
1965  else if (!isVSTfixed(Opc))
1966  Ops.push_back(Reg0);
1967  }
1968  Ops.push_back(SrcReg);
1969  Ops.push_back(Pred);
1970  Ops.push_back(Reg0);
1971  Ops.push_back(Chain);
1972  SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
1973 
1974  // Transfer memoperands.
1975  cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1);
1976 
1977  ReplaceNode(N, VSt);
1978  return;
1979  }
1980 
1981  // Otherwise, quad registers are stored with two separate instructions,
1982  // where one stores the even registers and the other stores the odd registers.
1983 
1984  // Form the QQQQ REG_SEQUENCE.
1985  SDValue V0 = N->getOperand(Vec0Idx + 0);
1986  SDValue V1 = N->getOperand(Vec0Idx + 1);
1987  SDValue V2 = N->getOperand(Vec0Idx + 2);
1988  SDValue V3 = (NumVecs == 3)
1989  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
1990  : N->getOperand(Vec0Idx + 3);
1991  SDValue RegSeq = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
1992 
1993  // Store the even D registers. This is always an updating store, so that it
1994  // provides the address to the second store for the odd subregs.
1995  const SDValue OpsA[] = { MemAddr, Align, Reg0, RegSeq, Pred, Reg0, Chain };
1996  SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl,
1997  MemAddr.getValueType(),
1998  MVT::Other, OpsA);
1999  cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1);
2000  Chain = SDValue(VStA, 1);
2001 
2002  // Store the odd D registers.
2003  Ops.push_back(SDValue(VStA, 0));
2004  Ops.push_back(Align);
2005  if (isUpdating) {
2006  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2007  assert(isa<ConstantSDNode>(Inc.getNode()) &&
2008  "only constant post-increment update allowed for VST3/4");
2009  (void)Inc;
2010  Ops.push_back(Reg0);
2011  }
2012  Ops.push_back(RegSeq);
2013  Ops.push_back(Pred);
2014  Ops.push_back(Reg0);
2015  Ops.push_back(Chain);
2016  SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys,
2017  Ops);
2018  cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1);
2019  ReplaceNode(N, VStB);
2020 }
2021 
2022 void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating,
2023  unsigned NumVecs,
2024  const uint16_t *DOpcodes,
2025  const uint16_t *QOpcodes) {
2026  assert(NumVecs >=2 && NumVecs <= 4 && "VLDSTLane NumVecs out-of-range");
2027  SDLoc dl(N);
2028 
2029  SDValue MemAddr, Align;
2030  unsigned AddrOpIdx = isUpdating ? 1 : 2;
2031  unsigned Vec0Idx = 3; // AddrOpIdx + (isUpdating ? 2 : 1)
2032  if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align))
2033  return;
2034 
2035  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2036  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2037 
2038  SDValue Chain = N->getOperand(0);
2039  unsigned Lane =
2040  cast<ConstantSDNode>(N->getOperand(Vec0Idx + NumVecs))->getZExtValue();
2041  EVT VT = N->getOperand(Vec0Idx).getValueType();
2042  bool is64BitVector = VT.is64BitVector();
2043 
2044  unsigned Alignment = 0;
2045  if (NumVecs != 3) {
2046  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2047  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2048  if (Alignment > NumBytes)
2049  Alignment = NumBytes;
2050  if (Alignment < 8 && Alignment < NumBytes)
2051  Alignment = 0;
2052  // Alignment must be a power of two; make sure of that.
2053  Alignment = (Alignment & -Alignment);
2054  if (Alignment == 1)
2055  Alignment = 0;
2056  }
2057  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2058 
2059  unsigned OpcodeIndex;
2060  switch (VT.getSimpleVT().SimpleTy) {
2061  default: llvm_unreachable("unhandled vld/vst lane type");
2062  // Double-register operations:
2063  case MVT::v8i8: OpcodeIndex = 0; break;
2064  case MVT::v4i16: OpcodeIndex = 1; break;
2065  case MVT::v2f32:
2066  case MVT::v2i32: OpcodeIndex = 2; break;
2067  // Quad-register operations:
2068  case MVT::v8i16: OpcodeIndex = 0; break;
2069  case MVT::v4f32:
2070  case MVT::v4i32: OpcodeIndex = 1; break;
2071  }
2072 
2073  std::vector<EVT> ResTys;
2074  if (IsLoad) {
2075  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2076  if (!is64BitVector)
2077  ResTyElts *= 2;
2078  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(),
2079  MVT::i64, ResTyElts));
2080  }
2081  if (isUpdating)
2082  ResTys.push_back(MVT::i32);
2083  ResTys.push_back(MVT::Other);
2084 
2085  SDValue Pred = getAL(CurDAG, dl);
2086  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2087 
2089  Ops.push_back(MemAddr);
2090  Ops.push_back(Align);
2091  if (isUpdating) {
2092  SDValue Inc = N->getOperand(AddrOpIdx + 1);
2093  bool IsImmUpdate =
2094  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2095  Ops.push_back(IsImmUpdate ? Reg0 : Inc);
2096  }
2097 
2098  SDValue SuperReg;
2099  SDValue V0 = N->getOperand(Vec0Idx + 0);
2100  SDValue V1 = N->getOperand(Vec0Idx + 1);
2101  if (NumVecs == 2) {
2102  if (is64BitVector)
2103  SuperReg = SDValue(createDRegPairNode(MVT::v2i64, V0, V1), 0);
2104  else
2105  SuperReg = SDValue(createQRegPairNode(MVT::v4i64, V0, V1), 0);
2106  } else {
2107  SDValue V2 = N->getOperand(Vec0Idx + 2);
2108  SDValue V3 = (NumVecs == 3)
2109  ? SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, dl, VT), 0)
2110  : N->getOperand(Vec0Idx + 3);
2111  if (is64BitVector)
2112  SuperReg = SDValue(createQuadDRegsNode(MVT::v4i64, V0, V1, V2, V3), 0);
2113  else
2114  SuperReg = SDValue(createQuadQRegsNode(MVT::v8i64, V0, V1, V2, V3), 0);
2115  }
2116  Ops.push_back(SuperReg);
2117  Ops.push_back(getI32Imm(Lane, dl));
2118  Ops.push_back(Pred);
2119  Ops.push_back(Reg0);
2120  Ops.push_back(Chain);
2121 
2122  unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] :
2123  QOpcodes[OpcodeIndex]);
2124  SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2125  cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1);
2126  if (!IsLoad) {
2127  ReplaceNode(N, VLdLn);
2128  return;
2129  }
2130 
2131  // Extract the subregisters.
2132  SuperReg = SDValue(VLdLn, 0);
2133  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7 &&
2134  ARM::qsub_3 == ARM::qsub_0 + 3,
2135  "Unexpected subreg numbering");
2136  unsigned Sub0 = is64BitVector ? ARM::dsub_0 : ARM::qsub_0;
2137  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2138  ReplaceUses(SDValue(N, Vec),
2139  CurDAG->getTargetExtractSubreg(Sub0 + Vec, dl, VT, SuperReg));
2140  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdLn, 1));
2141  if (isUpdating)
2142  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdLn, 2));
2143  CurDAG->RemoveDeadNode(N);
2144 }
2145 
2146 void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool isUpdating, unsigned NumVecs,
2147  const uint16_t *DOpcodes,
2148  const uint16_t *QOpcodes) {
2149  assert(NumVecs >= 1 && NumVecs <= 4 && "VLDDup NumVecs out-of-range");
2150  SDLoc dl(N);
2151 
2152  SDValue MemAddr, Align;
2153  if (!SelectAddrMode6(N, N->getOperand(1), MemAddr, Align))
2154  return;
2155 
2156  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2157  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
2158 
2159  SDValue Chain = N->getOperand(0);
2160  EVT VT = N->getValueType(0);
2161 
2162  unsigned Alignment = 0;
2163  if (NumVecs != 3) {
2164  Alignment = cast<ConstantSDNode>(Align)->getZExtValue();
2165  unsigned NumBytes = NumVecs * VT.getScalarSizeInBits() / 8;
2166  if (Alignment > NumBytes)
2167  Alignment = NumBytes;
2168  if (Alignment < 8 && Alignment < NumBytes)
2169  Alignment = 0;
2170  // Alignment must be a power of two; make sure of that.
2171  Alignment = (Alignment & -Alignment);
2172  if (Alignment == 1)
2173  Alignment = 0;
2174  }
2175  Align = CurDAG->getTargetConstant(Alignment, dl, MVT::i32);
2176 
2177  unsigned Opc;
2178  switch (VT.getSimpleVT().SimpleTy) {
2179  default: llvm_unreachable("unhandled vld-dup type");
2180  case MVT::v8i8: Opc = DOpcodes[0]; break;
2181  case MVT::v16i8: Opc = QOpcodes[0]; break;
2182  case MVT::v4i16: Opc = DOpcodes[1]; break;
2183  case MVT::v8i16: Opc = QOpcodes[1]; break;
2184  case MVT::v2f32:
2185  case MVT::v2i32: Opc = DOpcodes[2]; break;
2186  case MVT::v4f32:
2187  case MVT::v4i32: Opc = QOpcodes[2]; break;
2188  }
2189 
2190  SDValue Pred = getAL(CurDAG, dl);
2191  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2193  Ops.push_back(MemAddr);
2194  Ops.push_back(Align);
2195  if (isUpdating) {
2196  // fixed-stride update instructions don't have an explicit writeback
2197  // operand. It's implicit in the opcode itself.
2198  SDValue Inc = N->getOperand(2);
2199  bool IsImmUpdate =
2200  isPerfectIncrement(Inc, VT.getVectorElementType(), NumVecs);
2201  if (NumVecs <= 2 && !IsImmUpdate)
2202  Opc = getVLDSTRegisterUpdateOpcode(Opc);
2203  if (!IsImmUpdate)
2204  Ops.push_back(Inc);
2205  // FIXME: VLD3 and VLD4 haven't been updated to that form yet.
2206  else if (NumVecs > 2)
2207  Ops.push_back(Reg0);
2208  }
2209  Ops.push_back(Pred);
2210  Ops.push_back(Reg0);
2211  Ops.push_back(Chain);
2212 
2213  unsigned ResTyElts = (NumVecs == 3) ? 4 : NumVecs;
2214  std::vector<EVT> ResTys;
2215  ResTys.push_back(EVT::getVectorVT(*CurDAG->getContext(), MVT::i64,ResTyElts));
2216  if (isUpdating)
2217  ResTys.push_back(MVT::i32);
2218  ResTys.push_back(MVT::Other);
2219  SDNode *VLdDup = CurDAG->getMachineNode(Opc, dl, ResTys, Ops);
2220  cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1);
2221 
2222  // Extract the subregisters.
2223  if (NumVecs == 1) {
2224  ReplaceUses(SDValue(N, 0), SDValue(VLdDup, 0));
2225  } else {
2226  SDValue SuperReg = SDValue(VLdDup, 0);
2227  static_assert(ARM::dsub_7 == ARM::dsub_0 + 7, "Unexpected subreg numbering");
2228  unsigned SubIdx = ARM::dsub_0;
2229  for (unsigned Vec = 0; Vec < NumVecs; ++Vec)
2230  ReplaceUses(SDValue(N, Vec),
2231  CurDAG->getTargetExtractSubreg(SubIdx+Vec, dl, VT, SuperReg));
2232  }
2233  ReplaceUses(SDValue(N, NumVecs), SDValue(VLdDup, 1));
2234  if (isUpdating)
2235  ReplaceUses(SDValue(N, NumVecs + 1), SDValue(VLdDup, 2));
2236  CurDAG->RemoveDeadNode(N);
2237 }
2238 
2239 bool ARMDAGToDAGISel::tryV6T2BitfieldExtractOp(SDNode *N, bool isSigned) {
2240  if (!Subtarget->hasV6T2Ops())
2241  return false;
2242 
2243  unsigned Opc = isSigned
2244  ? (Subtarget->isThumb() ? ARM::t2SBFX : ARM::SBFX)
2245  : (Subtarget->isThumb() ? ARM::t2UBFX : ARM::UBFX);
2246  SDLoc dl(N);
2247 
2248  // For unsigned extracts, check for a shift right and mask
2249  unsigned And_imm = 0;
2250  if (N->getOpcode() == ISD::AND) {
2251  if (isOpcWithIntImmediate(N, ISD::AND, And_imm)) {
2252 
2253  // The immediate is a mask of the low bits iff imm & (imm+1) == 0
2254  if (And_imm & (And_imm + 1))
2255  return false;
2256 
2257  unsigned Srl_imm = 0;
2259  Srl_imm)) {
2260  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2261 
2262  // Note: The width operand is encoded as width-1.
2263  unsigned Width = countTrailingOnes(And_imm) - 1;
2264  unsigned LSB = Srl_imm;
2265 
2266  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2267 
2268  if ((LSB + Width + 1) == N->getValueType(0).getSizeInBits()) {
2269  // It's cheaper to use a right shift to extract the top bits.
2270  if (Subtarget->isThumb()) {
2271  Opc = isSigned ? ARM::t2ASRri : ARM::t2LSRri;
2272  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2273  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2274  getAL(CurDAG, dl), Reg0, Reg0 };
2275  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2276  return true;
2277  }
2278 
2279  // ARM models shift instructions as MOVsi with shifter operand.
2281  SDValue ShOpc =
2282  CurDAG->getTargetConstant(ARM_AM::getSORegOpc(ShOpcVal, LSB), dl,
2283  MVT::i32);
2284  SDValue Ops[] = { N->getOperand(0).getOperand(0), ShOpc,
2285  getAL(CurDAG, dl), Reg0, Reg0 };
2286  CurDAG->SelectNodeTo(N, ARM::MOVsi, MVT::i32, Ops);
2287  return true;
2288  }
2289 
2290  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2291  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2292  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2293  getAL(CurDAG, dl), Reg0 };
2294  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2295  return true;
2296  }
2297  }
2298  return false;
2299  }
2300 
2301  // Otherwise, we're looking for a shift of a shift
2302  unsigned Shl_imm = 0;
2303  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SHL, Shl_imm)) {
2304  assert(Shl_imm > 0 && Shl_imm < 32 && "bad amount in shift node!");
2305  unsigned Srl_imm = 0;
2306  if (isInt32Immediate(N->getOperand(1), Srl_imm)) {
2307  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2308  // Note: The width operand is encoded as width-1.
2309  unsigned Width = 32 - Srl_imm - 1;
2310  int LSB = Srl_imm - Shl_imm;
2311  if (LSB < 0)
2312  return false;
2313  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2314  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2315  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2316  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2317  getAL(CurDAG, dl), Reg0 };
2318  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2319  return true;
2320  }
2321  }
2322 
2323  // Or we are looking for a shift of an and, with a mask operand
2324  if (isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::AND, And_imm) &&
2325  isShiftedMask_32(And_imm)) {
2326  unsigned Srl_imm = 0;
2327  unsigned LSB = countTrailingZeros(And_imm);
2328  // Shift must be the same as the ands lsb
2329  if (isInt32Immediate(N->getOperand(1), Srl_imm) && Srl_imm == LSB) {
2330  assert(Srl_imm > 0 && Srl_imm < 32 && "bad amount in shift node!");
2331  unsigned MSB = 31 - countLeadingZeros(And_imm);
2332  // Note: The width operand is encoded as width-1.
2333  unsigned Width = MSB - LSB;
2334  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2335  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2336  CurDAG->getTargetConstant(Srl_imm, dl, MVT::i32),
2337  CurDAG->getTargetConstant(Width, dl, MVT::i32),
2338  getAL(CurDAG, dl), Reg0 };
2339  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2340  return true;
2341  }
2342  }
2343 
2344  if (N->getOpcode() == ISD::SIGN_EXTEND_INREG) {
2345  unsigned Width = cast<VTSDNode>(N->getOperand(1))->getVT().getSizeInBits();
2346  unsigned LSB = 0;
2347  if (!isOpcWithIntImmediate(N->getOperand(0).getNode(), ISD::SRL, LSB) &&
2349  return false;
2350 
2351  if (LSB + Width > 32)
2352  return false;
2353 
2354  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2355  SDValue Ops[] = { N->getOperand(0).getOperand(0),
2356  CurDAG->getTargetConstant(LSB, dl, MVT::i32),
2357  CurDAG->getTargetConstant(Width - 1, dl, MVT::i32),
2358  getAL(CurDAG, dl), Reg0 };
2359  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2360  return true;
2361  }
2362 
2363  return false;
2364 }
2365 
2366 /// Target-specific DAG combining for ISD::XOR.
2367 /// Target-independent combining lowers SELECT_CC nodes of the form
2368 /// select_cc setg[ge] X, 0, X, -X
2369 /// select_cc setgt X, -1, X, -X
2370 /// select_cc setl[te] X, 0, -X, X
2371 /// select_cc setlt X, 1, -X, X
2372 /// which represent Integer ABS into:
2373 /// Y = sra (X, size(X)-1); xor (add (X, Y), Y)
2374 /// ARM instruction selection detects the latter and matches it to
2375 /// ARM::ABS or ARM::t2ABS machine node.
2376 bool ARMDAGToDAGISel::tryABSOp(SDNode *N){
2377  SDValue XORSrc0 = N->getOperand(0);
2378  SDValue XORSrc1 = N->getOperand(1);
2379  EVT VT = N->getValueType(0);
2380 
2381  if (Subtarget->isThumb1Only())
2382  return false;
2383 
2384  if (XORSrc0.getOpcode() != ISD::ADD || XORSrc1.getOpcode() != ISD::SRA)
2385  return false;
2386 
2387  SDValue ADDSrc0 = XORSrc0.getOperand(0);
2388  SDValue ADDSrc1 = XORSrc0.getOperand(1);
2389  SDValue SRASrc0 = XORSrc1.getOperand(0);
2390  SDValue SRASrc1 = XORSrc1.getOperand(1);
2391  ConstantSDNode *SRAConstant = dyn_cast<ConstantSDNode>(SRASrc1);
2392  EVT XType = SRASrc0.getValueType();
2393  unsigned Size = XType.getSizeInBits() - 1;
2394 
2395  if (ADDSrc1 == XORSrc1 && ADDSrc0 == SRASrc0 &&
2396  XType.isInteger() && SRAConstant != nullptr &&
2397  Size == SRAConstant->getZExtValue()) {
2398  unsigned Opcode = Subtarget->isThumb2() ? ARM::t2ABS : ARM::ABS;
2399  CurDAG->SelectNodeTo(N, Opcode, VT, ADDSrc0);
2400  return true;
2401  }
2402 
2403  return false;
2404 }
2405 
2406 /// We've got special pseudo-instructions for these
2407 void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) {
2408  unsigned Opcode;
2409  EVT MemTy = cast<MemSDNode>(N)->getMemoryVT();
2410  if (MemTy == MVT::i8)
2411  Opcode = ARM::CMP_SWAP_8;
2412  else if (MemTy == MVT::i16)
2413  Opcode = ARM::CMP_SWAP_16;
2414  else if (MemTy == MVT::i32)
2415  Opcode = ARM::CMP_SWAP_32;
2416  else
2417  llvm_unreachable("Unknown AtomicCmpSwap type");
2418 
2419  SDValue Ops[] = {N->getOperand(1), N->getOperand(2), N->getOperand(3),
2420  N->getOperand(0)};
2421  SDNode *CmpSwap = CurDAG->getMachineNode(
2422  Opcode, SDLoc(N),
2423  CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops);
2424 
2425  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
2426  MemOp[0] = cast<MemSDNode>(N)->getMemOperand();
2427  cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1);
2428 
2429  ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0));
2430  ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2));
2431  CurDAG->RemoveDeadNode(N);
2432 }
2433 
2436  unsigned FirstOne = A.getBitWidth() - A.countLeadingZeros() - 1;
2437  unsigned LastOne = A.countTrailingZeros();
2438  if (A.countPopulation() != (FirstOne - LastOne + 1))
2440  return std::make_pair(FirstOne, LastOne);
2441 }
2442 
2443 void ARMDAGToDAGISel::SelectCMPZ(SDNode *N, bool &SwitchEQNEToPLMI) {
2444  assert(N->getOpcode() == ARMISD::CMPZ);
2445  SwitchEQNEToPLMI = false;
2446 
2447  if (!Subtarget->isThumb())
2448  // FIXME: Work out whether it is profitable to do this in A32 mode - LSL and
2449  // LSR don't exist as standalone instructions - they need the barrel shifter.
2450  return;
2451 
2452  // select (cmpz (and X, C), #0) -> (LSLS X) or (LSRS X) or (LSRS (LSLS X))
2453  SDValue And = N->getOperand(0);
2454  if (!And->hasOneUse())
2455  return;
2456 
2457  SDValue Zero = N->getOperand(1);
2458  if (!isa<ConstantSDNode>(Zero) || !cast<ConstantSDNode>(Zero)->isNullValue() ||
2459  And->getOpcode() != ISD::AND)
2460  return;
2461  SDValue X = And.getOperand(0);
2462  auto C = dyn_cast<ConstantSDNode>(And.getOperand(1));
2463 
2464  if (!C || !X->hasOneUse())
2465  return;
2466  auto Range = getContiguousRangeOfSetBits(C->getAPIntValue());
2467  if (!Range)
2468  return;
2469 
2470  // There are several ways to lower this:
2471  SDNode *NewN;
2472  SDLoc dl(N);
2473 
2474  auto EmitShift = [&](unsigned Opc, SDValue Src, unsigned Imm) -> SDNode* {
2475  if (Subtarget->isThumb2()) {
2476  Opc = (Opc == ARM::tLSLri) ? ARM::t2LSLri : ARM::t2LSRri;
2477  SDValue Ops[] = { Src, CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2478  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2479  CurDAG->getRegister(0, MVT::i32) };
2480  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2481  } else {
2482  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), Src,
2483  CurDAG->getTargetConstant(Imm, dl, MVT::i32),
2484  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2485  return CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2486  }
2487  };
2488 
2489  if (Range->second == 0) {
2490  // 1. Mask includes the LSB -> Simply shift the top N bits off
2491  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2492  ReplaceNode(And.getNode(), NewN);
2493  } else if (Range->first == 31) {
2494  // 2. Mask includes the MSB -> Simply shift the bottom N bits off
2495  NewN = EmitShift(ARM::tLSRri, X, Range->second);
2496  ReplaceNode(And.getNode(), NewN);
2497  } else if (Range->first == Range->second) {
2498  // 3. Only one bit is set. We can shift this into the sign bit and use a
2499  // PL/MI comparison.
2500  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2501  ReplaceNode(And.getNode(), NewN);
2502 
2503  SwitchEQNEToPLMI = true;
2504  } else if (!Subtarget->hasV6T2Ops()) {
2505  // 4. Do a double shift to clear bottom and top bits, but only in
2506  // thumb-1 mode as in thumb-2 we can use UBFX.
2507  NewN = EmitShift(ARM::tLSLri, X, 31 - Range->first);
2508  NewN = EmitShift(ARM::tLSRri, SDValue(NewN, 0),
2509  Range->second + (31 - Range->first));
2510  ReplaceNode(And.getNode(), NewN);
2511  }
2512 
2513 }
2514 
2516  SDLoc dl(N);
2517 
2518  if (N->isMachineOpcode()) {
2519  N->setNodeId(-1);
2520  return; // Already selected.
2521  }
2522 
2523  switch (N->getOpcode()) {
2524  default: break;
2525  case ISD::WRITE_REGISTER:
2526  if (tryWriteRegister(N))
2527  return;
2528  break;
2529  case ISD::READ_REGISTER:
2530  if (tryReadRegister(N))
2531  return;
2532  break;
2533  case ISD::INLINEASM:
2534  if (tryInlineAsm(N))
2535  return;
2536  break;
2537  case ISD::XOR:
2538  // Select special operations if XOR node forms integer ABS pattern
2539  if (tryABSOp(N))
2540  return;
2541  // Other cases are autogenerated.
2542  break;
2543  case ISD::Constant: {
2544  unsigned Val = cast<ConstantSDNode>(N)->getZExtValue();
2545  // If we can't materialize the constant we need to use a literal pool
2546  if (ConstantMaterializationCost(Val) > 2) {
2547  SDValue CPIdx = CurDAG->getTargetConstantPool(
2548  ConstantInt::get(Type::getInt32Ty(*CurDAG->getContext()), Val),
2549  TLI->getPointerTy(CurDAG->getDataLayout()));
2550 
2551  SDNode *ResNode;
2552  if (Subtarget->isThumb()) {
2553  SDValue Ops[] = {
2554  CPIdx,
2555  getAL(CurDAG, dl),
2556  CurDAG->getRegister(0, MVT::i32),
2557  CurDAG->getEntryNode()
2558  };
2559  ResNode = CurDAG->getMachineNode(ARM::tLDRpci, dl, MVT::i32, MVT::Other,
2560  Ops);
2561  } else {
2562  SDValue Ops[] = {
2563  CPIdx,
2564  CurDAG->getTargetConstant(0, dl, MVT::i32),
2565  getAL(CurDAG, dl),
2566  CurDAG->getRegister(0, MVT::i32),
2567  CurDAG->getEntryNode()
2568  };
2569  ResNode = CurDAG->getMachineNode(ARM::LDRcp, dl, MVT::i32, MVT::Other,
2570  Ops);
2571  }
2572  // Annotate the Node with memory operand information so that MachineInstr
2573  // queries work properly. This e.g. gives the register allocation the
2574  // required information for rematerialization.
2575  MachineFunction& MF = CurDAG->getMachineFunction();
2577  MemOp[0] = MF.getMachineMemOperand(
2580 
2581  cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp+1);
2582 
2583  ReplaceNode(N, ResNode);
2584  return;
2585  }
2586 
2587  // Other cases are autogenerated.
2588  break;
2589  }
2590  case ISD::FrameIndex: {
2591  // Selects to ADDri FI, 0 which in turn will become ADDri SP, imm.
2592  int FI = cast<FrameIndexSDNode>(N)->getIndex();
2593  SDValue TFI = CurDAG->getTargetFrameIndex(
2594  FI, TLI->getPointerTy(CurDAG->getDataLayout()));
2595  if (Subtarget->isThumb1Only()) {
2596  // Set the alignment of the frame object to 4, to avoid having to generate
2597  // more than one ADD
2598  MachineFrameInfo &MFI = MF->getFrameInfo();
2599  if (MFI.getObjectAlignment(FI) < 4)
2600  MFI.setObjectAlignment(FI, 4);
2601  CurDAG->SelectNodeTo(N, ARM::tADDframe, MVT::i32, TFI,
2602  CurDAG->getTargetConstant(0, dl, MVT::i32));
2603  return;
2604  } else {
2605  unsigned Opc = ((Subtarget->isThumb() && Subtarget->hasThumb2()) ?
2606  ARM::t2ADDri : ARM::ADDri);
2607  SDValue Ops[] = { TFI, CurDAG->getTargetConstant(0, dl, MVT::i32),
2608  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2609  CurDAG->getRegister(0, MVT::i32) };
2610  CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops);
2611  return;
2612  }
2613  }
2614  case ISD::SRL:
2615  if (tryV6T2BitfieldExtractOp(N, false))
2616  return;
2617  break;
2619  case ISD::SRA:
2620  if (tryV6T2BitfieldExtractOp(N, true))
2621  return;
2622  break;
2623  case ISD::MUL:
2624  if (Subtarget->isThumb1Only())
2625  break;
2626  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1))) {
2627  unsigned RHSV = C->getZExtValue();
2628  if (!RHSV) break;
2629  if (isPowerOf2_32(RHSV-1)) { // 2^n+1?
2630  unsigned ShImm = Log2_32(RHSV-1);
2631  if (ShImm >= 32)
2632  break;
2633  SDValue V = N->getOperand(0);
2634  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2635  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2636  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2637  if (Subtarget->isThumb()) {
2638  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2639  CurDAG->SelectNodeTo(N, ARM::t2ADDrs, MVT::i32, Ops);
2640  return;
2641  } else {
2642  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2643  Reg0 };
2644  CurDAG->SelectNodeTo(N, ARM::ADDrsi, MVT::i32, Ops);
2645  return;
2646  }
2647  }
2648  if (isPowerOf2_32(RHSV+1)) { // 2^n-1?
2649  unsigned ShImm = Log2_32(RHSV+1);
2650  if (ShImm >= 32)
2651  break;
2652  SDValue V = N->getOperand(0);
2653  ShImm = ARM_AM::getSORegOpc(ARM_AM::lsl, ShImm);
2654  SDValue ShImmOp = CurDAG->getTargetConstant(ShImm, dl, MVT::i32);
2655  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
2656  if (Subtarget->isThumb()) {
2657  SDValue Ops[] = { V, V, ShImmOp, getAL(CurDAG, dl), Reg0, Reg0 };
2658  CurDAG->SelectNodeTo(N, ARM::t2RSBrs, MVT::i32, Ops);
2659  return;
2660  } else {
2661  SDValue Ops[] = { V, V, Reg0, ShImmOp, getAL(CurDAG, dl), Reg0,
2662  Reg0 };
2663  CurDAG->SelectNodeTo(N, ARM::RSBrsi, MVT::i32, Ops);
2664  return;
2665  }
2666  }
2667  }
2668  break;
2669  case ISD::AND: {
2670  // Check for unsigned bitfield extract
2671  if (tryV6T2BitfieldExtractOp(N, false))
2672  return;
2673 
2674  // If an immediate is used in an AND node, it is possible that the immediate
2675  // can be more optimally materialized when negated. If this is the case we
2676  // can negate the immediate and use a BIC instead.
2677  auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
2678  if (N1C && N1C->hasOneUse() && Subtarget->isThumb()) {
2679  uint32_t Imm = (uint32_t) N1C->getZExtValue();
2680 
2681  // In Thumb2 mode, an AND can take a 12-bit immediate. If this
2682  // immediate can be negated and fit in the immediate operand of
2683  // a t2BIC, don't do any manual transform here as this can be
2684  // handled by the generic ISel machinery.
2685  bool PreferImmediateEncoding =
2686  Subtarget->hasThumb2() && (is_t2_so_imm(Imm) || is_t2_so_imm_not(Imm));
2687  if (!PreferImmediateEncoding &&
2688  ConstantMaterializationCost(Imm) >
2689  ConstantMaterializationCost(~Imm)) {
2690  // The current immediate costs more to materialize than a negated
2691  // immediate, so negate the immediate and use a BIC.
2692  SDValue NewImm =
2693  CurDAG->getConstant(~N1C->getZExtValue(), dl, MVT::i32);
2694  // If the new constant didn't exist before, reposition it in the topological
2695  // ordering so it is just before N. Otherwise, don't touch its location.
2696  if (NewImm->getNodeId() == -1)
2697  CurDAG->RepositionNode(N->getIterator(), NewImm.getNode());
2698 
2699  if (!Subtarget->hasThumb2()) {
2700  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32),
2701  N->getOperand(0), NewImm, getAL(CurDAG, dl),
2702  CurDAG->getRegister(0, MVT::i32)};
2703  ReplaceNode(N, CurDAG->getMachineNode(ARM::tBIC, dl, MVT::i32, Ops));
2704  return;
2705  } else {
2706  SDValue Ops[] = {N->getOperand(0), NewImm, getAL(CurDAG, dl),
2707  CurDAG->getRegister(0, MVT::i32),
2708  CurDAG->getRegister(0, MVT::i32)};
2709  ReplaceNode(N,
2710  CurDAG->getMachineNode(ARM::t2BICrr, dl, MVT::i32, Ops));
2711  return;
2712  }
2713  }
2714  }
2715 
2716  // (and (or x, c2), c1) and top 16-bits of c1 and c2 match, lower 16-bits
2717  // of c1 are 0xffff, and lower 16-bit of c2 are 0. That is, the top 16-bits
2718  // are entirely contributed by c2 and lower 16-bits are entirely contributed
2719  // by x. That's equal to (or (and x, 0xffff), (and c1, 0xffff0000)).
2720  // Select it to: "movt x, ((c1 & 0xffff) >> 16)
2721  EVT VT = N->getValueType(0);
2722  if (VT != MVT::i32)
2723  break;
2724  unsigned Opc = (Subtarget->isThumb() && Subtarget->hasThumb2())
2725  ? ARM::t2MOVTi16
2726  : (Subtarget->hasV6T2Ops() ? ARM::MOVTi16 : 0);
2727  if (!Opc)
2728  break;
2729  SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
2730  N1C = dyn_cast<ConstantSDNode>(N1);
2731  if (!N1C)
2732  break;
2733  if (N0.getOpcode() == ISD::OR && N0.getNode()->hasOneUse()) {
2734  SDValue N2 = N0.getOperand(1);
2736  if (!N2C)
2737  break;
2738  unsigned N1CVal = N1C->getZExtValue();
2739  unsigned N2CVal = N2C->getZExtValue();
2740  if ((N1CVal & 0xffff0000U) == (N2CVal & 0xffff0000U) &&
2741  (N1CVal & 0xffffU) == 0xffffU &&
2742  (N2CVal & 0xffffU) == 0x0U) {
2743  SDValue Imm16 = CurDAG->getTargetConstant((N2CVal & 0xFFFF0000U) >> 16,
2744  dl, MVT::i32);
2745  SDValue Ops[] = { N0.getOperand(0), Imm16,
2746  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32) };
2747  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, Ops));
2748  return;
2749  }
2750  }
2751 
2752  break;
2753  }
2754  case ARMISD::UMAAL: {
2755  unsigned Opc = Subtarget->isThumb() ? ARM::t2UMAAL : ARM::UMAAL;
2756  SDValue Ops[] = { N->getOperand(0), N->getOperand(1),
2757  N->getOperand(2), N->getOperand(3),
2758  getAL(CurDAG, dl),
2759  CurDAG->getRegister(0, MVT::i32) };
2760  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, MVT::i32, Ops));
2761  return;
2762  }
2763  case ARMISD::UMLAL:{
2764  if (Subtarget->isThumb()) {
2765  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2766  N->getOperand(3), getAL(CurDAG, dl),
2767  CurDAG->getRegister(0, MVT::i32)};
2768  ReplaceNode(
2769  N, CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops));
2770  return;
2771  }else{
2772  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2773  N->getOperand(3), getAL(CurDAG, dl),
2774  CurDAG->getRegister(0, MVT::i32),
2775  CurDAG->getRegister(0, MVT::i32) };
2776  ReplaceNode(N, CurDAG->getMachineNode(
2777  Subtarget->hasV6Ops() ? ARM::UMLAL : ARM::UMLALv5, dl,
2778  MVT::i32, MVT::i32, Ops));
2779  return;
2780  }
2781  }
2782  case ARMISD::SMLAL:{
2783  if (Subtarget->isThumb()) {
2784  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2785  N->getOperand(3), getAL(CurDAG, dl),
2786  CurDAG->getRegister(0, MVT::i32)};
2787  ReplaceNode(
2788  N, CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops));
2789  return;
2790  }else{
2791  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
2792  N->getOperand(3), getAL(CurDAG, dl),
2793  CurDAG->getRegister(0, MVT::i32),
2794  CurDAG->getRegister(0, MVT::i32) };
2795  ReplaceNode(N, CurDAG->getMachineNode(
2796  Subtarget->hasV6Ops() ? ARM::SMLAL : ARM::SMLALv5, dl,
2797  MVT::i32, MVT::i32, Ops));
2798  return;
2799  }
2800  }
2801  case ARMISD::SUBE: {
2802  if (!Subtarget->hasV6Ops() || !Subtarget->hasDSP())
2803  break;
2804  // Look for a pattern to match SMMLS
2805  // (sube a, (smul_loHi a, b), (subc 0, (smul_LOhi(a, b))))
2806  if (N->getOperand(1).getOpcode() != ISD::SMUL_LOHI ||
2807  N->getOperand(2).getOpcode() != ARMISD::SUBC ||
2808  !SDValue(N, 1).use_empty())
2809  break;
2810 
2811  if (Subtarget->isThumb())
2812  assert(Subtarget->hasThumb2() &&
2813  "This pattern should not be generated for Thumb");
2814 
2815  SDValue SmulLoHi = N->getOperand(1);
2816  SDValue Subc = N->getOperand(2);
2817  auto *Zero = dyn_cast<ConstantSDNode>(Subc.getOperand(0));
2818 
2819  if (!Zero || Zero->getZExtValue() != 0 ||
2820  Subc.getOperand(1) != SmulLoHi.getValue(0) ||
2821  N->getOperand(1) != SmulLoHi.getValue(1) ||
2822  N->getOperand(2) != Subc.getValue(1))
2823  break;
2824 
2825  unsigned Opc = Subtarget->isThumb2() ? ARM::t2SMMLS : ARM::SMMLS;
2826  SDValue Ops[] = { SmulLoHi.getOperand(0), SmulLoHi.getOperand(1),
2827  N->getOperand(0), getAL(CurDAG, dl),
2828  CurDAG->getRegister(0, MVT::i32) };
2829  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops));
2830  return;
2831  }
2832  case ISD::LOAD: {
2833  if (Subtarget->isThumb() && Subtarget->hasThumb2()) {
2834  if (tryT2IndexedLoad(N))
2835  return;
2836  } else if (Subtarget->isThumb()) {
2837  if (tryT1IndexedLoad(N))
2838  return;
2839  } else if (tryARMIndexedLoad(N))
2840  return;
2841  // Other cases are autogenerated.
2842  break;
2843  }
2844  case ARMISD::BRCOND: {
2845  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2846  // Emits: (Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2847  // Pattern complexity = 6 cost = 1 size = 0
2848 
2849  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2850  // Emits: (tBcc:void (bb:Other):$dst, (imm:i32):$cc)
2851  // Pattern complexity = 6 cost = 1 size = 0
2852 
2853  // Pattern: (ARMbrcond:void (bb:Other):$dst, (imm:i32):$cc)
2854  // Emits: (t2Bcc:void (bb:Other):$dst, (imm:i32):$cc)
2855  // Pattern complexity = 6 cost = 1 size = 0
2856 
2857  unsigned Opc = Subtarget->isThumb() ?
2858  ((Subtarget->hasThumb2()) ? ARM::t2Bcc : ARM::tBcc) : ARM::Bcc;
2859  SDValue Chain = N->getOperand(0);
2860  SDValue N1 = N->getOperand(1);
2861  SDValue N2 = N->getOperand(2);
2862  SDValue N3 = N->getOperand(3);
2863  SDValue InFlag = N->getOperand(4);
2864  assert(N1.getOpcode() == ISD::BasicBlock);
2865  assert(N2.getOpcode() == ISD::Constant);
2866  assert(N3.getOpcode() == ISD::Register);
2867 
2868  unsigned CC = (unsigned) cast<ConstantSDNode>(N2)->getZExtValue();
2869 
2870  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2871  bool SwitchEQNEToPLMI;
2872  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2873  InFlag = N->getOperand(4);
2874 
2875  if (SwitchEQNEToPLMI) {
2876  switch ((ARMCC::CondCodes)CC) {
2877  default: llvm_unreachable("CMPZ must be either NE or EQ!");
2878  case ARMCC::NE:
2879  CC = (unsigned)ARMCC::MI;
2880  break;
2881  case ARMCC::EQ:
2882  CC = (unsigned)ARMCC::PL;
2883  break;
2884  }
2885  }
2886  }
2887 
2888  SDValue Tmp2 = CurDAG->getTargetConstant(CC, dl, MVT::i32);
2889  SDValue Ops[] = { N1, Tmp2, N3, Chain, InFlag };
2890  SDNode *ResNode = CurDAG->getMachineNode(Opc, dl, MVT::Other,
2891  MVT::Glue, Ops);
2892  Chain = SDValue(ResNode, 0);
2893  if (N->getNumValues() == 2) {
2894  InFlag = SDValue(ResNode, 1);
2895  ReplaceUses(SDValue(N, 1), InFlag);
2896  }
2897  ReplaceUses(SDValue(N, 0),
2898  SDValue(Chain.getNode(), Chain.getResNo()));
2899  CurDAG->RemoveDeadNode(N);
2900  return;
2901  }
2902 
2903  case ARMISD::CMPZ: {
2904  // select (CMPZ X, #-C) -> (CMPZ (ADDS X, #C), #0)
2905  // This allows us to avoid materializing the expensive negative constant.
2906  // The CMPZ #0 is useless and will be peepholed away but we need to keep it
2907  // for its glue output.
2908  SDValue X = N->getOperand(0);
2909  auto *C = dyn_cast<ConstantSDNode>(N->getOperand(1).getNode());
2910  if (C && C->getSExtValue() < 0 && Subtarget->isThumb()) {
2911  int64_t Addend = -C->getSExtValue();
2912 
2913  SDNode *Add = nullptr;
2914  // ADDS can be better than CMN if the immediate fits in a
2915  // 16-bit ADDS, which means either [0,256) for tADDi8 or [0,8) for tADDi3.
2916  // Outside that range we can just use a CMN which is 32-bit but has a
2917  // 12-bit immediate range.
2918  if (Addend < 1<<8) {
2919  if (Subtarget->isThumb2()) {
2920  SDValue Ops[] = { X, CurDAG->getTargetConstant(Addend, dl, MVT::i32),
2921  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32),
2922  CurDAG->getRegister(0, MVT::i32) };
2923  Add = CurDAG->getMachineNode(ARM::t2ADDri, dl, MVT::i32, Ops);
2924  } else {
2925  unsigned Opc = (Addend < 1<<3) ? ARM::tADDi3 : ARM::tADDi8;
2926  SDValue Ops[] = {CurDAG->getRegister(ARM::CPSR, MVT::i32), X,
2927  CurDAG->getTargetConstant(Addend, dl, MVT::i32),
2928  getAL(CurDAG, dl), CurDAG->getRegister(0, MVT::i32)};
2929  Add = CurDAG->getMachineNode(Opc, dl, MVT::i32, Ops);
2930  }
2931  }
2932  if (Add) {
2933  SDValue Ops2[] = {SDValue(Add, 0), CurDAG->getConstant(0, dl, MVT::i32)};
2934  CurDAG->MorphNodeTo(N, ARMISD::CMPZ, CurDAG->getVTList(MVT::Glue), Ops2);
2935  }
2936  }
2937  // Other cases are autogenerated.
2938  break;
2939  }
2940 
2941  case ARMISD::CMOV: {
2942  SDValue InFlag = N->getOperand(4);
2943 
2944  if (InFlag.getOpcode() == ARMISD::CMPZ) {
2945  bool SwitchEQNEToPLMI;
2946  SelectCMPZ(InFlag.getNode(), SwitchEQNEToPLMI);
2947 
2948  if (SwitchEQNEToPLMI) {
2949  SDValue ARMcc = N->getOperand(2);
2950  ARMCC::CondCodes CC =
2951  (ARMCC::CondCodes)cast<ConstantSDNode>(ARMcc)->getZExtValue();
2952 
2953  switch (CC) {
2954  default: llvm_unreachable("CMPZ must be either NE or EQ!");
2955  case ARMCC::NE:
2956  CC = ARMCC::MI;
2957  break;
2958  case ARMCC::EQ:
2959  CC = ARMCC::PL;
2960  break;
2961  }
2962  SDValue NewARMcc = CurDAG->getConstant((unsigned)CC, dl, MVT::i32);
2963  SDValue Ops[] = {N->getOperand(0), N->getOperand(1), NewARMcc,
2964  N->getOperand(3), N->getOperand(4)};
2965  CurDAG->MorphNodeTo(N, ARMISD::CMOV, N->getVTList(), Ops);
2966  }
2967 
2968  }
2969  // Other cases are autogenerated.
2970  break;
2971  }
2972 
2973  case ARMISD::VZIP: {
2974  unsigned Opc = 0;
2975  EVT VT = N->getValueType(0);
2976  switch (VT.getSimpleVT().SimpleTy) {
2977  default: return;
2978  case MVT::v8i8: Opc = ARM::VZIPd8; break;
2979  case MVT::v4i16: Opc = ARM::VZIPd16; break;
2980  case MVT::v2f32:
2981  // vzip.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
2982  case MVT::v2i32: Opc = ARM::VTRNd32; break;
2983  case MVT::v16i8: Opc = ARM::VZIPq8; break;
2984  case MVT::v8i16: Opc = ARM::VZIPq16; break;
2985  case MVT::v4f32:
2986  case MVT::v4i32: Opc = ARM::VZIPq32; break;
2987  }
2988  SDValue Pred = getAL(CurDAG, dl);
2989  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
2990  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
2991  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
2992  return;
2993  }
2994  case ARMISD::VUZP: {
2995  unsigned Opc = 0;
2996  EVT VT = N->getValueType(0);
2997  switch (VT.getSimpleVT().SimpleTy) {
2998  default: return;
2999  case MVT::v8i8: Opc = ARM::VUZPd8; break;
3000  case MVT::v4i16: Opc = ARM::VUZPd16; break;
3001  case MVT::v2f32:
3002  // vuzp.32 Dd, Dm is a pseudo-instruction expanded to vtrn.32 Dd, Dm.
3003  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3004  case MVT::v16i8: Opc = ARM::VUZPq8; break;
3005  case MVT::v8i16: Opc = ARM::VUZPq16; break;
3006  case MVT::v4f32:
3007  case MVT::v4i32: Opc = ARM::VUZPq32; break;
3008  }
3009  SDValue Pred = getAL(CurDAG, dl);
3010  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3011  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3012  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3013  return;
3014  }
3015  case ARMISD::VTRN: {
3016  unsigned Opc = 0;
3017  EVT VT = N->getValueType(0);
3018  switch (VT.getSimpleVT().SimpleTy) {
3019  default: return;
3020  case MVT::v8i8: Opc = ARM::VTRNd8; break;
3021  case MVT::v4i16: Opc = ARM::VTRNd16; break;
3022  case MVT::v2f32:
3023  case MVT::v2i32: Opc = ARM::VTRNd32; break;
3024  case MVT::v16i8: Opc = ARM::VTRNq8; break;
3025  case MVT::v8i16: Opc = ARM::VTRNq16; break;
3026  case MVT::v4f32:
3027  case MVT::v4i32: Opc = ARM::VTRNq32; break;
3028  }
3029  SDValue Pred = getAL(CurDAG, dl);
3030  SDValue PredReg = CurDAG->getRegister(0, MVT::i32);
3031  SDValue Ops[] = { N->getOperand(0), N->getOperand(1), Pred, PredReg };
3032  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, VT, VT, Ops));
3033  return;
3034  }
3035  case ARMISD::BUILD_VECTOR: {
3036  EVT VecVT = N->getValueType(0);
3037  EVT EltVT = VecVT.getVectorElementType();
3038  unsigned NumElts = VecVT.getVectorNumElements();
3039  if (EltVT == MVT::f64) {
3040  assert(NumElts == 2 && "unexpected type for BUILD_VECTOR");
3041  ReplaceNode(
3042  N, createDRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3043  return;
3044  }
3045  assert(EltVT == MVT::f32 && "unexpected type for BUILD_VECTOR");
3046  if (NumElts == 2) {
3047  ReplaceNode(
3048  N, createSRegPairNode(VecVT, N->getOperand(0), N->getOperand(1)));
3049  return;
3050  }
3051  assert(NumElts == 4 && "unexpected type for BUILD_VECTOR");
3052  ReplaceNode(N,
3053  createQuadSRegsNode(VecVT, N->getOperand(0), N->getOperand(1),
3054  N->getOperand(2), N->getOperand(3)));
3055  return;
3056  }
3057 
3058  case ARMISD::VLD1DUP: {
3059  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8, ARM::VLD1DUPd16,
3060  ARM::VLD1DUPd32 };
3061  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8, ARM::VLD1DUPq16,
3062  ARM::VLD1DUPq32 };
3063  SelectVLDDup(N, false, 1, DOpcodes, QOpcodes);
3064  return;
3065  }
3066 
3067  case ARMISD::VLD2DUP: {
3068  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8, ARM::VLD2DUPd16,
3069  ARM::VLD2DUPd32 };
3070  SelectVLDDup(N, false, 2, Opcodes);
3071  return;
3072  }
3073 
3074  case ARMISD::VLD3DUP: {
3075  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo,
3076  ARM::VLD3DUPd16Pseudo,
3077  ARM::VLD3DUPd32Pseudo };
3078  SelectVLDDup(N, false, 3, Opcodes);
3079  return;
3080  }
3081 
3082  case ARMISD::VLD4DUP: {
3083  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo,
3084  ARM::VLD4DUPd16Pseudo,
3085  ARM::VLD4DUPd32Pseudo };
3086  SelectVLDDup(N, false, 4, Opcodes);
3087  return;
3088  }
3089 
3090  case ARMISD::VLD1DUP_UPD: {
3091  static const uint16_t DOpcodes[] = { ARM::VLD1DUPd8wb_fixed,
3092  ARM::VLD1DUPd16wb_fixed,
3093  ARM::VLD1DUPd32wb_fixed };
3094  static const uint16_t QOpcodes[] = { ARM::VLD1DUPq8wb_fixed,
3095  ARM::VLD1DUPq16wb_fixed,
3096  ARM::VLD1DUPq32wb_fixed };
3097  SelectVLDDup(N, true, 1, DOpcodes, QOpcodes);
3098  return;
3099  }
3100 
3101  case ARMISD::VLD2DUP_UPD: {
3102  static const uint16_t Opcodes[] = { ARM::VLD2DUPd8wb_fixed,
3103  ARM::VLD2DUPd16wb_fixed,
3104  ARM::VLD2DUPd32wb_fixed };
3105  SelectVLDDup(N, true, 2, Opcodes);
3106  return;
3107  }
3108 
3109  case ARMISD::VLD3DUP_UPD: {
3110  static const uint16_t Opcodes[] = { ARM::VLD3DUPd8Pseudo_UPD,
3111  ARM::VLD3DUPd16Pseudo_UPD,
3112  ARM::VLD3DUPd32Pseudo_UPD };
3113  SelectVLDDup(N, true, 3, Opcodes);
3114  return;
3115  }
3116 
3117  case ARMISD::VLD4DUP_UPD: {
3118  static const uint16_t Opcodes[] = { ARM::VLD4DUPd8Pseudo_UPD,
3119  ARM::VLD4DUPd16Pseudo_UPD,
3120  ARM::VLD4DUPd32Pseudo_UPD };
3121  SelectVLDDup(N, true, 4, Opcodes);
3122  return;
3123  }
3124 
3125  case ARMISD::VLD1_UPD: {
3126  static const uint16_t DOpcodes[] = { ARM::VLD1d8wb_fixed,
3127  ARM::VLD1d16wb_fixed,
3128  ARM::VLD1d32wb_fixed,
3129  ARM::VLD1d64wb_fixed };
3130  static const uint16_t QOpcodes[] = { ARM::VLD1q8wb_fixed,
3131  ARM::VLD1q16wb_fixed,
3132  ARM::VLD1q32wb_fixed,
3133  ARM::VLD1q64wb_fixed };
3134  SelectVLD(N, true, 1, DOpcodes, QOpcodes, nullptr);
3135  return;
3136  }
3137 
3138  case ARMISD::VLD2_UPD: {
3139  static const uint16_t DOpcodes[] = { ARM::VLD2d8wb_fixed,
3140  ARM::VLD2d16wb_fixed,
3141  ARM::VLD2d32wb_fixed,
3142  ARM::VLD1q64wb_fixed};
3143  static const uint16_t QOpcodes[] = { ARM::VLD2q8PseudoWB_fixed,
3144  ARM::VLD2q16PseudoWB_fixed,
3145  ARM::VLD2q32PseudoWB_fixed };
3146  SelectVLD(N, true, 2, DOpcodes, QOpcodes, nullptr);
3147  return;
3148  }
3149 
3150  case ARMISD::VLD3_UPD: {
3151  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo_UPD,
3152  ARM::VLD3d16Pseudo_UPD,
3153  ARM::VLD3d32Pseudo_UPD,
3154  ARM::VLD1d64TPseudoWB_fixed};
3155  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3156  ARM::VLD3q16Pseudo_UPD,
3157  ARM::VLD3q32Pseudo_UPD };
3158  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo_UPD,
3159  ARM::VLD3q16oddPseudo_UPD,
3160  ARM::VLD3q32oddPseudo_UPD };
3161  SelectVLD(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3162  return;
3163  }
3164 
3165  case ARMISD::VLD4_UPD: {
3166  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo_UPD,
3167  ARM::VLD4d16Pseudo_UPD,
3168  ARM::VLD4d32Pseudo_UPD,
3169  ARM::VLD1d64QPseudoWB_fixed};
3170  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3171  ARM::VLD4q16Pseudo_UPD,
3172  ARM::VLD4q32Pseudo_UPD };
3173  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo_UPD,
3174  ARM::VLD4q16oddPseudo_UPD,
3175  ARM::VLD4q32oddPseudo_UPD };
3176  SelectVLD(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3177  return;
3178  }
3179 
3180  case ARMISD::VLD2LN_UPD: {
3181  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo_UPD,
3182  ARM::VLD2LNd16Pseudo_UPD,
3183  ARM::VLD2LNd32Pseudo_UPD };
3184  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo_UPD,
3185  ARM::VLD2LNq32Pseudo_UPD };
3186  SelectVLDSTLane(N, true, true, 2, DOpcodes, QOpcodes);
3187  return;
3188  }
3189 
3190  case ARMISD::VLD3LN_UPD: {
3191  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo_UPD,
3192  ARM::VLD3LNd16Pseudo_UPD,
3193  ARM::VLD3LNd32Pseudo_UPD };
3194  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo_UPD,
3195  ARM::VLD3LNq32Pseudo_UPD };
3196  SelectVLDSTLane(N, true, true, 3, DOpcodes, QOpcodes);
3197  return;
3198  }
3199 
3200  case ARMISD::VLD4LN_UPD: {
3201  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo_UPD,
3202  ARM::VLD4LNd16Pseudo_UPD,
3203  ARM::VLD4LNd32Pseudo_UPD };
3204  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo_UPD,
3205  ARM::VLD4LNq32Pseudo_UPD };
3206  SelectVLDSTLane(N, true, true, 4, DOpcodes, QOpcodes);
3207  return;
3208  }
3209 
3210  case ARMISD::VST1_UPD: {
3211  static const uint16_t DOpcodes[] = { ARM::VST1d8wb_fixed,
3212  ARM::VST1d16wb_fixed,
3213  ARM::VST1d32wb_fixed,
3214  ARM::VST1d64wb_fixed };
3215  static const uint16_t QOpcodes[] = { ARM::VST1q8wb_fixed,
3216  ARM::VST1q16wb_fixed,
3217  ARM::VST1q32wb_fixed,
3218  ARM::VST1q64wb_fixed };
3219  SelectVST(N, true, 1, DOpcodes, QOpcodes, nullptr);
3220  return;
3221  }
3222 
3223  case ARMISD::VST2_UPD: {
3224  static const uint16_t DOpcodes[] = { ARM::VST2d8wb_fixed,
3225  ARM::VST2d16wb_fixed,
3226  ARM::VST2d32wb_fixed,
3227  ARM::VST1q64wb_fixed};
3228  static const uint16_t QOpcodes[] = { ARM::VST2q8PseudoWB_fixed,
3229  ARM::VST2q16PseudoWB_fixed,
3230  ARM::VST2q32PseudoWB_fixed };
3231  SelectVST(N, true, 2, DOpcodes, QOpcodes, nullptr);
3232  return;
3233  }
3234 
3235  case ARMISD::VST3_UPD: {
3236  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo_UPD,
3237  ARM::VST3d16Pseudo_UPD,
3238  ARM::VST3d32Pseudo_UPD,
3239  ARM::VST1d64TPseudoWB_fixed};
3240  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3241  ARM::VST3q16Pseudo_UPD,
3242  ARM::VST3q32Pseudo_UPD };
3243  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo_UPD,
3244  ARM::VST3q16oddPseudo_UPD,
3245  ARM::VST3q32oddPseudo_UPD };
3246  SelectVST(N, true, 3, DOpcodes, QOpcodes0, QOpcodes1);
3247  return;
3248  }
3249 
3250  case ARMISD::VST4_UPD: {
3251  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo_UPD,
3252  ARM::VST4d16Pseudo_UPD,
3253  ARM::VST4d32Pseudo_UPD,
3254  ARM::VST1d64QPseudoWB_fixed};
3255  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3256  ARM::VST4q16Pseudo_UPD,
3257  ARM::VST4q32Pseudo_UPD };
3258  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo_UPD,
3259  ARM::VST4q16oddPseudo_UPD,
3260  ARM::VST4q32oddPseudo_UPD };
3261  SelectVST(N, true, 4, DOpcodes, QOpcodes0, QOpcodes1);
3262  return;
3263  }
3264 
3265  case ARMISD::VST2LN_UPD: {
3266  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo_UPD,
3267  ARM::VST2LNd16Pseudo_UPD,
3268  ARM::VST2LNd32Pseudo_UPD };
3269  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo_UPD,
3270  ARM::VST2LNq32Pseudo_UPD };
3271  SelectVLDSTLane(N, false, true, 2, DOpcodes, QOpcodes);
3272  return;
3273  }
3274 
3275  case ARMISD::VST3LN_UPD: {
3276  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo_UPD,
3277  ARM::VST3LNd16Pseudo_UPD,
3278  ARM::VST3LNd32Pseudo_UPD };
3279  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo_UPD,
3280  ARM::VST3LNq32Pseudo_UPD };
3281  SelectVLDSTLane(N, false, true, 3, DOpcodes, QOpcodes);
3282  return;
3283  }
3284 
3285  case ARMISD::VST4LN_UPD: {
3286  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo_UPD,
3287  ARM::VST4LNd16Pseudo_UPD,
3288  ARM::VST4LNd32Pseudo_UPD };
3289  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo_UPD,
3290  ARM::VST4LNq32Pseudo_UPD };
3291  SelectVLDSTLane(N, false, true, 4, DOpcodes, QOpcodes);
3292  return;
3293  }
3294 
3295  case ISD::INTRINSIC_VOID:
3296  case ISD::INTRINSIC_W_CHAIN: {
3297  unsigned IntNo = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
3298  switch (IntNo) {
3299  default:
3300  break;
3301 
3302  case Intrinsic::arm_mrrc:
3303  case Intrinsic::arm_mrrc2: {
3304  SDLoc dl(N);
3305  SDValue Chain = N->getOperand(0);
3306  unsigned Opc;
3307 
3308  if (Subtarget->isThumb())
3309  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::t2MRRC : ARM::t2MRRC2);
3310  else
3311  Opc = (IntNo == Intrinsic::arm_mrrc ? ARM::MRRC : ARM::MRRC2);
3312 
3314  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(2))->getZExtValue(), dl)); /* coproc */
3315  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(3))->getZExtValue(), dl)); /* opc */
3316  Ops.push_back(getI32Imm(cast<ConstantSDNode>(N->getOperand(4))->getZExtValue(), dl)); /* CRm */
3317 
3318  // The mrrc2 instruction in ARM doesn't allow predicates, the top 4 bits of the encoded
3319  // instruction will always be '1111' but it is possible in assembly language to specify
3320  // AL as a predicate to mrrc2 but it doesn't make any difference to the encoded instruction.
3321  if (Opc != ARM::MRRC2) {
3322  Ops.push_back(getAL(CurDAG, dl));
3323  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3324  }
3325 
3326  Ops.push_back(Chain);
3327 
3328  // Writes to two registers.
3329  const EVT RetType[] = {MVT::i32, MVT::i32, MVT::Other};
3330 
3331  ReplaceNode(N, CurDAG->getMachineNode(Opc, dl, RetType, Ops));
3332  return;
3333  }
3334  case Intrinsic::arm_ldaexd:
3335  case Intrinsic::arm_ldrexd: {
3336  SDLoc dl(N);
3337  SDValue Chain = N->getOperand(0);
3338  SDValue MemAddr = N->getOperand(2);
3339  bool isThumb = Subtarget->isThumb() && Subtarget->hasV8MBaselineOps();
3340 
3341  bool IsAcquire = IntNo == Intrinsic::arm_ldaexd;
3342  unsigned NewOpc = isThumb ? (IsAcquire ? ARM::t2LDAEXD : ARM::t2LDREXD)
3343  : (IsAcquire ? ARM::LDAEXD : ARM::LDREXD);
3344 
3345  // arm_ldrexd returns a i64 value in {i32, i32}
3346  std::vector<EVT> ResTys;
3347  if (isThumb) {
3348  ResTys.push_back(MVT::i32);
3349  ResTys.push_back(MVT::i32);
3350  } else
3351  ResTys.push_back(MVT::Untyped);
3352  ResTys.push_back(MVT::Other);
3353 
3354  // Place arguments in the right order.
3355  SDValue Ops[] = {MemAddr, getAL(CurDAG, dl),
3356  CurDAG->getRegister(0, MVT::i32), Chain};
3357  SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3358  // Transfer memoperands.
3359  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3360  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3361  cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1);
3362 
3363  // Remap uses.
3364  SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1);
3365  if (!SDValue(N, 0).use_empty()) {
3366  SDValue Result;
3367  if (isThumb)
3368  Result = SDValue(Ld, 0);
3369  else {
3370  SDValue SubRegIdx =
3371  CurDAG->getTargetConstant(ARM::gsub_0, dl, MVT::i32);
3372  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3373  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3374  Result = SDValue(ResNode,0);
3375  }
3376  ReplaceUses(SDValue(N, 0), Result);
3377  }
3378  if (!SDValue(N, 1).use_empty()) {
3379  SDValue Result;
3380  if (isThumb)
3381  Result = SDValue(Ld, 1);
3382  else {
3383  SDValue SubRegIdx =
3384  CurDAG->getTargetConstant(ARM::gsub_1, dl, MVT::i32);
3385  SDNode *ResNode = CurDAG->getMachineNode(TargetOpcode::EXTRACT_SUBREG,
3386  dl, MVT::i32, SDValue(Ld, 0), SubRegIdx);
3387  Result = SDValue(ResNode,0);
3388  }
3389  ReplaceUses(SDValue(N, 1), Result);
3390  }
3391  ReplaceUses(SDValue(N, 2), OutChain);
3392  CurDAG->RemoveDeadNode(N);
3393  return;
3394  }
3395  case Intrinsic::arm_stlexd:
3396  case Intrinsic::arm_strexd: {
3397  SDLoc dl(N);
3398  SDValue Chain = N->getOperand(0);
3399  SDValue Val0 = N->getOperand(2);
3400  SDValue Val1 = N->getOperand(3);
3401  SDValue MemAddr = N->getOperand(4);
3402 
3403  // Store exclusive double return a i32 value which is the return status
3404  // of the issued store.
3405  const EVT ResTys[] = {MVT::i32, MVT::Other};
3406 
3407  bool isThumb = Subtarget->isThumb() && Subtarget->hasThumb2();
3408  // Place arguments in the right order.
3410  if (isThumb) {
3411  Ops.push_back(Val0);
3412  Ops.push_back(Val1);
3413  } else
3414  // arm_strexd uses GPRPair.
3415  Ops.push_back(SDValue(createGPRPairNode(MVT::Untyped, Val0, Val1), 0));
3416  Ops.push_back(MemAddr);
3417  Ops.push_back(getAL(CurDAG, dl));
3418  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3419  Ops.push_back(Chain);
3420 
3421  bool IsRelease = IntNo == Intrinsic::arm_stlexd;
3422  unsigned NewOpc = isThumb ? (IsRelease ? ARM::t2STLEXD : ARM::t2STREXD)
3423  : (IsRelease ? ARM::STLEXD : ARM::STREXD);
3424 
3425  SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops);
3426  // Transfer memoperands.
3427  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
3428  MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand();
3429  cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1);
3430 
3431  ReplaceNode(N, St);
3432  return;
3433  }
3434 
3435  case Intrinsic::arm_neon_vld1: {
3436  static const uint16_t DOpcodes[] = { ARM::VLD1d8, ARM::VLD1d16,
3437  ARM::VLD1d32, ARM::VLD1d64 };
3438  static const uint16_t QOpcodes[] = { ARM::VLD1q8, ARM::VLD1q16,
3439  ARM::VLD1q32, ARM::VLD1q64};
3440  SelectVLD(N, false, 1, DOpcodes, QOpcodes, nullptr);
3441  return;
3442  }
3443 
3444  case Intrinsic::arm_neon_vld2: {
3445  static const uint16_t DOpcodes[] = { ARM::VLD2d8, ARM::VLD2d16,
3446  ARM::VLD2d32, ARM::VLD1q64 };
3447  static const uint16_t QOpcodes[] = { ARM::VLD2q8Pseudo, ARM::VLD2q16Pseudo,
3448  ARM::VLD2q32Pseudo };
3449  SelectVLD(N, false, 2, DOpcodes, QOpcodes, nullptr);
3450  return;
3451  }
3452 
3453  case Intrinsic::arm_neon_vld3: {
3454  static const uint16_t DOpcodes[] = { ARM::VLD3d8Pseudo,
3455  ARM::VLD3d16Pseudo,
3456  ARM::VLD3d32Pseudo,
3457  ARM::VLD1d64TPseudo };
3458  static const uint16_t QOpcodes0[] = { ARM::VLD3q8Pseudo_UPD,
3459  ARM::VLD3q16Pseudo_UPD,
3460  ARM::VLD3q32Pseudo_UPD };
3461  static const uint16_t QOpcodes1[] = { ARM::VLD3q8oddPseudo,
3462  ARM::VLD3q16oddPseudo,
3463  ARM::VLD3q32oddPseudo };
3464  SelectVLD(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3465  return;
3466  }
3467 
3468  case Intrinsic::arm_neon_vld4: {
3469  static const uint16_t DOpcodes[] = { ARM::VLD4d8Pseudo,
3470  ARM::VLD4d16Pseudo,
3471  ARM::VLD4d32Pseudo,
3472  ARM::VLD1d64QPseudo };
3473  static const uint16_t QOpcodes0[] = { ARM::VLD4q8Pseudo_UPD,
3474  ARM::VLD4q16Pseudo_UPD,
3475  ARM::VLD4q32Pseudo_UPD };
3476  static const uint16_t QOpcodes1[] = { ARM::VLD4q8oddPseudo,
3477  ARM::VLD4q16oddPseudo,
3478  ARM::VLD4q32oddPseudo };
3479  SelectVLD(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3480  return;
3481  }
3482 
3483  case Intrinsic::arm_neon_vld2lane: {
3484  static const uint16_t DOpcodes[] = { ARM::VLD2LNd8Pseudo,
3485  ARM::VLD2LNd16Pseudo,
3486  ARM::VLD2LNd32Pseudo };
3487  static const uint16_t QOpcodes[] = { ARM::VLD2LNq16Pseudo,
3488  ARM::VLD2LNq32Pseudo };
3489  SelectVLDSTLane(N, true, false, 2, DOpcodes, QOpcodes);
3490  return;
3491  }
3492 
3493  case Intrinsic::arm_neon_vld3lane: {
3494  static const uint16_t DOpcodes[] = { ARM::VLD3LNd8Pseudo,
3495  ARM::VLD3LNd16Pseudo,
3496  ARM::VLD3LNd32Pseudo };
3497  static const uint16_t QOpcodes[] = { ARM::VLD3LNq16Pseudo,
3498  ARM::VLD3LNq32Pseudo };
3499  SelectVLDSTLane(N, true, false, 3, DOpcodes, QOpcodes);
3500  return;
3501  }
3502 
3503  case Intrinsic::arm_neon_vld4lane: {
3504  static const uint16_t DOpcodes[] = { ARM::VLD4LNd8Pseudo,
3505  ARM::VLD4LNd16Pseudo,
3506  ARM::VLD4LNd32Pseudo };
3507  static const uint16_t QOpcodes[] = { ARM::VLD4LNq16Pseudo,
3508  ARM::VLD4LNq32Pseudo };
3509  SelectVLDSTLane(N, true, false, 4, DOpcodes, QOpcodes);
3510  return;
3511  }
3512 
3513  case Intrinsic::arm_neon_vst1: {
3514  static const uint16_t DOpcodes[] = { ARM::VST1d8, ARM::VST1d16,
3515  ARM::VST1d32, ARM::VST1d64 };
3516  static const uint16_t QOpcodes[] = { ARM::VST1q8, ARM::VST1q16,
3517  ARM::VST1q32, ARM::VST1q64 };
3518  SelectVST(N, false, 1, DOpcodes, QOpcodes, nullptr);
3519  return;
3520  }
3521 
3522  case Intrinsic::arm_neon_vst2: {
3523  static const uint16_t DOpcodes[] = { ARM::VST2d8, ARM::VST2d16,
3524  ARM::VST2d32, ARM::VST1q64 };
3525  static const uint16_t QOpcodes[] = { ARM::VST2q8Pseudo, ARM::VST2q16Pseudo,
3526  ARM::VST2q32Pseudo };
3527  SelectVST(N, false, 2, DOpcodes, QOpcodes, nullptr);
3528  return;
3529  }
3530 
3531  case Intrinsic::arm_neon_vst3: {
3532  static const uint16_t DOpcodes[] = { ARM::VST3d8Pseudo,
3533  ARM::VST3d16Pseudo,
3534  ARM::VST3d32Pseudo,
3535  ARM::VST1d64TPseudo };
3536  static const uint16_t QOpcodes0[] = { ARM::VST3q8Pseudo_UPD,
3537  ARM::VST3q16Pseudo_UPD,
3538  ARM::VST3q32Pseudo_UPD };
3539  static const uint16_t QOpcodes1[] = { ARM::VST3q8oddPseudo,
3540  ARM::VST3q16oddPseudo,
3541  ARM::VST3q32oddPseudo };
3542  SelectVST(N, false, 3, DOpcodes, QOpcodes0, QOpcodes1);
3543  return;
3544  }
3545 
3546  case Intrinsic::arm_neon_vst4: {
3547  static const uint16_t DOpcodes[] = { ARM::VST4d8Pseudo,
3548  ARM::VST4d16Pseudo,
3549  ARM::VST4d32Pseudo,
3550  ARM::VST1d64QPseudo };
3551  static const uint16_t QOpcodes0[] = { ARM::VST4q8Pseudo_UPD,
3552  ARM::VST4q16Pseudo_UPD,
3553  ARM::VST4q32Pseudo_UPD };
3554  static const uint16_t QOpcodes1[] = { ARM::VST4q8oddPseudo,
3555  ARM::VST4q16oddPseudo,
3556  ARM::VST4q32oddPseudo };
3557  SelectVST(N, false, 4, DOpcodes, QOpcodes0, QOpcodes1);
3558  return;
3559  }
3560 
3561  case Intrinsic::arm_neon_vst2lane: {
3562  static const uint16_t DOpcodes[] = { ARM::VST2LNd8Pseudo,
3563  ARM::VST2LNd16Pseudo,
3564  ARM::VST2LNd32Pseudo };
3565  static const uint16_t QOpcodes[] = { ARM::VST2LNq16Pseudo,
3566  ARM::VST2LNq32Pseudo };
3567  SelectVLDSTLane(N, false, false, 2, DOpcodes, QOpcodes);
3568  return;
3569  }
3570 
3571  case Intrinsic::arm_neon_vst3lane: {
3572  static const uint16_t DOpcodes[] = { ARM::VST3LNd8Pseudo,
3573  ARM::VST3LNd16Pseudo,
3574  ARM::VST3LNd32Pseudo };
3575  static const uint16_t QOpcodes[] = { ARM::VST3LNq16Pseudo,
3576  ARM::VST3LNq32Pseudo };
3577  SelectVLDSTLane(N, false, false, 3, DOpcodes, QOpcodes);
3578  return;
3579  }
3580 
3581  case Intrinsic::arm_neon_vst4lane: {
3582  static const uint16_t DOpcodes[] = { ARM::VST4LNd8Pseudo,
3583  ARM::VST4LNd16Pseudo,
3584  ARM::VST4LNd32Pseudo };
3585  static const uint16_t QOpcodes[] = { ARM::VST4LNq16Pseudo,
3586  ARM::VST4LNq32Pseudo };
3587  SelectVLDSTLane(N, false, false, 4, DOpcodes, QOpcodes);
3588  return;
3589  }
3590  }
3591  break;
3592  }
3593 
3594  case ISD::ATOMIC_CMP_SWAP:
3595  SelectCMP_SWAP(N);
3596  return;
3597  }
3598 
3599  SelectCode(N);
3600 }
3601 
3602 // Inspect a register string of the form
3603 // cp<coprocessor>:<opc1>:c<CRn>:c<CRm>:<opc2> (32bit) or
3604 // cp<coprocessor>:<opc1>:c<CRm> (64bit) inspect the fields of the string
3605 // and obtain the integer operands from them, adding these operands to the
3606 // provided vector.
3608  SelectionDAG *CurDAG,
3609  const SDLoc &DL,
3610  std::vector<SDValue> &Ops) {
3612  RegString.split(Fields, ':');
3613 
3614  if (Fields.size() > 1) {
3615  bool AllIntFields = true;
3616 
3617  for (StringRef Field : Fields) {
3618  // Need to trim out leading 'cp' characters and get the integer field.
3619  unsigned IntField;
3620  AllIntFields &= !Field.trim("CPcp").getAsInteger(10, IntField);
3621  Ops.push_back(CurDAG->getTargetConstant(IntField, DL, MVT::i32));
3622  }
3623 
3624  assert(AllIntFields &&
3625  "Unexpected non-integer value in special register string.");
3626  }
3627 }
3628 
3629 // Maps a Banked Register string to its mask value. The mask value returned is
3630 // for use in the MRSbanked / MSRbanked instruction nodes as the Banked Register
3631 // mask operand, which expresses which register is to be used, e.g. r8, and in
3632 // which mode it is to be used, e.g. usr. Returns -1 to signify that the string
3633 // was invalid.
3634 static inline int getBankedRegisterMask(StringRef RegString) {
3635  auto TheReg = ARMBankedReg::lookupBankedRegByName(RegString.lower());
3636  if (!TheReg)
3637  return -1;
3638  return TheReg->Encoding;
3639 }
3640 
3641 // The flags here are common to those allowed for apsr in the A class cores and
3642 // those allowed for the special registers in the M class cores. Returns a
3643 // value representing which flags were present, -1 if invalid.
3644 static inline int getMClassFlagsMask(StringRef Flags) {
3645  return StringSwitch<int>(Flags)
3646  .Case("", 0x2) // no flags means nzcvq for psr registers, and 0x2 is
3647  // correct when flags are not permitted
3648  .Case("g", 0x1)
3649  .Case("nzcvq", 0x2)
3650  .Case("nzcvqg", 0x3)
3651  .Default(-1);
3652 }
3653 
3654 // Maps MClass special registers string to its value for use in the
3655 // t2MRS_M/t2MSR_M instruction nodes as the SYSm value operand.
3656 // Returns -1 to signify that the string was invalid.
3657 static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget) {
3658  auto TheReg = ARMSysReg::lookupMClassSysRegByName(Reg);
3659  const FeatureBitset &FeatureBits = Subtarget->getFeatureBits();
3660  if (!TheReg || !TheReg->hasRequiredFeatures(FeatureBits))
3661  return -1;
3662  return (int)(TheReg->Encoding & 0xFFF); // SYSm value
3663 }
3664 
3666  // The mask operand contains the special register (R Bit) in bit 4, whether
3667  // the register is spsr (R bit is 1) or one of cpsr/apsr (R bit is 0), and
3668  // bits 3-0 contains the fields to be accessed in the special register, set by
3669  // the flags provided with the register.
3670  int Mask = 0;
3671  if (Reg == "apsr") {
3672  // The flags permitted for apsr are the same flags that are allowed in
3673  // M class registers. We get the flag value and then shift the flags into
3674  // the correct place to combine with the mask.
3675  Mask = getMClassFlagsMask(Flags);
3676  if (Mask == -1)
3677  return -1;
3678  return Mask << 2;
3679  }
3680 
3681  if (Reg != "cpsr" && Reg != "spsr") {
3682  return -1;
3683  }
3684 
3685  // This is the same as if the flags were "fc"
3686  if (Flags.empty() || Flags == "all")
3687  return Mask | 0x9;
3688 
3689  // Inspect the supplied flags string and set the bits in the mask for
3690  // the relevant and valid flags allowed for cpsr and spsr.
3691  for (char Flag : Flags) {
3692  int FlagVal;
3693  switch (Flag) {
3694  case 'c':
3695  FlagVal = 0x1;
3696  break;
3697  case 'x':
3698  FlagVal = 0x2;
3699  break;
3700  case 's':
3701  FlagVal = 0x4;
3702  break;
3703  case 'f':
3704  FlagVal = 0x8;
3705  break;
3706  default:
3707  FlagVal = 0;
3708  }
3709 
3710  // This avoids allowing strings where the same flag bit appears twice.
3711  if (!FlagVal || (Mask & FlagVal))
3712  return -1;
3713  Mask |= FlagVal;
3714  }
3715 
3716  // If the register is spsr then we need to set the R bit.
3717  if (Reg == "spsr")
3718  Mask |= 0x10;
3719 
3720  return Mask;
3721 }
3722 
3723 // Lower the read_register intrinsic to ARM specific DAG nodes
3724 // using the supplied metadata string to select the instruction node to use
3725 // and the registers/masks to construct as operands for the node.
3726 bool ARMDAGToDAGISel::tryReadRegister(SDNode *N){
3727  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
3728  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
3729  bool IsThumb2 = Subtarget->isThumb2();
3730  SDLoc DL(N);
3731 
3732  std::vector<SDValue> Ops;
3733  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
3734 
3735  if (!Ops.empty()) {
3736  // If the special register string was constructed of fields (as defined
3737  // in the ACLE) then need to lower to MRC node (32 bit) or
3738  // MRRC node(64 bit), we can make the distinction based on the number of
3739  // operands we have.
3740  unsigned Opcode;
3741  SmallVector<EVT, 3> ResTypes;
3742  if (Ops.size() == 5){
3743  Opcode = IsThumb2 ? ARM::t2MRC : ARM::MRC;
3744  ResTypes.append({ MVT::i32, MVT::Other });
3745  } else {
3746  assert(Ops.size() == 3 &&
3747  "Invalid number of fields in special register string.");
3748  Opcode = IsThumb2 ? ARM::t2MRRC : ARM::MRRC;
3749  ResTypes.append({ MVT::i32, MVT::i32, MVT::Other });
3750  }
3751 
3752  Ops.push_back(getAL(CurDAG, DL));
3753  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3754  Ops.push_back(N->getOperand(0));
3755  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, ResTypes, Ops));
3756  return true;
3757  }
3758 
3759  std::string SpecialReg = RegString->getString().lower();
3760 
3761  int BankedReg = getBankedRegisterMask(SpecialReg);
3762  if (BankedReg != -1) {
3763  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32),
3764  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3765  N->getOperand(0) };
3766  ReplaceNode(
3767  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSbanked : ARM::MRSbanked,
3768  DL, MVT::i32, MVT::Other, Ops));
3769  return true;
3770  }
3771 
3772  // The VFP registers are read by creating SelectionDAG nodes with opcodes
3773  // corresponding to the register that is being read from. So we switch on the
3774  // string to find which opcode we need to use.
3775  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
3776  .Case("fpscr", ARM::VMRS)
3777  .Case("fpexc", ARM::VMRS_FPEXC)
3778  .Case("fpsid", ARM::VMRS_FPSID)
3779  .Case("mvfr0", ARM::VMRS_MVFR0)
3780  .Case("mvfr1", ARM::VMRS_MVFR1)
3781  .Case("mvfr2", ARM::VMRS_MVFR2)
3782  .Case("fpinst", ARM::VMRS_FPINST)
3783  .Case("fpinst2", ARM::VMRS_FPINST2)
3784  .Default(0);
3785 
3786  // If an opcode was found then we can lower the read to a VFP instruction.
3787  if (Opcode) {
3788  if (!Subtarget->hasVFP2())
3789  return false;
3790  if (Opcode == ARM::VMRS_MVFR2 && !Subtarget->hasFPARMv8())
3791  return false;
3792 
3793  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3794  N->getOperand(0) };
3795  ReplaceNode(N,
3796  CurDAG->getMachineNode(Opcode, DL, MVT::i32, MVT::Other, Ops));
3797  return true;
3798  }
3799 
3800  // If the target is M Class then need to validate that the register string
3801  // is an acceptable value, so check that a mask can be constructed from the
3802  // string.
3803  if (Subtarget->isMClass()) {
3804  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
3805  if (SYSmValue == -1)
3806  return false;
3807 
3808  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
3809  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3810  N->getOperand(0) };
3811  ReplaceNode(
3812  N, CurDAG->getMachineNode(ARM::t2MRS_M, DL, MVT::i32, MVT::Other, Ops));
3813  return true;
3814  }
3815 
3816  // Here we know the target is not M Class so we need to check if it is one
3817  // of the remaining possible values which are apsr, cpsr or spsr.
3818  if (SpecialReg == "apsr" || SpecialReg == "cpsr") {
3819  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3820  N->getOperand(0) };
3821  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRS_AR : ARM::MRS,
3822  DL, MVT::i32, MVT::Other, Ops));
3823  return true;
3824  }
3825 
3826  if (SpecialReg == "spsr") {
3827  Ops = { getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3828  N->getOperand(0) };
3829  ReplaceNode(
3830  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MRSsys_AR : ARM::MRSsys, DL,
3831  MVT::i32, MVT::Other, Ops));
3832  return true;
3833  }
3834 
3835  return false;
3836 }
3837 
3838 // Lower the write_register intrinsic to ARM specific DAG nodes
3839 // using the supplied metadata string to select the instruction node to use
3840 // and the registers/masks to use in the nodes
3841 bool ARMDAGToDAGISel::tryWriteRegister(SDNode *N){
3842  const MDNodeSDNode *MD = dyn_cast<MDNodeSDNode>(N->getOperand(1));
3843  const MDString *RegString = dyn_cast<MDString>(MD->getMD()->getOperand(0));
3844  bool IsThumb2 = Subtarget->isThumb2();
3845  SDLoc DL(N);
3846 
3847  std::vector<SDValue> Ops;
3848  getIntOperandsFromRegisterString(RegString->getString(), CurDAG, DL, Ops);
3849 
3850  if (!Ops.empty()) {
3851  // If the special register string was constructed of fields (as defined
3852  // in the ACLE) then need to lower to MCR node (32 bit) or
3853  // MCRR node(64 bit), we can make the distinction based on the number of
3854  // operands we have.
3855  unsigned Opcode;
3856  if (Ops.size() == 5) {
3857  Opcode = IsThumb2 ? ARM::t2MCR : ARM::MCR;
3858  Ops.insert(Ops.begin()+2, N->getOperand(2));
3859  } else {
3860  assert(Ops.size() == 3 &&
3861  "Invalid number of fields in special register string.");
3862  Opcode = IsThumb2 ? ARM::t2MCRR : ARM::MCRR;
3863  SDValue WriteValue[] = { N->getOperand(2), N->getOperand(3) };
3864  Ops.insert(Ops.begin()+2, WriteValue, WriteValue+2);
3865  }
3866 
3867  Ops.push_back(getAL(CurDAG, DL));
3868  Ops.push_back(CurDAG->getRegister(0, MVT::i32));
3869  Ops.push_back(N->getOperand(0));
3870 
3871  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
3872  return true;
3873  }
3874 
3875  std::string SpecialReg = RegString->getString().lower();
3876  int BankedReg = getBankedRegisterMask(SpecialReg);
3877  if (BankedReg != -1) {
3878  Ops = { CurDAG->getTargetConstant(BankedReg, DL, MVT::i32), N->getOperand(2),
3879  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3880  N->getOperand(0) };
3881  ReplaceNode(
3882  N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSRbanked : ARM::MSRbanked,
3883  DL, MVT::Other, Ops));
3884  return true;
3885  }
3886 
3887  // The VFP registers are written to by creating SelectionDAG nodes with
3888  // opcodes corresponding to the register that is being written. So we switch
3889  // on the string to find which opcode we need to use.
3890  unsigned Opcode = StringSwitch<unsigned>(SpecialReg)
3891  .Case("fpscr", ARM::VMSR)
3892  .Case("fpexc", ARM::VMSR_FPEXC)
3893  .Case("fpsid", ARM::VMSR_FPSID)
3894  .Case("fpinst", ARM::VMSR_FPINST)
3895  .Case("fpinst2", ARM::VMSR_FPINST2)
3896  .Default(0);
3897 
3898  if (Opcode) {
3899  if (!Subtarget->hasVFP2())
3900  return false;
3901  Ops = { N->getOperand(2), getAL(CurDAG, DL),
3902  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
3903  ReplaceNode(N, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops));
3904  return true;
3905  }
3906 
3907  std::pair<StringRef, StringRef> Fields;
3908  Fields = StringRef(SpecialReg).rsplit('_');
3909  std::string Reg = Fields.first.str();
3910  StringRef Flags = Fields.second;
3911 
3912  // If the target was M Class then need to validate the special register value
3913  // and retrieve the mask for use in the instruction node.
3914  if (Subtarget->isMClass()) {
3915  int SYSmValue = getMClassRegisterMask(SpecialReg, Subtarget);
3916  if (SYSmValue == -1)
3917  return false;
3918 
3919  SDValue Ops[] = { CurDAG->getTargetConstant(SYSmValue, DL, MVT::i32),
3920  N->getOperand(2), getAL(CurDAG, DL),
3921  CurDAG->getRegister(0, MVT::i32), N->getOperand(0) };
3922  ReplaceNode(N, CurDAG->getMachineNode(ARM::t2MSR_M, DL, MVT::Other, Ops));
3923  return true;
3924  }
3925 
3926  // We then check to see if a valid mask can be constructed for one of the
3927  // register string values permitted for the A and R class cores. These values
3928  // are apsr, spsr and cpsr; these are also valid on older cores.
3929  int Mask = getARClassRegisterMask(Reg, Flags);
3930  if (Mask != -1) {
3931  Ops = { CurDAG->getTargetConstant(Mask, DL, MVT::i32), N->getOperand(2),
3932  getAL(CurDAG, DL), CurDAG->getRegister(0, MVT::i32),
3933  N->getOperand(0) };
3934  ReplaceNode(N, CurDAG->getMachineNode(IsThumb2 ? ARM::t2MSR_AR : ARM::MSR,
3935  DL, MVT::Other, Ops));
3936  return true;
3937  }
3938 
3939  return false;
3940 }
3941 
3942 bool ARMDAGToDAGISel::tryInlineAsm(SDNode *N){
3943  std::vector<SDValue> AsmNodeOperands;
3944  unsigned Flag, Kind;
3945  bool Changed = false;
3946  unsigned NumOps = N->getNumOperands();
3947 
3948  // Normally, i64 data is bounded to two arbitrary GRPs for "%r" constraint.
3949  // However, some instrstions (e.g. ldrexd/strexd in ARM mode) require
3950  // (even/even+1) GPRs and use %n and %Hn to refer to the individual regs
3951  // respectively. Since there is no constraint to explicitly specify a
3952  // reg pair, we use GPRPair reg class for "%r" for 64-bit data. For Thumb,
3953  // the 64-bit data may be referred by H, Q, R modifiers, so we still pack
3954  // them into a GPRPair.
3955 
3956  SDLoc dl(N);
3957  SDValue Glue = N->getGluedNode() ? N->getOperand(NumOps-1)
3958  : SDValue(nullptr,0);
3959 
3960  SmallVector<bool, 8> OpChanged;
3961  // Glue node will be appended late.
3962  for(unsigned i = 0, e = N->getGluedNode() ? NumOps - 1 : NumOps; i < e; ++i) {
3963  SDValue op = N->getOperand(i);
3964  AsmNodeOperands.push_back(op);
3965 
3967  continue;
3968 
3969  if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(i))) {
3970  Flag = C->getZExtValue();
3971  Kind = InlineAsm::getKind(Flag);
3972  }
3973  else
3974  continue;
3975 
3976  // Immediate operands to inline asm in the SelectionDAG are modeled with
3977  // two operands. The first is a constant of value InlineAsm::Kind_Imm, and
3978  // the second is a constant with the value of the immediate. If we get here
3979  // and we have a Kind_Imm, skip the next operand, and continue.
3980  if (Kind == InlineAsm::Kind_Imm) {
3981  SDValue op = N->getOperand(++i);
3982  AsmNodeOperands.push_back(op);
3983  continue;
3984  }
3985 
3986  unsigned NumRegs = InlineAsm::getNumOperandRegisters(Flag);
3987  if (NumRegs)
3988  OpChanged.push_back(false);
3989 
3990  unsigned DefIdx = 0;
3991  bool IsTiedToChangedOp = false;
3992  // If it's a use that is tied with a previous def, it has no
3993  // reg class constraint.
3994  if (Changed && InlineAsm::isUseOperandTiedToDef(Flag, DefIdx))
3995  IsTiedToChangedOp = OpChanged[DefIdx];
3996 
3997  // Memory operands to inline asm in the SelectionDAG are modeled with two
3998  // operands: a constant of value InlineAsm::Kind_Mem followed by the input
3999  // operand. If we get here and we have a Kind_Mem, skip the next operand (so
4000  // it doesn't get misinterpreted), and continue. We do this here because
4001  // it's important to update the OpChanged array correctly before moving on.
4002  if (Kind == InlineAsm::Kind_Mem) {
4003  SDValue op = N->getOperand(++i);
4004  AsmNodeOperands.push_back(op);
4005  continue;
4006  }
4007 
4008  if (Kind != InlineAsm::Kind_RegUse && Kind != InlineAsm::Kind_RegDef
4010  continue;
4011 
4012  unsigned RC;
4013  bool HasRC = InlineAsm::hasRegClassConstraint(Flag, RC);
4014  if ((!IsTiedToChangedOp && (!HasRC || RC != ARM::GPRRegClassID))
4015  || NumRegs != 2)
4016  continue;
4017 
4018  assert((i+2 < NumOps) && "Invalid number of operands in inline asm");
4019  SDValue V0 = N->getOperand(i+1);
4020  SDValue V1 = N->getOperand(i+2);
4021  unsigned Reg0 = cast<RegisterSDNode>(V0)->getReg();
4022  unsigned Reg1 = cast<RegisterSDNode>(V1)->getReg();
4023  SDValue PairedReg;
4024  MachineRegisterInfo &MRI = MF->getRegInfo();
4025 
4026  if (Kind == InlineAsm::Kind_RegDef ||
4028  // Replace the two GPRs with 1 GPRPair and copy values from GPRPair to
4029  // the original GPRs.
4030 
4031  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4032  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4033  SDValue Chain = SDValue(N,0);
4034 
4035  SDNode *GU = N->getGluedUser();
4036  SDValue RegCopy = CurDAG->getCopyFromReg(Chain, dl, GPVR, MVT::Untyped,
4037  Chain.getValue(1));
4038 
4039  // Extract values from a GPRPair reg and copy to the original GPR reg.
4040  SDValue Sub0 = CurDAG->getTargetExtractSubreg(ARM::gsub_0, dl, MVT::i32,
4041  RegCopy);
4042  SDValue Sub1 = CurDAG->getTargetExtractSubreg(ARM::gsub_1, dl, MVT::i32,
4043  RegCopy);
4044  SDValue T0 = CurDAG->getCopyToReg(Sub0, dl, Reg0, Sub0,
4045  RegCopy.getValue(1));
4046  SDValue T1 = CurDAG->getCopyToReg(Sub1, dl, Reg1, Sub1, T0.getValue(1));
4047 
4048  // Update the original glue user.
4049  std::vector<SDValue> Ops(GU->op_begin(), GU->op_end()-1);
4050  Ops.push_back(T1.getValue(1));
4051  CurDAG->UpdateNodeOperands(GU, Ops);
4052  }
4053  else {
4054  // For Kind == InlineAsm::Kind_RegUse, we first copy two GPRs into a
4055  // GPRPair and then pass the GPRPair to the inline asm.
4056  SDValue Chain = AsmNodeOperands[InlineAsm::Op_InputChain];
4057 
4058  // As REG_SEQ doesn't take RegisterSDNode, we copy them first.
4059  SDValue T0 = CurDAG->getCopyFromReg(Chain, dl, Reg0, MVT::i32,
4060  Chain.getValue(1));
4061  SDValue T1 = CurDAG->getCopyFromReg(Chain, dl, Reg1, MVT::i32,
4062  T0.getValue(1));
4063  SDValue Pair = SDValue(createGPRPairNode(MVT::Untyped, T0, T1), 0);
4064 
4065  // Copy REG_SEQ into a GPRPair-typed VR and replace the original two
4066  // i32 VRs of inline asm with it.
4067  unsigned GPVR = MRI.createVirtualRegister(&ARM::GPRPairRegClass);
4068  PairedReg = CurDAG->getRegister(GPVR, MVT::Untyped);
4069  Chain = CurDAG->getCopyToReg(T1, dl, GPVR, Pair, T1.getValue(1));
4070 
4071  AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
4072  Glue = Chain.getValue(1);
4073  }
4074 
4075  Changed = true;
4076 
4077  if(PairedReg.getNode()) {
4078  OpChanged[OpChanged.size() -1 ] = true;
4079  Flag = InlineAsm::getFlagWord(Kind, 1 /* RegNum*/);
4080  if (IsTiedToChangedOp)
4081  Flag = InlineAsm::getFlagWordForMatchingOp(Flag, DefIdx);
4082  else
4083  Flag = InlineAsm::getFlagWordForRegClass(Flag, ARM::GPRPairRegClassID);
4084  // Replace the current flag.
4085  AsmNodeOperands[AsmNodeOperands.size() -1] = CurDAG->getTargetConstant(
4086  Flag, dl, MVT::i32);
4087  // Add the new register node and skip the original two GPRs.
4088  AsmNodeOperands.push_back(PairedReg);
4089  // Skip the next two GPRs.
4090  i += 2;
4091  }
4092  }
4093 
4094  if (Glue.getNode())
4095  AsmNodeOperands.push_back(Glue);
4096  if (!Changed)
4097  return false;
4098 
4099  SDValue New = CurDAG->getNode(ISD::INLINEASM, SDLoc(N),
4100  CurDAG->getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
4101  New->setNodeId(-1);
4102  ReplaceNode(N, New.getNode());
4103  return true;
4104 }
4105 
4106 
4107 bool ARMDAGToDAGISel::
4108 SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID,
4109  std::vector<SDValue> &OutOps) {
4110  switch(ConstraintID) {
4111  default:
4112  llvm_unreachable("Unexpected asm memory constraint");
4114  // FIXME: It seems strange that 'i' is needed here since it's supposed to
4115  // be an immediate and not a memory constraint.
4127  // Require the address to be in a register. That is safe for all ARM
4128  // variants and it is hard to do anything much smarter without knowing
4129  // how the operand is used.
4130  OutOps.push_back(Op);
4131  return false;
4132  }
4133  return true;
4134 }
4135 
4136 /// createARMISelDag - This pass converts a legalized DAG into a
4137 /// ARM-specific DAG, ready for instruction scheduling.
4138 ///
4140  CodeGenOpt::Level OptLevel) {
4141  return new ARMDAGToDAGISel(TM, OptLevel);
4142 }
uint64_t CallInst * C
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
EVT getValueType() const
Return the ValueType of the referenced return value.
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const SDValue & getOffset() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isThumb() const
Definition: ARMSubtarget.h:677
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
Various leaf nodes.
Definition: ISDOpcodes.h:60
static int getARClassRegisterMask(StringRef Reg, StringRef Flags)
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:235
LLVM_ATTRIBUTE_ALWAYS_INLINE size_type size() const
Definition: SmallVector.h:136
const SDValue & getBasePtr() const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
static unsigned getFlagWord(unsigned Kind, unsigned NumOps)
Definition: InlineAsm.h:269
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:163
SDVTList getVTList() const
static Optional< std::pair< unsigned, unsigned > > getContiguousRangeOfSetBits(const APInt &A)
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
const SDValue & getChain() const
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc, or post-dec.
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:131
unsigned getAlignment() const
static bool isVSTfixed(unsigned Opc)
bool isSOImmTwoPartVal(unsigned V)
isSOImmTwoPartVal - Return true if the specified value can be obtained by or&#39;ing together two SOImmVa...
bool hasV6Ops() const
Definition: ARMSubtarget.h:504
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1067
unsigned getAM3Opc(AddrOpc Opc, unsigned char Offset, unsigned IdxMode=0)
getAM3Opc - This function encodes the addrmode3 opc field.
static bool isScaledConstantInRange(SDValue Node, int Scale, int RangeMin, int RangeMax, int &ScaledConstant)
Check whether a particular node is a constant value representable as (N * Scale) where (N in [RangeMi...
bool isThumb1Only() const
Definition: ARMSubtarget.h:678
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
#define op(i)
std::size_t countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the most significant bit to the least stopping at the first 1...
Definition: MathExtras.h:181
static bool isThumb(const MCSubtargetInfo &STI)
static bool isUseOperandTiedToDef(unsigned Flag, unsigned &Idx)
isUseOperandTiedToDef - Return true if the flag of the inline asm operand indicates it is an use oper...
Definition: InlineAsm.h:342
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1488
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:159
bool hasV8MBaselineOps() const
Definition: ARMSubtarget.h:513
unsigned countTrailingZeros() const
Count the number of trailing zero bits.
Definition: APInt.h:1611
bool hasOneUse() const
Return true if there is exactly one use of this node.
bool isFpMLxInstruction(unsigned Opcode) const
isFpMLxInstruction - Return true if the specified opcode is a fp MLA / MLS instruction.
A description of a memory reference used in the backend.
static unsigned getFlagWordForRegClass(unsigned InputFlag, unsigned RC)
getFlagWordForRegClass - Augment an existing flag word returned by getFlagWord with the required regi...
Definition: InlineAsm.h:300
const HexagonInstrInfo * TII
Shift and rotation operations.
Definition: ISDOpcodes.h:380
std::size_t countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:470
Base class for LoadSDNode and StoreSDNode.
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:369
A Use represents the edge between a Value definition and its users.
Definition: Use.h:56
CopyToReg - This node has three operands: a chain, a register number to set to this value...
Definition: ISDOpcodes.h:170
const MDNode * getMD() const
op_iterator op_end() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Reg
All possible values of the reg field in the ModR/M byte.
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static ShiftOpc getShiftOpcForNode(unsigned Opcode)
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE R Default(T Value)
Definition: StringSwitch.h:203
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
INLINEASM - Represents an inline asm block.
Definition: ISDOpcodes.h:635
bool hasVFP2() const
Definition: ARMSubtarget.h:533
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:298
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:292
bool hasDSP() const
Definition: ARMSubtarget.h:590
bool useMovt(const MachineFunction &MF) const
static bool isVLDfixed(unsigned Opc)
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:201
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:133
LLVM_NODISCARD std::pair< StringRef, StringRef > rsplit(char Separator) const
Split into two substrings around the last occurrence of a separator character.
Definition: StringRef.h:798
bool hasV6T2Ops() const
Definition: ARMSubtarget.h:507
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:561
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:121
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:43
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:406
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:166
unsigned getAM5Opc(AddrOpc Opc, unsigned char Offset)
getAM5Opc - This function encodes the addrmode5 opc field.
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:85
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
Container class for subtarget features.
std::size_t countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0&#39;s from the least significant bit to the most stopping at the first 1...
Definition: MathExtras.h:112
static int getBankedRegisterMask(StringRef RegString)
unsigned countPopulation() const
Count the number of bits set.
Definition: APInt.h:1637
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:421
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
static unsigned getVLDSTRegisterUpdateOpcode(unsigned Opc)
static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned &Imm)
bool isMClass() const
Definition: ARMSubtarget.h:681
bool isThumbImmShiftedVal(unsigned V)
isThumbImmShiftedVal - Return true if the specified value can be obtained by left shifting a 8-bit im...
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:273
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static cl::opt< bool > DisableShifterOp("disable-shifter-op", cl::Hidden, cl::desc("Disable isel of shifter-op"), cl::init(false))
SDNode * getGluedUser() const
If this node has a glue value with a user, return the user (there is at most one).
const SDValue & getOperand(unsigned Num) const
This file contains the declarations for the subclasses of Constant, which represent the different fla...
bool hasFPARMv8() const
Definition: ARMSubtarget.h:536
static unsigned getNumOperandRegisters(unsigned Flag)
getNumOperandRegisters - Extract the number of registers field from the inline asm operand flag...
Definition: InlineAsm.h:336
int getT2SOImmVal(unsigned Arg)
getT2SOImmVal - Given a 32-bit immediate, if it is something that can fit into a Thumb-2 shifter_oper...
static unsigned getKind(unsigned Flags)
Definition: InlineAsm.h:325
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:285
int getT2SOImmValSplatVal(unsigned V)
getT2SOImmValSplat - Return the 12-bit encoded representation if the specified value can be obtained ...
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
self_iterator getIterator()
Definition: ilist_node.h:82
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo...
Definition: ISDOpcodes.h:764
Extended Value Type.
Definition: ValueTypes.h:34
static bool isInt32Immediate(SDNode *N, unsigned &Imm)
isInt32Immediate - This method tests to see if the node is a 32-bit constant operand.
static SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl)
getAL - Returns a ARMCC::AL immediate node.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode...
unsigned getNumOperands() const
Return the number of values used by this operation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Iterator for intrusive lists based on ilist_node.
int getSOImmVal(unsigned Arg)
getSOImmVal - Given a 32-bit immediate, if it is something that can fit into an shifter_operand immed...
static SDValue createGPRPairNode(SelectionDAG &DAG, SDValue V)
unsigned getAM5FP16Opc(AddrOpc Opc, unsigned char Offset)
getAM5FP16Opc - This function encodes the addrmode5fp16 opc field.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:265
unsigned getAM2Opc(AddrOpc Opc, unsigned Imm12, ShiftOpc SO, unsigned IdxMode=0)
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:212
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:862
LLVM_NODISCARD std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:727
This is an abstract virtual class for memory operations.
unsigned getSORegOpc(ShiftOpc ShOp, unsigned Imm)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:611
Represents one node in the SelectionDAG.
#define NC
Definition: regutils.h:42
SelectionDAGISel - This is the common base class used for SelectionDAG-based pattern-matching instruc...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:531
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:924
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT...
Definition: ValueTypes.h:73
EVT getMemoryVT() const
Return the type of the in-memory value.
Class for arbitrary precision integers.
Definition: APInt.h:69
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
bool mayStore() const
Return true if this instruction could possibly modify memory.
Definition: MCInstrDesc.h:393
LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:70
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:396
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getNodeId() const
Return the unique node id.
static bool hasRegClassConstraint(unsigned Flag, unsigned &RC)
hasRegClassConstraint - Returns true if the flag contains a register class constraint.
Definition: InlineAsm.h:351
bool isThumb2() const
Definition: ARMSubtarget.h:679
bool is64BitVector() const
Return true if this is a 64-bit vector type.
Definition: ValueTypes.h:177
const SDValue & getValue() const
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:363
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:176
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:206
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:464
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:582
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static int getMClassRegisterMask(StringRef Reg, const ARMSubtarget *Subtarget)
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:323
unsigned getOpcode() const
SDValue getValue(unsigned R) const
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
FunctionPass * createARMISelDag(ARMBaseTargetMachine &TM, CodeGenOpt::Level OptLevel)
createARMISelDag - This pass converts a legalized DAG into a ARM-specific DAG, ready for instruction ...
bool hasVMLxHazards() const
Definition: ARMSubtarget.h:575
const unsigned Kind
LLVM_NODISCARD std::string lower() const
Definition: StringRef.cpp:108
This class is used to form a handle around another node that is persistent and is updated across invo...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getReg() const
bool isSwift() const
Definition: ARMSubtarget.h:524
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
unsigned getResNo() const
get the index which selects a specific result in the SDNode
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
static void getIntOperandsFromRegisterString(StringRef RegString, SelectionDAG *CurDAG, const SDLoc &DL, std::vector< SDValue > &Ops)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
unsigned getOpcode() const
Return the opcode number for this descriptor.
Definition: MCInstrDesc.h:203
static bool isPerfectIncrement(SDValue Inc, EVT VecTy, unsigned NumVecs)
Returns true if the given increment is a Constant known to be equal to the access size performed by a...
constexpr bool isShiftedMask_32(uint32_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (32 bit ver...
Definition: MathExtras.h:409
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
A single uniqued string.
Definition: Metadata.h:602
static int getMClassFlagsMask(StringRef Flags)
unsigned countLeadingZeros() const
The APInt version of the countLeadingZeros functions in MathExtras.h.
Definition: APInt.h:1575
const SDValue & getOperand(unsigned i) const
uint64_t getZExtValue() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void setObjectAlignment(int ObjectIdx, unsigned Align)
setObjectAlignment - Change the alignment of the specified stack object.
MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num)
allocateMemRefsArray - Allocate an array to hold MachineMemOperand pointers.
bool hasThumb2() const
Definition: ARMSubtarget.h:680
bool isLikeA9() const
Definition: ARMSubtarget.h:526
#define T1
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
static unsigned getFlagWordForMatchingOp(unsigned InputFlag, unsigned MatchedOperandNo)
getFlagWordForMatchingOp - Augment an existing flag word returned by getFlagWord with information ind...
Definition: InlineAsm.h:288
This file describes how to lower LLVM code to machine code.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:873
This class is used to represent ISD::LOAD nodes.