LLVM  16.0.0git
RISCVISelDAGToDAG.cpp
Go to the documentation of this file.
1 //===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISCV ------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines an instruction selector for the RISCV target.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "RISCVISelDAGToDAG.h"
16 #include "RISCVISelLowering.h"
19 #include "llvm/IR/IntrinsicsRISCV.h"
20 #include "llvm/Support/Alignment.h"
21 #include "llvm/Support/Debug.h"
22 #include "llvm/Support/KnownBits.h"
25 #include <optional>
26 
27 using namespace llvm;
28 
29 #define DEBUG_TYPE "riscv-isel"
30 
31 namespace llvm::RISCV {
32 #define GET_RISCVVSSEGTable_IMPL
33 #define GET_RISCVVLSEGTable_IMPL
34 #define GET_RISCVVLXSEGTable_IMPL
35 #define GET_RISCVVSXSEGTable_IMPL
36 #define GET_RISCVVLETable_IMPL
37 #define GET_RISCVVSETable_IMPL
38 #define GET_RISCVVLXTable_IMPL
39 #define GET_RISCVVSXTable_IMPL
40 #define GET_RISCVMaskedPseudosTable_IMPL
41 #include "RISCVGenSearchableTables.inc"
42 } // namespace llvm::RISCV
43 
44 static unsigned getLastNonGlueOrChainOpIdx(const SDNode *Node) {
45  assert(Node->getNumOperands() > 0 && "Node with no operands");
46  unsigned LastOpIdx = Node->getNumOperands() - 1;
47  if (Node->getOperand(LastOpIdx).getValueType() == MVT::Glue)
48  --LastOpIdx;
49  if (Node->getOperand(LastOpIdx).getValueType() == MVT::Other)
50  --LastOpIdx;
51  return LastOpIdx;
52 }
53 
54 static unsigned getVecPolicyOpIdx(const SDNode *Node, const MCInstrDesc &MCID) {
56  (void)MCID;
57  return getLastNonGlueOrChainOpIdx(Node);
58 }
59 
62 
63  bool MadeChange = false;
64  while (Position != CurDAG->allnodes_begin()) {
65  SDNode *N = &*--Position;
66  if (N->use_empty())
67  continue;
68 
69  SDValue Result;
70  switch (N->getOpcode()) {
71  case ISD::SPLAT_VECTOR: {
72  // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
73  // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
74  MVT VT = N->getSimpleValueType(0);
75  unsigned Opc =
77  SDLoc DL(N);
78  SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
79  Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT),
80  N->getOperand(0), VL);
81  break;
82  }
84  // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
85  // load. Done after lowering and combining so that we have a chance to
86  // optimize this to VMV_V_X_VL when the upper bits aren't needed.
87  assert(N->getNumOperands() == 4 && "Unexpected number of operands");
88  MVT VT = N->getSimpleValueType(0);
89  SDValue Passthru = N->getOperand(0);
90  SDValue Lo = N->getOperand(1);
91  SDValue Hi = N->getOperand(2);
92  SDValue VL = N->getOperand(3);
94  Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
95  "Unexpected VTs!");
99  SDLoc DL(N);
100 
101  // We use the same frame index we use for moving two i32s into 64-bit FPR.
102  // This is an analogous operation.
103  int FI = FuncInfo->getMoveF64FrameIndex(MF);
106  SDValue StackSlot =
108 
109  SDValue Chain = CurDAG->getEntryNode();
110  Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
111 
112  SDValue OffsetSlot =
114  Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
115  Align(8));
116 
117  Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
118 
119  SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
120  SDValue IntID =
121  CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
122  SDValue Ops[] = {Chain,
123  IntID,
124  Passthru,
125  StackSlot,
126  CurDAG->getRegister(RISCV::X0, MVT::i64),
127  VL};
128 
130  MVT::i64, MPI, Align(8),
132  break;
133  }
134  }
135 
136  if (Result) {
137  LLVM_DEBUG(dbgs() << "RISCV DAG preprocessing replacing:\nOld: ");
138  LLVM_DEBUG(N->dump(CurDAG));
139  LLVM_DEBUG(dbgs() << "\nNew: ");
140  LLVM_DEBUG(Result->dump(CurDAG));
141  LLVM_DEBUG(dbgs() << "\n");
142 
144  MadeChange = true;
145  }
146  }
147 
148  if (MadeChange)
150 }
151 
155 
156  bool MadeChange = false;
157  while (Position != CurDAG->allnodes_begin()) {
158  SDNode *N = &*--Position;
159  // Skip dead nodes and any non-machine opcodes.
160  if (N->use_empty() || !N->isMachineOpcode())
161  continue;
162 
163  MadeChange |= doPeepholeSExtW(N);
164  MadeChange |= doPeepholeMaskedRVV(N);
165  }
166 
167  CurDAG->setRoot(Dummy.getValue());
168 
169  MadeChange |= doPeepholeMergeVVMFold();
170 
171  if (MadeChange)
173 }
174 
175 static SDNode *selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
176  RISCVMatInt::InstSeq &Seq) {
177  SDNode *Result = nullptr;
178  SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
179  for (RISCVMatInt::Inst &Inst : Seq) {
180  SDValue SDImm = CurDAG->getTargetConstant(Inst.Imm, DL, VT);
181  switch (Inst.getOpndKind()) {
182  case RISCVMatInt::Imm:
183  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SDImm);
184  break;
185  case RISCVMatInt::RegX0:
186  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg,
187  CurDAG->getRegister(RISCV::X0, VT));
188  break;
189  case RISCVMatInt::RegReg:
190  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SrcReg);
191  break;
192  case RISCVMatInt::RegImm:
193  Result = CurDAG->getMachineNode(Inst.Opc, DL, VT, SrcReg, SDImm);
194  break;
195  }
196 
197  // Only the first instruction has X0 as its source.
198  SrcReg = SDValue(Result, 0);
199  }
200 
201  return Result;
202 }
203 
204 static SDNode *selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
205  int64_t Imm, const RISCVSubtarget &Subtarget) {
207  RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
208 
209  return selectImmSeq(CurDAG, DL, VT, Seq);
210 }
211 
213  unsigned NF, RISCVII::VLMUL LMUL) {
214  static const unsigned M1TupleRegClassIDs[] = {
215  RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
216  RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
217  RISCV::VRN8M1RegClassID};
218  static const unsigned M2TupleRegClassIDs[] = {RISCV::VRN2M2RegClassID,
219  RISCV::VRN3M2RegClassID,
220  RISCV::VRN4M2RegClassID};
221 
222  assert(Regs.size() >= 2 && Regs.size() <= 8);
223 
224  unsigned RegClassID;
225  unsigned SubReg0;
226  switch (LMUL) {
227  default:
228  llvm_unreachable("Invalid LMUL.");
233  static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
234  "Unexpected subreg numbering");
235  SubReg0 = RISCV::sub_vrm1_0;
236  RegClassID = M1TupleRegClassIDs[NF - 2];
237  break;
239  static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
240  "Unexpected subreg numbering");
241  SubReg0 = RISCV::sub_vrm2_0;
242  RegClassID = M2TupleRegClassIDs[NF - 2];
243  break;
245  static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
246  "Unexpected subreg numbering");
247  SubReg0 = RISCV::sub_vrm4_0;
248  RegClassID = RISCV::VRN2M4RegClassID;
249  break;
250  }
251 
252  SDLoc DL(Regs[0]);
254 
255  Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
256 
257  for (unsigned I = 0; I < Regs.size(); ++I) {
258  Ops.push_back(Regs[I]);
259  Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
260  }
261  SDNode *N =
262  CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
263  return SDValue(N, 0);
264 }
265 
267  SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
268  bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
269  bool IsLoad, MVT *IndexVT) {
270  SDValue Chain = Node->getOperand(0);
271  SDValue Glue;
272 
273  Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
274 
275  if (IsStridedOrIndexed) {
276  Operands.push_back(Node->getOperand(CurOp++)); // Index.
277  if (IndexVT)
278  *IndexVT = Operands.back()->getSimpleValueType(0);
279  }
280 
281  if (IsMasked) {
282  // Mask needs to be copied to V0.
283  SDValue Mask = Node->getOperand(CurOp++);
284  Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
285  Glue = Chain.getValue(1);
286  Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
287  }
288  SDValue VL;
289  selectVLOp(Node->getOperand(CurOp++), VL);
290  Operands.push_back(VL);
291 
292  MVT XLenVT = Subtarget->getXLenVT();
293  SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
294  Operands.push_back(SEWOp);
295 
296  // Masked load has the tail policy argument.
297  if (IsMasked && IsLoad) {
298  // Policy must be a constant.
299  uint64_t Policy = Node->getConstantOperandVal(CurOp++);
300  SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
301  Operands.push_back(PolicyOp);
302  }
303 
304  Operands.push_back(Chain); // Chain.
305  if (Glue)
306  Operands.push_back(Glue);
307 }
308 
309 static bool isAllUndef(ArrayRef<SDValue> Values) {
310  return llvm::all_of(Values, [](SDValue V) { return V->isUndef(); });
311 }
312 
313 void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
314  bool IsStrided) {
315  SDLoc DL(Node);
316  unsigned NF = Node->getNumValues() - 1;
317  MVT VT = Node->getSimpleValueType(0);
318  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
320 
321  unsigned CurOp = 2;
323 
324  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
325  Node->op_begin() + CurOp + NF);
326  bool IsTU = IsMasked || !isAllUndef(Regs);
327  if (IsTU) {
328  SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL);
329  Operands.push_back(Merge);
330  }
331  CurOp += NF;
332 
333  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
334  Operands, /*IsLoad=*/true);
335 
336  const RISCV::VLSEGPseudo *P =
337  RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW,
338  static_cast<unsigned>(LMUL));
341 
342  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
343  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
344 
345  SDValue SuperReg = SDValue(Load, 0);
346  for (unsigned I = 0; I < NF; ++I) {
347  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
348  ReplaceUses(SDValue(Node, I),
349  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
350  }
351 
352  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
353  CurDAG->RemoveDeadNode(Node);
354 }
355 
356 void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
357  SDLoc DL(Node);
358  unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
359  MVT VT = Node->getSimpleValueType(0);
360  MVT XLenVT = Subtarget->getXLenVT();
361  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
363 
364  unsigned CurOp = 2;
366 
367  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
368  Node->op_begin() + CurOp + NF);
369  bool IsTU = IsMasked || !isAllUndef(Regs);
370  if (IsTU) {
371  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
372  Operands.push_back(MaskedOff);
373  }
374  CurOp += NF;
375 
376  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
377  /*IsStridedOrIndexed*/ false, Operands,
378  /*IsLoad=*/true);
379 
380  const RISCV::VLSEGPseudo *P =
381  RISCV::getVLSEGPseudo(NF, IsMasked, IsTU, /*Strided*/ false, /*FF*/ true,
382  Log2SEW, static_cast<unsigned>(LMUL));
384  XLenVT, MVT::Other, Operands);
385 
386  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
387  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
388 
389  SDValue SuperReg = SDValue(Load, 0);
390  for (unsigned I = 0; I < NF; ++I) {
391  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
392  ReplaceUses(SDValue(Node, I),
393  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
394  }
395 
396  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL
397  ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain
398  CurDAG->RemoveDeadNode(Node);
399 }
400 
401 void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
402  bool IsOrdered) {
403  SDLoc DL(Node);
404  unsigned NF = Node->getNumValues() - 1;
405  MVT VT = Node->getSimpleValueType(0);
406  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
408 
409  unsigned CurOp = 2;
411 
412  SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
413  Node->op_begin() + CurOp + NF);
414  bool IsTU = IsMasked || !isAllUndef(Regs);
415  if (IsTU) {
416  SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
417  Operands.push_back(MaskedOff);
418  }
419  CurOp += NF;
420 
421  MVT IndexVT;
422  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
423  /*IsStridedOrIndexed*/ true, Operands,
424  /*IsLoad=*/true, &IndexVT);
425 
427  "Element count mismatch");
428 
429  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
430  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
431  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
432  report_fatal_error("The V extension does not support EEW=64 for index "
433  "values when XLEN=32");
434  }
435  const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
436  NF, IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
437  static_cast<unsigned>(IndexLMUL));
440 
441  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
442  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
443 
444  SDValue SuperReg = SDValue(Load, 0);
445  for (unsigned I = 0; I < NF; ++I) {
446  unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
447  ReplaceUses(SDValue(Node, I),
448  CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
449  }
450 
451  ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
452  CurDAG->RemoveDeadNode(Node);
453 }
454 
455 void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
456  bool IsStrided) {
457  SDLoc DL(Node);
458  unsigned NF = Node->getNumOperands() - 4;
459  if (IsStrided)
460  NF--;
461  if (IsMasked)
462  NF--;
463  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
464  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
466  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
467  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
468 
470  Operands.push_back(StoreVal);
471  unsigned CurOp = 2 + NF;
472 
473  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
474  Operands);
475 
476  const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
477  NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
479  CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
480 
481  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
482  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
483 
484  ReplaceNode(Node, Store);
485 }
486 
487 void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
488  bool IsOrdered) {
489  SDLoc DL(Node);
490  unsigned NF = Node->getNumOperands() - 5;
491  if (IsMasked)
492  --NF;
493  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
494  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
496  SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
497  SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
498 
500  Operands.push_back(StoreVal);
501  unsigned CurOp = 2 + NF;
502 
503  MVT IndexVT;
504  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
505  /*IsStridedOrIndexed*/ true, Operands,
506  /*IsLoad=*/false, &IndexVT);
507 
509  "Element count mismatch");
510 
511  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
512  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
513  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
514  report_fatal_error("The V extension does not support EEW=64 for index "
515  "values when XLEN=32");
516  }
517  const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
518  NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
519  static_cast<unsigned>(IndexLMUL));
521  CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
522 
523  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
524  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
525 
526  ReplaceNode(Node, Store);
527 }
528 
530  if (!Subtarget->hasVInstructions())
531  return;
532 
533  assert((Node->getOpcode() == ISD::INTRINSIC_W_CHAIN ||
534  Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN) &&
535  "Unexpected opcode");
536 
537  SDLoc DL(Node);
538  MVT XLenVT = Subtarget->getXLenVT();
539 
540  bool HasChain = Node->getOpcode() == ISD::INTRINSIC_W_CHAIN;
541  unsigned IntNoOffset = HasChain ? 1 : 0;
542  unsigned IntNo = Node->getConstantOperandVal(IntNoOffset);
543 
544  assert((IntNo == Intrinsic::riscv_vsetvli ||
545  IntNo == Intrinsic::riscv_vsetvlimax ||
546  IntNo == Intrinsic::riscv_vsetvli_opt ||
547  IntNo == Intrinsic::riscv_vsetvlimax_opt) &&
548  "Unexpected vsetvli intrinsic");
549 
550  bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax ||
551  IntNo == Intrinsic::riscv_vsetvlimax_opt;
552  unsigned Offset = IntNoOffset + (VLMax ? 1 : 2);
553 
554  assert(Node->getNumOperands() == Offset + 2 &&
555  "Unexpected number of operands");
556 
557  unsigned SEW =
558  RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
559  RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>(
560  Node->getConstantOperandVal(Offset + 1) & 0x7);
561 
562  unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
563  /*MaskAgnostic*/ false);
564  SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
565 
566  SmallVector<EVT, 2> VTs = {XLenVT};
567  if (HasChain)
568  VTs.push_back(MVT::Other);
569 
570  SDValue VLOperand;
571  unsigned Opcode = RISCV::PseudoVSETVLI;
572  if (VLMax) {
573  VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
574  Opcode = RISCV::PseudoVSETVLIX0;
575  } else {
576  VLOperand = Node->getOperand(IntNoOffset + 1);
577 
578  if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
579  uint64_t AVL = C->getZExtValue();
580  if (isUInt<5>(AVL)) {
581  SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
582  SmallVector<SDValue, 3> Ops = {VLImm, VTypeIOp};
583  if (HasChain)
584  Ops.push_back(Node->getOperand(0));
585  ReplaceNode(
586  Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL, VTs, Ops));
587  return;
588  }
589  }
590  }
591 
592  SmallVector<SDValue, 3> Ops = {VLOperand, VTypeIOp};
593  if (HasChain)
594  Ops.push_back(Node->getOperand(0));
595 
596  ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, VTs, Ops));
597 }
598 
600  MVT VT = Node->getSimpleValueType(0);
601  unsigned Opcode = Node->getOpcode();
602  assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
603  "Unexpected opcode");
604  SDLoc DL(Node);
605 
606  // For operations of the form (x << C1) op C2, check if we can use
607  // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
608  SDValue N0 = Node->getOperand(0);
609  SDValue N1 = Node->getOperand(1);
610 
611  ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1);
612  if (!Cst)
613  return false;
614 
615  int64_t Val = Cst->getSExtValue();
616 
617  // Check if immediate can already use ANDI/ORI/XORI.
618  if (isInt<12>(Val))
619  return false;
620 
621  SDValue Shift = N0;
622 
623  // If Val is simm32 and we have a sext_inreg from i32, then the binop
624  // produces at least 33 sign bits. We can peek through the sext_inreg and use
625  // a SLLIW at the end.
626  bool SignExt = false;
627  if (isInt<32>(Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
628  N0.hasOneUse() && cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32) {
629  SignExt = true;
630  Shift = N0.getOperand(0);
631  }
632 
633  if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
634  return false;
635 
636  ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
637  if (!ShlCst)
638  return false;
639 
640  uint64_t ShAmt = ShlCst->getZExtValue();
641 
642  // Make sure that we don't change the operation by removing bits.
643  // This only matters for OR and XOR, AND is unaffected.
644  uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(ShAmt);
645  if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
646  return false;
647 
648  int64_t ShiftedVal = Val >> ShAmt;
649  if (!isInt<12>(ShiftedVal))
650  return false;
651 
652  // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
653  if (SignExt && ShAmt >= 32)
654  return false;
655 
656  // Ok, we can reorder to get a smaller immediate.
657  unsigned BinOpc;
658  switch (Opcode) {
659  default: llvm_unreachable("Unexpected opcode");
660  case ISD::AND: BinOpc = RISCV::ANDI; break;
661  case ISD::OR: BinOpc = RISCV::ORI; break;
662  case ISD::XOR: BinOpc = RISCV::XORI; break;
663  }
664 
665  unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
666 
667  SDNode *BinOp =
668  CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
669  CurDAG->getTargetConstant(ShiftedVal, DL, VT));
670  SDNode *SLLI =
671  CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
672  CurDAG->getTargetConstant(ShAmt, DL, VT));
673  ReplaceNode(Node, SLLI);
674  return true;
675 }
676 
678  // If we have a custom node, we have already selected.
679  if (Node->isMachineOpcode()) {
680  LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
681  Node->setNodeId(-1);
682  return;
683  }
684 
685  // Instruction Selection not handled by the auto-generated tablegen selection
686  // should be handled here.
687  unsigned Opcode = Node->getOpcode();
688  MVT XLenVT = Subtarget->getXLenVT();
689  SDLoc DL(Node);
690  MVT VT = Node->getSimpleValueType(0);
691 
692  switch (Opcode) {
693  case ISD::Constant: {
694  auto *ConstNode = cast<ConstantSDNode>(Node);
695  if (VT == XLenVT && ConstNode->isZero()) {
696  SDValue New =
697  CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, XLenVT);
698  ReplaceNode(Node, New.getNode());
699  return;
700  }
701  int64_t Imm = ConstNode->getSExtValue();
702  // If the upper XLen-16 bits are not used, try to convert this to a simm12
703  // by sign extending bit 15.
704  if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) &&
705  hasAllHUsers(Node))
706  Imm = SignExtend64<16>(Imm);
707  // If the upper 32-bits are not used try to convert this into a simm32 by
708  // sign extending bit 32.
709  if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node))
710  Imm = SignExtend64<32>(Imm);
711 
712  ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget));
713  return;
714  }
715  case ISD::SHL: {
716  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
717  if (!N1C)
718  break;
719  SDValue N0 = Node->getOperand(0);
720  if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
721  !isa<ConstantSDNode>(N0.getOperand(1)))
722  break;
723  unsigned ShAmt = N1C->getZExtValue();
725 
726  // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C) where C2 has
727  // 32 leading zeros and C3 trailing zeros.
728  if (ShAmt <= 32 && isShiftedMask_64(Mask)) {
729  unsigned XLen = Subtarget->getXLen();
730  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask));
731  unsigned TrailingZeros = countTrailingZeros(Mask);
732  if (TrailingZeros > 0 && LeadingZeros == 32) {
733  SDNode *SRLIW = CurDAG->getMachineNode(
734  RISCV::SRLIW, DL, VT, N0->getOperand(0),
735  CurDAG->getTargetConstant(TrailingZeros, DL, VT));
736  SDNode *SLLI = CurDAG->getMachineNode(
737  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
738  CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT));
739  ReplaceNode(Node, SLLI);
740  return;
741  }
742  }
743  break;
744  }
745  case ISD::SRL: {
746  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
747  if (!N1C)
748  break;
749  SDValue N0 = Node->getOperand(0);
750  if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
751  break;
752  unsigned ShAmt = N1C->getZExtValue();
754 
755  // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
756  // 32 leading zeros and C3 trailing zeros.
757  if (isShiftedMask_64(Mask) && N0.hasOneUse()) {
758  unsigned XLen = Subtarget->getXLen();
759  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(Mask));
760  unsigned TrailingZeros = countTrailingZeros(Mask);
761  if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
762  SDNode *SRLIW = CurDAG->getMachineNode(
763  RISCV::SRLIW, DL, VT, N0->getOperand(0),
764  CurDAG->getTargetConstant(TrailingZeros, DL, VT));
765  SDNode *SLLI = CurDAG->getMachineNode(
766  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
767  CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT));
768  ReplaceNode(Node, SLLI);
769  return;
770  }
771  }
772 
773  // Optimize (srl (and X, C2), C) ->
774  // (srli (slli X, (XLen-C3), (XLen-C3) + C)
775  // Where C2 is a mask with C3 trailing ones.
776  // Taking into account that the C2 may have had lower bits unset by
777  // SimplifyDemandedBits. This avoids materializing the C2 immediate.
778  // This pattern occurs when type legalizing right shifts for types with
779  // less than XLen bits.
780  Mask |= maskTrailingOnes<uint64_t>(ShAmt);
781  if (!isMask_64(Mask))
782  break;
783  unsigned TrailingOnes = countTrailingOnes(Mask);
784  if (ShAmt >= TrailingOnes)
785  break;
786  // If the mask has 32 trailing ones, use SRLIW.
787  if (TrailingOnes == 32) {
788  SDNode *SRLIW =
789  CurDAG->getMachineNode(RISCV::SRLIW, DL, VT, N0->getOperand(0),
790  CurDAG->getTargetConstant(ShAmt, DL, VT));
791  ReplaceNode(Node, SRLIW);
792  return;
793  }
794 
795  // Only do the remaining transforms if the shift has one use.
796  if (!N0.hasOneUse())
797  break;
798 
799  // If C2 is (1 << ShAmt) use bexti if possible.
800  if (Subtarget->hasStdExtZbs() && ShAmt + 1 == TrailingOnes) {
801  SDNode *BEXTI =
802  CurDAG->getMachineNode(RISCV::BEXTI, DL, VT, N0->getOperand(0),
803  CurDAG->getTargetConstant(ShAmt, DL, VT));
804  ReplaceNode(Node, BEXTI);
805  return;
806  }
807  unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
808  SDNode *SLLI =
809  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
810  CurDAG->getTargetConstant(LShAmt, DL, VT));
811  SDNode *SRLI = CurDAG->getMachineNode(
812  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
813  CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
814  ReplaceNode(Node, SRLI);
815  return;
816  }
817  case ISD::SRA: {
818  // Optimize (sra (sext_inreg X, i16), C) ->
819  // (srai (slli X, (XLen-16), (XLen-16) + C)
820  // And (sra (sext_inreg X, i8), C) ->
821  // (srai (slli X, (XLen-8), (XLen-8) + C)
822  // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
823  // This transform matches the code we get without Zbb. The shifts are more
824  // compressible, and this can help expose CSE opportunities in the sdiv by
825  // constant optimization.
826  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
827  if (!N1C)
828  break;
829  SDValue N0 = Node->getOperand(0);
830  if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
831  break;
832  unsigned ShAmt = N1C->getZExtValue();
833  unsigned ExtSize =
834  cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
835  // ExtSize of 32 should use sraiw via tablegen pattern.
836  if (ExtSize >= 32 || ShAmt >= ExtSize)
837  break;
838  unsigned LShAmt = Subtarget->getXLen() - ExtSize;
839  SDNode *SLLI =
840  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
841  CurDAG->getTargetConstant(LShAmt, DL, VT));
842  SDNode *SRAI = CurDAG->getMachineNode(
843  RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
844  CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
845  ReplaceNode(Node, SRAI);
846  return;
847  }
848  case ISD::OR:
849  case ISD::XOR:
850  if (tryShrinkShlLogicImm(Node))
851  return;
852 
853  break;
854  case ISD::AND: {
855  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
856  if (!N1C)
857  break;
858 
859  SDValue N0 = Node->getOperand(0);
860 
861  bool LeftShift = N0.getOpcode() == ISD::SHL;
862  if (LeftShift || N0.getOpcode() == ISD::SRL) {
863  auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
864  if (!C)
865  break;
866  unsigned C2 = C->getZExtValue();
867  unsigned XLen = Subtarget->getXLen();
868  assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
869 
870  uint64_t C1 = N1C->getZExtValue();
871 
872  // Keep track of whether this is a c.andi. If we can't use c.andi, the
873  // shift pair might offer more compression opportunities.
874  // TODO: We could check for C extension here, but we don't have many lit
875  // tests with the C extension enabled so not checking gets better
876  // coverage.
877  // TODO: What if ANDI faster than shift?
878  bool IsCANDI = isInt<6>(N1C->getSExtValue());
879 
880  // Clear irrelevant bits in the mask.
881  if (LeftShift)
882  C1 &= maskTrailingZeros<uint64_t>(C2);
883  else
884  C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
885 
886  // Some transforms should only be done if the shift has a single use or
887  // the AND would become (srli (slli X, 32), 32)
888  bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
889 
890  SDValue X = N0.getOperand(0);
891 
892  // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
893  // with c3 leading zeros.
894  if (!LeftShift && isMask_64(C1)) {
895  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
896  if (C2 < Leading) {
897  // If the number of leading zeros is C2+32 this can be SRLIW.
898  if (C2 + 32 == Leading) {
899  SDNode *SRLIW = CurDAG->getMachineNode(
900  RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT));
901  ReplaceNode(Node, SRLIW);
902  return;
903  }
904 
905  // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
906  // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
907  //
908  // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
909  // legalized and goes through DAG combine.
910  if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
911  X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
912  cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) {
913  SDNode *SRAIW =
914  CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0),
915  CurDAG->getTargetConstant(31, DL, VT));
916  SDNode *SRLIW = CurDAG->getMachineNode(
917  RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0),
918  CurDAG->getTargetConstant(Leading - 32, DL, VT));
919  ReplaceNode(Node, SRLIW);
920  return;
921  }
922 
923  // (srli (slli x, c3-c2), c3).
924  // Skip if we could use (zext.w (sraiw X, C2)).
925  bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
926  X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
927  cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32;
928  // Also Skip if we can use bexti.
929  Skip |= Subtarget->hasStdExtZbs() && Leading == XLen - 1;
930  if (OneUseOrZExtW && !Skip) {
931  SDNode *SLLI = CurDAG->getMachineNode(
932  RISCV::SLLI, DL, VT, X,
933  CurDAG->getTargetConstant(Leading - C2, DL, VT));
934  SDNode *SRLI = CurDAG->getMachineNode(
935  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
936  CurDAG->getTargetConstant(Leading, DL, VT));
937  ReplaceNode(Node, SRLI);
938  return;
939  }
940  }
941  }
942 
943  // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
944  // shifted by c2 bits with c3 leading zeros.
945  if (LeftShift && isShiftedMask_64(C1)) {
946  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
947 
948  if (C2 + Leading < XLen &&
949  C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) {
950  // Use slli.uw when possible.
951  if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
952  SDNode *SLLI_UW =
953  CurDAG->getMachineNode(RISCV::SLLI_UW, DL, VT, X,
954  CurDAG->getTargetConstant(C2, DL, VT));
955  ReplaceNode(Node, SLLI_UW);
956  return;
957  }
958 
959  // (srli (slli c2+c3), c3)
960  if (OneUseOrZExtW && !IsCANDI) {
961  SDNode *SLLI = CurDAG->getMachineNode(
962  RISCV::SLLI, DL, VT, X,
963  CurDAG->getTargetConstant(C2 + Leading, DL, VT));
964  SDNode *SRLI = CurDAG->getMachineNode(
965  RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
966  CurDAG->getTargetConstant(Leading, DL, VT));
967  ReplaceNode(Node, SRLI);
968  return;
969  }
970  }
971  }
972 
973  // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
974  // shifted mask with c2 leading zeros and c3 trailing zeros.
975  if (!LeftShift && isShiftedMask_64(C1)) {
976  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
977  unsigned Trailing = countTrailingZeros(C1);
978  if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
979  !IsCANDI) {
980  unsigned SrliOpc = RISCV::SRLI;
981  // If the input is zexti32 we should use SRLIW.
982  if (X.getOpcode() == ISD::AND &&
983  isa<ConstantSDNode>(X.getOperand(1)) &&
984  X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) {
985  SrliOpc = RISCV::SRLIW;
986  X = X.getOperand(0);
987  }
988  SDNode *SRLI = CurDAG->getMachineNode(
989  SrliOpc, DL, VT, X,
990  CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
991  SDNode *SLLI = CurDAG->getMachineNode(
992  RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
993  CurDAG->getTargetConstant(Trailing, DL, VT));
994  ReplaceNode(Node, SLLI);
995  return;
996  }
997  // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
998  if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
999  OneUseOrZExtW && !IsCANDI) {
1000  SDNode *SRLIW = CurDAG->getMachineNode(
1001  RISCV::SRLIW, DL, VT, X,
1002  CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1003  SDNode *SLLI = CurDAG->getMachineNode(
1004  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1005  CurDAG->getTargetConstant(Trailing, DL, VT));
1006  ReplaceNode(Node, SLLI);
1007  return;
1008  }
1009  }
1010 
1011  // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1012  // shifted mask with no leading zeros and c3 trailing zeros.
1013  if (LeftShift && isShiftedMask_64(C1)) {
1014  unsigned Leading = XLen - (64 - countLeadingZeros(C1));
1015  unsigned Trailing = countTrailingZeros(C1);
1016  if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1017  SDNode *SRLI = CurDAG->getMachineNode(
1018  RISCV::SRLI, DL, VT, X,
1019  CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1020  SDNode *SLLI = CurDAG->getMachineNode(
1021  RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1022  CurDAG->getTargetConstant(Trailing, DL, VT));
1023  ReplaceNode(Node, SLLI);
1024  return;
1025  }
1026  // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1027  if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1028  SDNode *SRLIW = CurDAG->getMachineNode(
1029  RISCV::SRLIW, DL, VT, X,
1030  CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1031  SDNode *SLLI = CurDAG->getMachineNode(
1032  RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1033  CurDAG->getTargetConstant(Trailing, DL, VT));
1034  ReplaceNode(Node, SLLI);
1035  return;
1036  }
1037  }
1038  }
1039 
1040  if (tryShrinkShlLogicImm(Node))
1041  return;
1042 
1043  break;
1044  }
1045  case ISD::MUL: {
1046  // Special case for calculating (mul (and X, C2), C1) where the full product
1047  // fits in XLen bits. We can shift X left by the number of leading zeros in
1048  // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1049  // product has XLen trailing zeros, putting it in the output of MULHU. This
1050  // can avoid materializing a constant in a register for C2.
1051 
1052  // RHS should be a constant.
1053  auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1054  if (!N1C || !N1C->hasOneUse())
1055  break;
1056 
1057  // LHS should be an AND with constant.
1058  SDValue N0 = Node->getOperand(0);
1059  if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1060  break;
1061 
1062  uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
1063 
1064  // Constant should be a mask.
1065  if (!isMask_64(C2))
1066  break;
1067 
1068  // If this can be an ANDI, ZEXT.H or ZEXT.W, don't do this if the ANDI/ZEXT
1069  // has multiple users or the constant is a simm12. This prevents inserting
1070  // a shift and still have uses of the AND/ZEXT. Shifting a simm12 will
1071  // likely make it more costly to materialize. Otherwise, using a SLLI
1072  // might allow it to be compressed.
1073  bool IsANDIOrZExt =
1074  isInt<12>(C2) ||
1075  (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb()) ||
1076  (C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba());
1077  if (IsANDIOrZExt && (isInt<12>(N1C->getSExtValue()) || !N0.hasOneUse()))
1078  break;
1079 
1080  // We need to shift left the AND input and C1 by a total of XLen bits.
1081 
1082  // How far left do we need to shift the AND input?
1083  unsigned XLen = Subtarget->getXLen();
1084  unsigned LeadingZeros = XLen - (64 - countLeadingZeros(C2));
1085 
1086  // The constant gets shifted by the remaining amount unless that would
1087  // shift bits out.
1088  uint64_t C1 = N1C->getZExtValue();
1089  unsigned ConstantShift = XLen - LeadingZeros;
1090  if (ConstantShift > (XLen - (64 - countLeadingZeros(C1))))
1091  break;
1092 
1093  uint64_t ShiftedC1 = C1 << ConstantShift;
1094  // If this RV32, we need to sign extend the constant.
1095  if (XLen == 32)
1096  ShiftedC1 = SignExtend64<32>(ShiftedC1);
1097 
1098  // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1099  SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget);
1100  SDNode *SLLI =
1101  CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1102  CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1104  SDValue(SLLI, 0), SDValue(Imm, 0));
1105  ReplaceNode(Node, MULHU);
1106  return;
1107  }
1108  case ISD::INTRINSIC_WO_CHAIN: {
1109  unsigned IntNo = Node->getConstantOperandVal(0);
1110  switch (IntNo) {
1111  // By default we do not custom select any intrinsic.
1112  default:
1113  break;
1114  case Intrinsic::riscv_vmsgeu:
1115  case Intrinsic::riscv_vmsge: {
1116  SDValue Src1 = Node->getOperand(1);
1117  SDValue Src2 = Node->getOperand(2);
1118  bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
1119  bool IsCmpUnsignedZero = false;
1120  // Only custom select scalar second operand.
1121  if (Src2.getValueType() != XLenVT)
1122  break;
1123  // Small constants are handled with patterns.
1124  if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1125  int64_t CVal = C->getSExtValue();
1126  if (CVal >= -15 && CVal <= 16) {
1127  if (!IsUnsigned || CVal != 0)
1128  break;
1129  IsCmpUnsignedZero = true;
1130  }
1131  }
1132  MVT Src1VT = Src1.getSimpleValueType();
1133  unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode;
1134  switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1135  default:
1136  llvm_unreachable("Unexpected LMUL!");
1137 #define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \
1138  case RISCVII::VLMUL::lmulenum: \
1139  VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1140  : RISCV::PseudoVMSLT_VX_##suffix; \
1141  VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
1142  VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \
1143  break;
1151 #undef CASE_VMSLT_VMNAND_VMSET_OPCODES
1152  }
1154  Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1155  SDValue VL;
1156  selectVLOp(Node->getOperand(3), VL);
1157 
1158  // If vmsgeu with 0 immediate, expand it to vmset.
1159  if (IsCmpUnsignedZero) {
1160  ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW));
1161  return;
1162  }
1163 
1164  // Expand to
1165  // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
1166  SDValue Cmp = SDValue(
1167  CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1168  0);
1169  ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
1170  {Cmp, Cmp, VL, SEW}));
1171  return;
1172  }
1173  case Intrinsic::riscv_vmsgeu_mask:
1174  case Intrinsic::riscv_vmsge_mask: {
1175  SDValue Src1 = Node->getOperand(2);
1176  SDValue Src2 = Node->getOperand(3);
1177  bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
1178  bool IsCmpUnsignedZero = false;
1179  // Only custom select scalar second operand.
1180  if (Src2.getValueType() != XLenVT)
1181  break;
1182  // Small constants are handled with patterns.
1183  if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1184  int64_t CVal = C->getSExtValue();
1185  if (CVal >= -15 && CVal <= 16) {
1186  if (!IsUnsigned || CVal != 0)
1187  break;
1188  IsCmpUnsignedZero = true;
1189  }
1190  }
1191  MVT Src1VT = Src1.getSimpleValueType();
1192  unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
1193  VMOROpcode;
1194  switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1195  default:
1196  llvm_unreachable("Unexpected LMUL!");
1197 #define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \
1198  case RISCVII::VLMUL::lmulenum: \
1199  VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1200  : RISCV::PseudoVMSLT_VX_##suffix; \
1201  VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
1202  : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
1203  break;
1204  CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1)
1205  CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2)
1206  CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4)
1208  CASE_VMSLT_OPCODES(LMUL_2, M2, B16)
1209  CASE_VMSLT_OPCODES(LMUL_4, M4, B32)
1210  CASE_VMSLT_OPCODES(LMUL_8, M8, B64)
1211 #undef CASE_VMSLT_OPCODES
1212  }
1213  // Mask operations use the LMUL from the mask type.
1214  switch (RISCVTargetLowering::getLMUL(VT)) {
1215  default:
1216  llvm_unreachable("Unexpected LMUL!");
1217 #define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
1218  case RISCVII::VLMUL::lmulenum: \
1219  VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
1220  VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
1221  VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
1222  break;
1230 #undef CASE_VMXOR_VMANDN_VMOR_OPCODES
1231  }
1233  Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1234  SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
1235  SDValue VL;
1236  selectVLOp(Node->getOperand(5), VL);
1237  SDValue MaskedOff = Node->getOperand(1);
1238  SDValue Mask = Node->getOperand(4);
1239 
1240  // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff.
1241  if (IsCmpUnsignedZero) {
1242  // We don't need vmor if the MaskedOff and the Mask are the same
1243  // value.
1244  if (Mask == MaskedOff) {
1245  ReplaceUses(Node, Mask.getNode());
1246  return;
1247  }
1248  ReplaceNode(Node,
1249  CurDAG->getMachineNode(VMOROpcode, DL, VT,
1250  {Mask, MaskedOff, VL, MaskSEW}));
1251  return;
1252  }
1253 
1254  // If the MaskedOff value and the Mask are the same value use
1255  // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
1256  // This avoids needing to copy v0 to vd before starting the next sequence.
1257  if (Mask == MaskedOff) {
1258  SDValue Cmp = SDValue(
1259  CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1260  0);
1261  ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT,
1262  {Mask, Cmp, VL, MaskSEW}));
1263  return;
1264  }
1265 
1266  // Mask needs to be copied to V0.
1268  RISCV::V0, Mask, SDValue());
1269  SDValue Glue = Chain.getValue(1);
1270  SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
1271 
1272  // Otherwise use
1273  // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
1274  // The result is mask undisturbed.
1275  // We use the same instructions to emulate mask agnostic behavior, because
1276  // the agnostic result can be either undisturbed or all 1.
1277  SDValue Cmp = SDValue(
1278  CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
1279  {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
1280  0);
1281  // vmxor.mm vd, vd, v0 is used to update active value.
1282  ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
1283  {Cmp, Mask, VL, MaskSEW}));
1284  return;
1285  }
1286  case Intrinsic::riscv_vsetvli_opt:
1287  case Intrinsic::riscv_vsetvlimax_opt:
1288  return selectVSETVLI(Node);
1289  }
1290  break;
1291  }
1292  case ISD::INTRINSIC_W_CHAIN: {
1293  unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1294  switch (IntNo) {
1295  // By default we do not custom select any intrinsic.
1296  default:
1297  break;
1298  case Intrinsic::riscv_vsetvli:
1299  case Intrinsic::riscv_vsetvlimax:
1300  return selectVSETVLI(Node);
1301  case Intrinsic::riscv_vlseg2:
1302  case Intrinsic::riscv_vlseg3:
1303  case Intrinsic::riscv_vlseg4:
1304  case Intrinsic::riscv_vlseg5:
1305  case Intrinsic::riscv_vlseg6:
1306  case Intrinsic::riscv_vlseg7:
1307  case Intrinsic::riscv_vlseg8: {
1308  selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1309  return;
1310  }
1311  case Intrinsic::riscv_vlseg2_mask:
1312  case Intrinsic::riscv_vlseg3_mask:
1313  case Intrinsic::riscv_vlseg4_mask:
1314  case Intrinsic::riscv_vlseg5_mask:
1315  case Intrinsic::riscv_vlseg6_mask:
1316  case Intrinsic::riscv_vlseg7_mask:
1317  case Intrinsic::riscv_vlseg8_mask: {
1318  selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1319  return;
1320  }
1321  case Intrinsic::riscv_vlsseg2:
1322  case Intrinsic::riscv_vlsseg3:
1323  case Intrinsic::riscv_vlsseg4:
1324  case Intrinsic::riscv_vlsseg5:
1325  case Intrinsic::riscv_vlsseg6:
1326  case Intrinsic::riscv_vlsseg7:
1327  case Intrinsic::riscv_vlsseg8: {
1328  selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1329  return;
1330  }
1331  case Intrinsic::riscv_vlsseg2_mask:
1332  case Intrinsic::riscv_vlsseg3_mask:
1333  case Intrinsic::riscv_vlsseg4_mask:
1334  case Intrinsic::riscv_vlsseg5_mask:
1335  case Intrinsic::riscv_vlsseg6_mask:
1336  case Intrinsic::riscv_vlsseg7_mask:
1337  case Intrinsic::riscv_vlsseg8_mask: {
1338  selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1339  return;
1340  }
1341  case Intrinsic::riscv_vloxseg2:
1342  case Intrinsic::riscv_vloxseg3:
1343  case Intrinsic::riscv_vloxseg4:
1344  case Intrinsic::riscv_vloxseg5:
1345  case Intrinsic::riscv_vloxseg6:
1346  case Intrinsic::riscv_vloxseg7:
1347  case Intrinsic::riscv_vloxseg8:
1348  selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1349  return;
1350  case Intrinsic::riscv_vluxseg2:
1351  case Intrinsic::riscv_vluxseg3:
1352  case Intrinsic::riscv_vluxseg4:
1353  case Intrinsic::riscv_vluxseg5:
1354  case Intrinsic::riscv_vluxseg6:
1355  case Intrinsic::riscv_vluxseg7:
1356  case Intrinsic::riscv_vluxseg8:
1357  selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1358  return;
1359  case Intrinsic::riscv_vloxseg2_mask:
1360  case Intrinsic::riscv_vloxseg3_mask:
1361  case Intrinsic::riscv_vloxseg4_mask:
1362  case Intrinsic::riscv_vloxseg5_mask:
1363  case Intrinsic::riscv_vloxseg6_mask:
1364  case Intrinsic::riscv_vloxseg7_mask:
1365  case Intrinsic::riscv_vloxseg8_mask:
1366  selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1367  return;
1368  case Intrinsic::riscv_vluxseg2_mask:
1369  case Intrinsic::riscv_vluxseg3_mask:
1370  case Intrinsic::riscv_vluxseg4_mask:
1371  case Intrinsic::riscv_vluxseg5_mask:
1372  case Intrinsic::riscv_vluxseg6_mask:
1373  case Intrinsic::riscv_vluxseg7_mask:
1374  case Intrinsic::riscv_vluxseg8_mask:
1375  selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1376  return;
1377  case Intrinsic::riscv_vlseg8ff:
1378  case Intrinsic::riscv_vlseg7ff:
1379  case Intrinsic::riscv_vlseg6ff:
1380  case Intrinsic::riscv_vlseg5ff:
1381  case Intrinsic::riscv_vlseg4ff:
1382  case Intrinsic::riscv_vlseg3ff:
1383  case Intrinsic::riscv_vlseg2ff: {
1384  selectVLSEGFF(Node, /*IsMasked*/ false);
1385  return;
1386  }
1387  case Intrinsic::riscv_vlseg8ff_mask:
1388  case Intrinsic::riscv_vlseg7ff_mask:
1389  case Intrinsic::riscv_vlseg6ff_mask:
1390  case Intrinsic::riscv_vlseg5ff_mask:
1391  case Intrinsic::riscv_vlseg4ff_mask:
1392  case Intrinsic::riscv_vlseg3ff_mask:
1393  case Intrinsic::riscv_vlseg2ff_mask: {
1394  selectVLSEGFF(Node, /*IsMasked*/ true);
1395  return;
1396  }
1397  case Intrinsic::riscv_vloxei:
1398  case Intrinsic::riscv_vloxei_mask:
1399  case Intrinsic::riscv_vluxei:
1400  case Intrinsic::riscv_vluxei_mask: {
1401  bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
1402  IntNo == Intrinsic::riscv_vluxei_mask;
1403  bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
1404  IntNo == Intrinsic::riscv_vloxei_mask;
1405 
1406  MVT VT = Node->getSimpleValueType(0);
1407  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1408 
1409  unsigned CurOp = 2;
1410  // Masked intrinsic only have TU version pseduo instructions.
1411  bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef();
1413  if (IsTU)
1414  Operands.push_back(Node->getOperand(CurOp++));
1415  else
1416  // Skip the undef passthru operand for nomask TA version pseudo
1417  CurOp++;
1418 
1419  MVT IndexVT;
1420  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1421  /*IsStridedOrIndexed*/ true, Operands,
1422  /*IsLoad=*/true, &IndexVT);
1423 
1425  "Element count mismatch");
1426 
1428  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1429  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1430  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1431  report_fatal_error("The V extension does not support EEW=64 for index "
1432  "values when XLEN=32");
1433  }
1434  const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1435  IsMasked, IsTU, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1436  static_cast<unsigned>(IndexLMUL));
1437  MachineSDNode *Load =
1438  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1439 
1440  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1441  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1442 
1443  ReplaceNode(Node, Load);
1444  return;
1445  }
1446  case Intrinsic::riscv_vlm:
1447  case Intrinsic::riscv_vle:
1448  case Intrinsic::riscv_vle_mask:
1449  case Intrinsic::riscv_vlse:
1450  case Intrinsic::riscv_vlse_mask: {
1451  bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
1452  IntNo == Intrinsic::riscv_vlse_mask;
1453  bool IsStrided =
1454  IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
1455 
1456  MVT VT = Node->getSimpleValueType(0);
1457  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1458 
1459  unsigned CurOp = 2;
1460  // The riscv_vlm intrinsic are always tail agnostic and no passthru operand.
1461  bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
1462  // Masked intrinsic only have TU version pseduo instructions.
1463  bool IsTU = HasPassthruOperand &&
1464  (IsMasked || !Node->getOperand(CurOp).isUndef());
1466  if (IsTU)
1467  Operands.push_back(Node->getOperand(CurOp++));
1468  else if (HasPassthruOperand)
1469  // Skip the undef passthru operand for nomask TA version pseudo
1470  CurOp++;
1471 
1472  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1473  Operands, /*IsLoad=*/true);
1474 
1476  const RISCV::VLEPseudo *P =
1477  RISCV::getVLEPseudo(IsMasked, IsTU, IsStrided, /*FF*/ false, Log2SEW,
1478  static_cast<unsigned>(LMUL));
1479  MachineSDNode *Load =
1480  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1481 
1482  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1483  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1484 
1485  ReplaceNode(Node, Load);
1486  return;
1487  }
1488  case Intrinsic::riscv_vleff:
1489  case Intrinsic::riscv_vleff_mask: {
1490  bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
1491 
1492  MVT VT = Node->getSimpleValueType(0);
1493  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1494 
1495  unsigned CurOp = 2;
1496  // Masked intrinsic only have TU version pseduo instructions.
1497  bool IsTU = IsMasked || !Node->getOperand(CurOp).isUndef();
1499  if (IsTU)
1500  Operands.push_back(Node->getOperand(CurOp++));
1501  else
1502  // Skip the undef passthru operand for nomask TA version pseudo
1503  CurOp++;
1504 
1505  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1506  /*IsStridedOrIndexed*/ false, Operands,
1507  /*IsLoad=*/true);
1508 
1510  const RISCV::VLEPseudo *P =
1511  RISCV::getVLEPseudo(IsMasked, IsTU, /*Strided*/ false, /*FF*/ true,
1512  Log2SEW, static_cast<unsigned>(LMUL));
1514  P->Pseudo, DL, Node->getVTList(), Operands);
1515  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1516  CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1517 
1518  ReplaceNode(Node, Load);
1519  return;
1520  }
1521  }
1522  break;
1523  }
1524  case ISD::INTRINSIC_VOID: {
1525  unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1526  switch (IntNo) {
1527  case Intrinsic::riscv_vsseg2:
1528  case Intrinsic::riscv_vsseg3:
1529  case Intrinsic::riscv_vsseg4:
1530  case Intrinsic::riscv_vsseg5:
1531  case Intrinsic::riscv_vsseg6:
1532  case Intrinsic::riscv_vsseg7:
1533  case Intrinsic::riscv_vsseg8: {
1534  selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1535  return;
1536  }
1537  case Intrinsic::riscv_vsseg2_mask:
1538  case Intrinsic::riscv_vsseg3_mask:
1539  case Intrinsic::riscv_vsseg4_mask:
1540  case Intrinsic::riscv_vsseg5_mask:
1541  case Intrinsic::riscv_vsseg6_mask:
1542  case Intrinsic::riscv_vsseg7_mask:
1543  case Intrinsic::riscv_vsseg8_mask: {
1544  selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1545  return;
1546  }
1547  case Intrinsic::riscv_vssseg2:
1548  case Intrinsic::riscv_vssseg3:
1549  case Intrinsic::riscv_vssseg4:
1550  case Intrinsic::riscv_vssseg5:
1551  case Intrinsic::riscv_vssseg6:
1552  case Intrinsic::riscv_vssseg7:
1553  case Intrinsic::riscv_vssseg8: {
1554  selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1555  return;
1556  }
1557  case Intrinsic::riscv_vssseg2_mask:
1558  case Intrinsic::riscv_vssseg3_mask:
1559  case Intrinsic::riscv_vssseg4_mask:
1560  case Intrinsic::riscv_vssseg5_mask:
1561  case Intrinsic::riscv_vssseg6_mask:
1562  case Intrinsic::riscv_vssseg7_mask:
1563  case Intrinsic::riscv_vssseg8_mask: {
1564  selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1565  return;
1566  }
1567  case Intrinsic::riscv_vsoxseg2:
1568  case Intrinsic::riscv_vsoxseg3:
1569  case Intrinsic::riscv_vsoxseg4:
1570  case Intrinsic::riscv_vsoxseg5:
1571  case Intrinsic::riscv_vsoxseg6:
1572  case Intrinsic::riscv_vsoxseg7:
1573  case Intrinsic::riscv_vsoxseg8:
1574  selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1575  return;
1576  case Intrinsic::riscv_vsuxseg2:
1577  case Intrinsic::riscv_vsuxseg3:
1578  case Intrinsic::riscv_vsuxseg4:
1579  case Intrinsic::riscv_vsuxseg5:
1580  case Intrinsic::riscv_vsuxseg6:
1581  case Intrinsic::riscv_vsuxseg7:
1582  case Intrinsic::riscv_vsuxseg8:
1583  selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1584  return;
1585  case Intrinsic::riscv_vsoxseg2_mask:
1586  case Intrinsic::riscv_vsoxseg3_mask:
1587  case Intrinsic::riscv_vsoxseg4_mask:
1588  case Intrinsic::riscv_vsoxseg5_mask:
1589  case Intrinsic::riscv_vsoxseg6_mask:
1590  case Intrinsic::riscv_vsoxseg7_mask:
1591  case Intrinsic::riscv_vsoxseg8_mask:
1592  selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1593  return;
1594  case Intrinsic::riscv_vsuxseg2_mask:
1595  case Intrinsic::riscv_vsuxseg3_mask:
1596  case Intrinsic::riscv_vsuxseg4_mask:
1597  case Intrinsic::riscv_vsuxseg5_mask:
1598  case Intrinsic::riscv_vsuxseg6_mask:
1599  case Intrinsic::riscv_vsuxseg7_mask:
1600  case Intrinsic::riscv_vsuxseg8_mask:
1601  selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1602  return;
1603  case Intrinsic::riscv_vsoxei:
1604  case Intrinsic::riscv_vsoxei_mask:
1605  case Intrinsic::riscv_vsuxei:
1606  case Intrinsic::riscv_vsuxei_mask: {
1607  bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
1608  IntNo == Intrinsic::riscv_vsuxei_mask;
1609  bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
1610  IntNo == Intrinsic::riscv_vsoxei_mask;
1611 
1612  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1613  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1614 
1615  unsigned CurOp = 2;
1617  Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1618 
1619  MVT IndexVT;
1620  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1621  /*IsStridedOrIndexed*/ true, Operands,
1622  /*IsLoad=*/false, &IndexVT);
1623 
1625  "Element count mismatch");
1626 
1628  RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1629  unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1630  if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1631  report_fatal_error("The V extension does not support EEW=64 for index "
1632  "values when XLEN=32");
1633  }
1634  const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1635  IsMasked, /*TU*/ false, IsOrdered, IndexLog2EEW,
1636  static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL));
1637  MachineSDNode *Store =
1638  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1639 
1640  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1641  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1642 
1643  ReplaceNode(Node, Store);
1644  return;
1645  }
1646  case Intrinsic::riscv_vsm:
1647  case Intrinsic::riscv_vse:
1648  case Intrinsic::riscv_vse_mask:
1649  case Intrinsic::riscv_vsse:
1650  case Intrinsic::riscv_vsse_mask: {
1651  bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
1652  IntNo == Intrinsic::riscv_vsse_mask;
1653  bool IsStrided =
1654  IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
1655 
1656  MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1657  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1658 
1659  unsigned CurOp = 2;
1661  Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1662 
1663  addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1664  Operands);
1665 
1667  const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1668  IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1669  MachineSDNode *Store =
1670  CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1671  if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1672  CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1673 
1674  ReplaceNode(Node, Store);
1675  return;
1676  }
1677  }
1678  break;
1679  }
1680  case ISD::BITCAST: {
1681  MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1682  // Just drop bitcasts between vectors if both are fixed or both are
1683  // scalable.
1684  if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1685  (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1686  ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1687  CurDAG->RemoveDeadNode(Node);
1688  return;
1689  }
1690  break;
1691  }
1692  case ISD::INSERT_SUBVECTOR: {
1693  SDValue V = Node->getOperand(0);
1694  SDValue SubV = Node->getOperand(1);
1695  SDLoc DL(SubV);
1696  auto Idx = Node->getConstantOperandVal(2);
1697  MVT SubVecVT = SubV.getSimpleValueType();
1698 
1699  const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1700  MVT SubVecContainerVT = SubVecVT;
1701  // Establish the correct scalable-vector types for any fixed-length type.
1702  if (SubVecVT.isFixedLengthVector())
1703  SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
1704  if (VT.isFixedLengthVector())
1705  VT = TLI.getContainerForFixedLengthVector(VT);
1706 
1707  const auto *TRI = Subtarget->getRegisterInfo();
1708  unsigned SubRegIdx;
1709  std::tie(SubRegIdx, Idx) =
1711  VT, SubVecContainerVT, Idx, TRI);
1712 
1713  // If the Idx hasn't been completely eliminated then this is a subvector
1714  // insert which doesn't naturally align to a vector register. These must
1715  // be handled using instructions to manipulate the vector registers.
1716  if (Idx != 0)
1717  break;
1718 
1719  RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
1720  bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
1721  SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
1722  SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
1723  (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
1724  assert((!IsSubVecPartReg || V.isUndef()) &&
1725  "Expecting lowering to have created legal INSERT_SUBVECTORs when "
1726  "the subvector is smaller than a full-sized register");
1727 
1728  // If we haven't set a SubRegIdx, then we must be going between
1729  // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1730  if (SubRegIdx == RISCV::NoSubRegister) {
1731  unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
1733  InRegClassID &&
1734  "Unexpected subvector extraction");
1735  SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1736  SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
1737  DL, VT, SubV, RC);
1738  ReplaceNode(Node, NewNode);
1739  return;
1740  }
1741 
1742  SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
1743  ReplaceNode(Node, Insert.getNode());
1744  return;
1745  }
1746  case ISD::EXTRACT_SUBVECTOR: {
1747  SDValue V = Node->getOperand(0);
1748  auto Idx = Node->getConstantOperandVal(1);
1749  MVT InVT = V.getSimpleValueType();
1750  SDLoc DL(V);
1751 
1752  const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
1753  MVT SubVecContainerVT = VT;
1754  // Establish the correct scalable-vector types for any fixed-length type.
1755  if (VT.isFixedLengthVector())
1756  SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
1757  if (InVT.isFixedLengthVector())
1758  InVT = TLI.getContainerForFixedLengthVector(InVT);
1759 
1760  const auto *TRI = Subtarget->getRegisterInfo();
1761  unsigned SubRegIdx;
1762  std::tie(SubRegIdx, Idx) =
1764  InVT, SubVecContainerVT, Idx, TRI);
1765 
1766  // If the Idx hasn't been completely eliminated then this is a subvector
1767  // extract which doesn't naturally align to a vector register. These must
1768  // be handled using instructions to manipulate the vector registers.
1769  if (Idx != 0)
1770  break;
1771 
1772  // If we haven't set a SubRegIdx, then we must be going between
1773  // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
1774  if (SubRegIdx == RISCV::NoSubRegister) {
1775  unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
1777  InRegClassID &&
1778  "Unexpected subvector extraction");
1779  SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
1780  SDNode *NewNode =
1781  CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
1782  ReplaceNode(Node, NewNode);
1783  return;
1784  }
1785 
1786  SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
1787  ReplaceNode(Node, Extract.getNode());
1788  return;
1789  }
1790  case RISCVISD::VMV_S_X_VL:
1791  case RISCVISD::VFMV_S_F_VL:
1792  case RISCVISD::VMV_V_X_VL:
1793  case RISCVISD::VFMV_V_F_VL: {
1794  // Only if we have optimized zero-stride vector load.
1795  if (!Subtarget->hasOptimizedZeroStrideLoad())
1796  break;
1797 
1798  // Try to match splat of a scalar load to a strided load with stride of x0.
1799  bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
1800  Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
1801  if (!Node->getOperand(0).isUndef())
1802  break;
1803  SDValue Src = Node->getOperand(1);
1804  auto *Ld = dyn_cast<LoadSDNode>(Src);
1805  if (!Ld)
1806  break;
1807  EVT MemVT = Ld->getMemoryVT();
1808  // The memory VT should be the same size as the element type.
1809  if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
1810  break;
1811  if (!IsProfitableToFold(Src, Node, Node) ||
1812  !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
1813  break;
1814 
1815  SDValue VL;
1816  if (IsScalarMove) {
1817  // We could deal with more VL if we update the VSETVLI insert pass to
1818  // avoid introducing more VSETVLI.
1819  if (!isOneConstant(Node->getOperand(2)))
1820  break;
1821  selectVLOp(Node->getOperand(2), VL);
1822  } else
1823  selectVLOp(Node->getOperand(2), VL);
1824 
1825  unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1827 
1828  SDValue Operands[] = {Ld->getBasePtr(),
1829  CurDAG->getRegister(RISCV::X0, XLenVT), VL, SEW,
1830  Ld->getChain()};
1831 
1833  const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
1834  /*IsMasked*/ false, /*IsTU*/ false, /*IsStrided*/ true, /*FF*/ false,
1835  Log2SEW, static_cast<unsigned>(LMUL));
1836  MachineSDNode *Load =
1837  CurDAG->getMachineNode(P->Pseudo, DL, {VT, MVT::Other}, Operands);
1838  // Update the chain.
1839  ReplaceUses(Src.getValue(1), SDValue(Load, 1));
1840  // Record the mem-refs
1841  CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
1842  // Replace the splat with the vlse.
1843  ReplaceNode(Node, Load);
1844  return;
1845  }
1846  }
1847 
1848  // Select the default instruction.
1849  SelectCode(Node);
1850 }
1851 
1853  const SDValue &Op, unsigned ConstraintID, std::vector<SDValue> &OutOps) {
1854  switch (ConstraintID) {
1856  // We just support simple memory operands that have a single address
1857  // operand and need no special handling.
1858  OutOps.push_back(Op);
1859  return false;
1861  OutOps.push_back(Op);
1862  return false;
1863  default:
1864  break;
1865  }
1866 
1867  return true;
1868 }
1869 
1871  SDValue &Offset) {
1872  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
1873  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
1874  Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT());
1875  return true;
1876  }
1877 
1878  return false;
1879 }
1880 
1881 // Select a frame index and an optional immediate offset from an ADD or OR.
1883  SDValue &Offset) {
1884  if (SelectAddrFrameIndex(Addr, Base, Offset))
1885  return true;
1886 
1888  return false;
1889 
1890  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
1891  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1892  if (isInt<12>(CVal)) {
1893  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
1894  Subtarget->getXLenVT());
1895  Offset = CurDAG->getTargetConstant(CVal, SDLoc(Addr),
1896  Subtarget->getXLenVT());
1897  return true;
1898  }
1899  }
1900 
1901  return false;
1902 }
1903 
1904 // Fold constant addresses.
1905 static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
1906  const MVT VT, const RISCVSubtarget *Subtarget,
1907  SDValue Addr, SDValue &Base, SDValue &Offset) {
1908  if (!isa<ConstantSDNode>(Addr))
1909  return false;
1910 
1911  int64_t CVal = cast<ConstantSDNode>(Addr)->getSExtValue();
1912 
1913  // If the constant is a simm12, we can fold the whole constant and use X0 as
1914  // the base. If the constant can be materialized with LUI+simm12, use LUI as
1915  // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
1916  int64_t Lo12 = SignExtend64<12>(CVal);
1917  int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1918  if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
1919  if (Hi) {
1920  int64_t Hi20 = (Hi >> 12) & 0xfffff;
1921  Base = SDValue(
1922  CurDAG->getMachineNode(RISCV::LUI, DL, VT,
1923  CurDAG->getTargetConstant(Hi20, DL, VT)),
1924  0);
1925  } else {
1926  Base = CurDAG->getRegister(RISCV::X0, VT);
1927  }
1928  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
1929  return true;
1930  }
1931 
1932  // Ask how constant materialization would handle this constant.
1933  RISCVMatInt::InstSeq Seq =
1934  RISCVMatInt::generateInstSeq(CVal, Subtarget->getFeatureBits());
1935 
1936  // If the last instruction would be an ADDI, we can fold its immediate and
1937  // emit the rest of the sequence as the base.
1938  if (Seq.back().Opc != RISCV::ADDI)
1939  return false;
1940  Lo12 = Seq.back().Imm;
1941 
1942  // Drop the last instruction.
1943  Seq.pop_back();
1944  assert(!Seq.empty() && "Expected more instructions in sequence");
1945 
1946  Base = SDValue(selectImmSeq(CurDAG, DL, VT, Seq), 0);
1947  Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
1948  return true;
1949 }
1950 
1951 // Is this ADD instruction only used as the base pointer of scalar loads and
1952 // stores?
1953 static bool isWorthFoldingAdd(SDValue Add) {
1954  for (auto *Use : Add->uses()) {
1955  if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
1956  Use->getOpcode() != ISD::ATOMIC_LOAD &&
1957  Use->getOpcode() != ISD::ATOMIC_STORE)
1958  return false;
1959  EVT VT = cast<MemSDNode>(Use)->getMemoryVT();
1960  if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
1961  VT != MVT::f64)
1962  return false;
1963  // Don't allow stores of the value. It must be used as the address.
1964  if (Use->getOpcode() == ISD::STORE &&
1965  cast<StoreSDNode>(Use)->getValue() == Add)
1966  return false;
1967  if (Use->getOpcode() == ISD::ATOMIC_STORE &&
1968  cast<AtomicSDNode>(Use)->getVal() == Add)
1969  return false;
1970  }
1971 
1972  return true;
1973 }
1974 
1976  SDValue &Offset) {
1977  if (SelectAddrFrameIndex(Addr, Base, Offset))
1978  return true;
1979 
1980  SDLoc DL(Addr);
1981  MVT VT = Addr.getSimpleValueType();
1982 
1983  if (Addr.getOpcode() == RISCVISD::ADD_LO) {
1984  Base = Addr.getOperand(0);
1985  Offset = Addr.getOperand(1);
1986  return true;
1987  }
1988 
1990  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
1991  if (isInt<12>(CVal)) {
1992  Base = Addr.getOperand(0);
1993  if (Base.getOpcode() == RISCVISD::ADD_LO) {
1994  SDValue LoOperand = Base.getOperand(1);
1995  if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) {
1996  // If the Lo in (ADD_LO hi, lo) is a global variable's address
1997  // (its low part, really), then we can rely on the alignment of that
1998  // variable to provide a margin of safety before low part can overflow
1999  // the 12 bits of the load/store offset. Check if CVal falls within
2000  // that margin; if so (low part + CVal) can't overflow.
2001  const DataLayout &DL = CurDAG->getDataLayout();
2002  Align Alignment = commonAlignment(
2003  GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
2004  if (CVal == 0 || Alignment > CVal) {
2005  int64_t CombinedOffset = CVal + GA->getOffset();
2006  Base = Base.getOperand(0);
2007  Offset = CurDAG->getTargetGlobalAddress(
2008  GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(),
2009  CombinedOffset, GA->getTargetFlags());
2010  return true;
2011  }
2012  }
2013  }
2014 
2015  if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
2016  Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
2017  Offset = CurDAG->getTargetConstant(CVal, DL, VT);
2018  return true;
2019  }
2020  }
2021 
2022  // Handle ADD with large immediates.
2023  if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
2024  int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2025  assert(!isInt<12>(CVal) && "simm12 not already handled?");
2026 
2027  // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
2028  // an ADDI for part of the offset and fold the rest into the load/store.
2029  // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
2030  if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
2031  int64_t Adj = CVal < 0 ? -2048 : 2047;
2032  Base = SDValue(
2033  CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
2034  CurDAG->getTargetConstant(Adj, DL, VT)),
2035  0);
2036  Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
2037  return true;
2038  }
2039 
2040  // For larger immediates, we might be able to save one instruction from
2041  // constant materialization by folding the Lo12 bits of the immediate into
2042  // the address. We should only do this if the ADD is only used by loads and
2043  // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
2044  // separately with the full materialized immediate creating extra
2045  // instructions.
2046  if (isWorthFoldingAdd(Addr) &&
2047  selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
2048  Offset)) {
2049  // Insert an ADD instruction with the materialized Hi52 bits.
2050  Base = SDValue(
2051  CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
2052  0);
2053  return true;
2054  }
2055  }
2056 
2057  if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset))
2058  return true;
2059 
2060  Base = Addr;
2061  Offset = CurDAG->getTargetConstant(0, DL, VT);
2062  return true;
2063 }
2064 
2066  SDValue &ShAmt) {
2067  ShAmt = N;
2068 
2069  // Shift instructions on RISCV only read the lower 5 or 6 bits of the shift
2070  // amount. If there is an AND on the shift amount, we can bypass it if it
2071  // doesn't affect any of those bits.
2072  if (ShAmt.getOpcode() == ISD::AND && isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2073  const APInt &AndMask = ShAmt.getConstantOperandAPInt(1);
2074 
2075  // Since the max shift amount is a power of 2 we can subtract 1 to make a
2076  // mask that covers the bits needed to represent all shift amounts.
2077  assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
2078  APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
2079 
2080  if (ShMask.isSubsetOf(AndMask)) {
2081  ShAmt = ShAmt.getOperand(0);
2082  } else {
2083  // SimplifyDemandedBits may have optimized the mask so try restoring any
2084  // bits that are known zero.
2085  KnownBits Known = CurDAG->computeKnownBits(ShAmt.getOperand(0));
2086  if (!ShMask.isSubsetOf(AndMask | Known.Zero))
2087  return true;
2088  ShAmt = ShAmt.getOperand(0);
2089  }
2090  }
2091 
2092  if (ShAmt.getOpcode() == ISD::SUB &&
2093  isa<ConstantSDNode>(ShAmt.getOperand(0))) {
2094  uint64_t Imm = ShAmt.getConstantOperandVal(0);
2095  // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2096  // generate a NEG instead of a SUB of a constant.
2097  if (Imm != 0 && Imm % ShiftWidth == 0) {
2098  SDLoc DL(ShAmt);
2099  EVT VT = ShAmt.getValueType();
2100  SDValue Zero = CurDAG->getRegister(RISCV::X0, VT);
2101  unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
2102  MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero,
2103  ShAmt.getOperand(1));
2104  ShAmt = SDValue(Neg, 0);
2105  return true;
2106  }
2107  }
2108 
2109  return true;
2110 }
2111 
2113  if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2114  cast<VTSDNode>(N.getOperand(1))->getVT() == MVT::i32) {
2115  Val = N.getOperand(0);
2116  return true;
2117  }
2118  MVT VT = N.getSimpleValueType();
2119  if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - 32)) {
2120  Val = N;
2121  return true;
2122  }
2123 
2124  return false;
2125 }
2126 
2128  if (N.getOpcode() == ISD::AND) {
2129  auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2130  if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
2131  Val = N.getOperand(0);
2132  return true;
2133  }
2134  }
2135  MVT VT = N.getSimpleValueType();
2137  if (CurDAG->MaskedValueIsZero(N, Mask)) {
2138  Val = N;
2139  return true;
2140  }
2141 
2142  return false;
2143 }
2144 
2145 /// Look for various patterns that can be done with a SHL that can be folded
2146 /// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
2147 /// SHXADD we are trying to match.
2149  SDValue &Val) {
2150  if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
2151  SDValue N0 = N.getOperand(0);
2152 
2153  bool LeftShift = N0.getOpcode() == ISD::SHL;
2154  if ((LeftShift || N0.getOpcode() == ISD::SRL) &&
2155  isa<ConstantSDNode>(N0.getOperand(1))) {
2156  uint64_t Mask = N.getConstantOperandVal(1);
2157  unsigned C2 = N0.getConstantOperandVal(1);
2158 
2159  unsigned XLen = Subtarget->getXLen();
2160  if (LeftShift)
2161  Mask &= maskTrailingZeros<uint64_t>(C2);
2162  else
2163  Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
2164 
2165  // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
2166  // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
2167  // followed by a SHXADD with c3 for the X amount.
2168  if (isShiftedMask_64(Mask)) {
2169  unsigned Leading = XLen - (64 - countLeadingZeros(Mask));
2170  unsigned Trailing = countTrailingZeros(Mask);
2171  if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
2172  SDLoc DL(N);
2173  EVT VT = N.getValueType();
2174  Val = SDValue(CurDAG->getMachineNode(
2175  RISCV::SRLI, DL, VT, N0.getOperand(0),
2176  CurDAG->getTargetConstant(Trailing - C2, DL, VT)),
2177  0);
2178  return true;
2179  }
2180  // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
2181  // leading zeros and c3 trailing zeros. We can use an SRLI by C3
2182  // followed by a SHXADD using c3 for the X amount.
2183  if (!LeftShift && Leading == C2 && Trailing == ShAmt) {
2184  SDLoc DL(N);
2185  EVT VT = N.getValueType();
2186  Val = SDValue(
2188  RISCV::SRLI, DL, VT, N0.getOperand(0),
2189  CurDAG->getTargetConstant(Leading + Trailing, DL, VT)),
2190  0);
2191  return true;
2192  }
2193  }
2194  }
2195  }
2196 
2197  bool LeftShift = N.getOpcode() == ISD::SHL;
2198  if ((LeftShift || N.getOpcode() == ISD::SRL) &&
2199  isa<ConstantSDNode>(N.getOperand(1))) {
2200  SDValue N0 = N.getOperand(0);
2201  if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
2202  isa<ConstantSDNode>(N0.getOperand(1))) {
2204  if (isShiftedMask_64(Mask)) {
2205  unsigned C1 = N.getConstantOperandVal(1);
2206  unsigned XLen = Subtarget->getXLen();
2207  unsigned Leading = XLen - (64 - countLeadingZeros(Mask));
2208  unsigned Trailing = countTrailingZeros(Mask);
2209  // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
2210  // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
2211  if (LeftShift && Leading == 32 && Trailing > 0 &&
2212  (Trailing + C1) == ShAmt) {
2213  SDLoc DL(N);
2214  EVT VT = N.getValueType();
2215  Val = SDValue(CurDAG->getMachineNode(
2216  RISCV::SRLIW, DL, VT, N0.getOperand(0),
2217  CurDAG->getTargetConstant(Trailing, DL, VT)),
2218  0);
2219  return true;
2220  }
2221  // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
2222  // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
2223  if (!LeftShift && Leading == 32 && Trailing > C1 &&
2224  (Trailing - C1) == ShAmt) {
2225  SDLoc DL(N);
2226  EVT VT = N.getValueType();
2227  Val = SDValue(CurDAG->getMachineNode(
2228  RISCV::SRLIW, DL, VT, N0.getOperand(0),
2229  CurDAG->getTargetConstant(Trailing, DL, VT)),
2230  0);
2231  return true;
2232  }
2233  }
2234  }
2235  }
2236 
2237  return false;
2238 }
2239 
2240 /// Look for various patterns that can be done with a SHL that can be folded
2241 /// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
2242 /// SHXADD_UW we are trying to match.
2244  SDValue &Val) {
2245  if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
2246  N.hasOneUse()) {
2247  SDValue N0 = N.getOperand(0);
2248  if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2249  N0.hasOneUse()) {
2250  uint64_t Mask = N.getConstantOperandVal(1);
2251  unsigned C2 = N0.getConstantOperandVal(1);
2252 
2253  Mask &= maskTrailingZeros<uint64_t>(C2);
2254 
2255  // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
2256  // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
2257  // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
2258  if (isShiftedMask_64(Mask)) {
2259  unsigned Leading = countLeadingZeros(Mask);
2260  unsigned Trailing = countTrailingZeros(Mask);
2261  if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
2262  SDLoc DL(N);
2263  EVT VT = N.getValueType();
2264  Val = SDValue(CurDAG->getMachineNode(
2265  RISCV::SLLI, DL, VT, N0.getOperand(0),
2266  CurDAG->getTargetConstant(C2 - ShAmt, DL, VT)),
2267  0);
2268  return true;
2269  }
2270  }
2271  }
2272  }
2273 
2274  return false;
2275 }
2276 
2277 // Return true if all users of this SDNode* only consume the lower \p Bits.
2278 // This can be used to form W instructions for add/sub/mul/shl even when the
2279 // root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
2280 // SimplifyDemandedBits has made it so some users see a sext_inreg and some
2281 // don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
2282 // the add/sub/mul/shl to become non-W instructions. By checking the users we
2283 // may be able to use a W instruction and CSE with the other instruction if
2284 // this has happened. We could try to detect that the CSE opportunity exists
2285 // before doing this, but that would be more complicated.
2286 // TODO: Does this need to look through AND/OR/XOR to their users to find more
2287 // opportunities.
2288 bool RISCVDAGToDAGISel::hasAllNBitUsers(SDNode *Node, unsigned Bits) const {
2289  assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
2290  Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
2291  Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
2292  Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
2293  Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
2294  isa<ConstantSDNode>(Node)) &&
2295  "Unexpected opcode");
2296 
2297  for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) {
2298  SDNode *User = *UI;
2299  // Users of this node should have already been instruction selected
2300  if (!User->isMachineOpcode())
2301  return false;
2302 
2303  // TODO: Add more opcodes?
2304  switch (User->getMachineOpcode()) {
2305  default:
2306  return false;
2307  case RISCV::ADDW:
2308  case RISCV::ADDIW:
2309  case RISCV::SUBW:
2310  case RISCV::MULW:
2311  case RISCV::SLLW:
2312  case RISCV::SLLIW:
2313  case RISCV::SRAW:
2314  case RISCV::SRAIW:
2315  case RISCV::SRLW:
2316  case RISCV::SRLIW:
2317  case RISCV::DIVW:
2318  case RISCV::DIVUW:
2319  case RISCV::REMW:
2320  case RISCV::REMUW:
2321  case RISCV::ROLW:
2322  case RISCV::RORW:
2323  case RISCV::RORIW:
2324  case RISCV::CLZW:
2325  case RISCV::CTZW:
2326  case RISCV::CPOPW:
2327  case RISCV::SLLI_UW:
2328  case RISCV::FMV_W_X:
2329  case RISCV::FCVT_H_W:
2330  case RISCV::FCVT_H_WU:
2331  case RISCV::FCVT_S_W:
2332  case RISCV::FCVT_S_WU:
2333  case RISCV::FCVT_D_W:
2334  case RISCV::FCVT_D_WU:
2335  if (Bits < 32)
2336  return false;
2337  break;
2338  case RISCV::SLL:
2339  case RISCV::SRA:
2340  case RISCV::SRL:
2341  case RISCV::ROL:
2342  case RISCV::ROR:
2343  case RISCV::BSET:
2344  case RISCV::BCLR:
2345  case RISCV::BINV:
2346  // Shift amount operands only use log2(Xlen) bits.
2347  if (UI.getOperandNo() != 1 || Bits < Log2_32(Subtarget->getXLen()))
2348  return false;
2349  break;
2350  case RISCV::SLLI:
2351  // SLLI only uses the lower (XLen - ShAmt) bits.
2352  if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1))
2353  return false;
2354  break;
2355  case RISCV::ANDI:
2356  if (Bits < (64 - countLeadingZeros(User->getConstantOperandVal(1))))
2357  return false;
2358  break;
2359  case RISCV::ORI: {
2360  uint64_t Imm = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
2361  if (Bits < (64 - countLeadingOnes(Imm)))
2362  return false;
2363  break;
2364  }
2365  case RISCV::SEXT_B:
2366  case RISCV::PACKH:
2367  if (Bits < 8)
2368  return false;
2369  break;
2370  case RISCV::SEXT_H:
2371  case RISCV::FMV_H_X:
2372  case RISCV::ZEXT_H_RV32:
2373  case RISCV::ZEXT_H_RV64:
2374  case RISCV::PACKW:
2375  if (Bits < 16)
2376  return false;
2377  break;
2378  case RISCV::PACK:
2379  if (Bits < (Subtarget->getXLen() / 2))
2380  return false;
2381  break;
2382  case RISCV::ADD_UW:
2383  case RISCV::SH1ADD_UW:
2384  case RISCV::SH2ADD_UW:
2385  case RISCV::SH3ADD_UW:
2386  // The first operand to add.uw/shXadd.uw is implicitly zero extended from
2387  // 32 bits.
2388  if (UI.getOperandNo() != 0 || Bits < 32)
2389  return false;
2390  break;
2391  case RISCV::SB:
2392  if (UI.getOperandNo() != 0 || Bits < 8)
2393  return false;
2394  break;
2395  case RISCV::SH:
2396  if (UI.getOperandNo() != 0 || Bits < 16)
2397  return false;
2398  break;
2399  case RISCV::SW:
2400  if (UI.getOperandNo() != 0 || Bits < 32)
2401  return false;
2402  break;
2403  }
2404  }
2405 
2406  return true;
2407 }
2408 
2409 // Select VL as a 5 bit immediate or a value that will become a register. This
2410 // allows us to choose betwen VSETIVLI or VSETVLI later.
2412  auto *C = dyn_cast<ConstantSDNode>(N);
2413  if (C && isUInt<5>(C->getZExtValue())) {
2414  VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
2415  N->getValueType(0));
2416  } else if (C && C->isAllOnesValue()) {
2417  // Treat all ones as VLMax.
2419  N->getValueType(0));
2420  } else if (isa<RegisterSDNode>(N) &&
2421  cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
2422  // All our VL operands use an operand that allows GPRNoX0 or an immediate
2423  // as the register class. Convert X0 to a special immediate to pass the
2424  // MachineVerifier. This is recognized specially by the vsetvli insertion
2425  // pass.
2427  N->getValueType(0));
2428  } else {
2429  VL = N;
2430  }
2431 
2432  return true;
2433 }
2434 
2436  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef())
2437  return false;
2438  assert(N.getNumOperands() == 3 && "Unexpected number of operands");
2439  SplatVal = N.getOperand(1);
2440  return true;
2441 }
2442 
2443 using ValidateFn = bool (*)(int64_t);
2444 
2445 static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal,
2446  SelectionDAG &DAG,
2447  const RISCVSubtarget &Subtarget,
2448  ValidateFn ValidateImm) {
2449  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() ||
2450  !isa<ConstantSDNode>(N.getOperand(1)))
2451  return false;
2452  assert(N.getNumOperands() == 3 && "Unexpected number of operands");
2453 
2454  int64_t SplatImm =
2455  cast<ConstantSDNode>(N.getOperand(1))->getSExtValue();
2456 
2457  // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
2458  // type is wider than the resulting vector element type: an implicit
2459  // truncation first takes place. Therefore, perform a manual
2460  // truncation/sign-extension in order to ignore any truncated bits and catch
2461  // any zero-extended immediate.
2462  // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
2463  // sign-extending to (XLenVT -1).
2464  MVT XLenVT = Subtarget.getXLenVT();
2465  assert(XLenVT == N.getOperand(1).getSimpleValueType() &&
2466  "Unexpected splat operand type");
2467  MVT EltVT = N.getSimpleValueType().getVectorElementType();
2468  if (EltVT.bitsLT(XLenVT))
2469  SplatImm = SignExtend64(SplatImm, EltVT.getSizeInBits());
2470 
2471  if (!ValidateImm(SplatImm))
2472  return false;
2473 
2474  SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), XLenVT);
2475  return true;
2476 }
2477 
2479  return selectVSplatSimmHelper(N, SplatVal, *CurDAG, *Subtarget,
2480  [](int64_t Imm) { return isInt<5>(Imm); });
2481 }
2482 
2484  return selectVSplatSimmHelper(
2485  N, SplatVal, *CurDAG, *Subtarget,
2486  [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
2487 }
2488 
2490  SDValue &SplatVal) {
2491  return selectVSplatSimmHelper(
2492  N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
2493  return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
2494  });
2495 }
2496 
2498  if (N.getOpcode() != RISCVISD::VMV_V_X_VL || !N.getOperand(0).isUndef() ||
2499  !isa<ConstantSDNode>(N.getOperand(1)))
2500  return false;
2501 
2502  int64_t SplatImm =
2503  cast<ConstantSDNode>(N.getOperand(1))->getSExtValue();
2504 
2505  if (!isUInt<5>(SplatImm))
2506  return false;
2507 
2508  SplatVal =
2509  CurDAG->getTargetConstant(SplatImm, SDLoc(N), Subtarget->getXLenVT());
2510 
2511  return true;
2512 }
2513 
2515  SDValue &Imm) {
2516  if (auto *C = dyn_cast<ConstantSDNode>(N)) {
2517  int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
2518 
2519  if (!isInt<5>(ImmVal))
2520  return false;
2521 
2522  Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
2523  return true;
2524  }
2525 
2526  return false;
2527 }
2528 
2529 // Try to remove sext.w if the input is a W instruction or can be made into
2530 // a W instruction cheaply.
2531 bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
2532  // Look for the sext.w pattern, addiw rd, rs1, 0.
2533  if (N->getMachineOpcode() != RISCV::ADDIW ||
2534  !isNullConstant(N->getOperand(1)))
2535  return false;
2536 
2537  SDValue N0 = N->getOperand(0);
2538  if (!N0.isMachineOpcode())
2539  return false;
2540 
2541  switch (N0.getMachineOpcode()) {
2542  default:
2543  break;
2544  case RISCV::ADD:
2545  case RISCV::ADDI:
2546  case RISCV::SUB:
2547  case RISCV::MUL:
2548  case RISCV::SLLI: {
2549  // Convert sext.w+add/sub/mul to their W instructions. This will create
2550  // a new independent instruction. This improves latency.
2551  unsigned Opc;
2552  switch (N0.getMachineOpcode()) {
2553  default:
2554  llvm_unreachable("Unexpected opcode!");
2555  case RISCV::ADD: Opc = RISCV::ADDW; break;
2556  case RISCV::ADDI: Opc = RISCV::ADDIW; break;
2557  case RISCV::SUB: Opc = RISCV::SUBW; break;
2558  case RISCV::MUL: Opc = RISCV::MULW; break;
2559  case RISCV::SLLI: Opc = RISCV::SLLIW; break;
2560  }
2561 
2562  SDValue N00 = N0.getOperand(0);
2563  SDValue N01 = N0.getOperand(1);
2564 
2565  // Shift amount needs to be uimm5.
2566  if (N0.getMachineOpcode() == RISCV::SLLI &&
2567  !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue()))
2568  break;
2569 
2570  SDNode *Result =
2571  CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
2572  N00, N01);
2573  ReplaceUses(N, Result);
2574  return true;
2575  }
2576  case RISCV::ADDW:
2577  case RISCV::ADDIW:
2578  case RISCV::SUBW:
2579  case RISCV::MULW:
2580  case RISCV::SLLIW:
2581  case RISCV::PACKW:
2582  // Result is already sign extended just remove the sext.w.
2583  // NOTE: We only handle the nodes that are selected with hasAllWUsers.
2584  ReplaceUses(N, N0.getNode());
2585  return true;
2586  }
2587 
2588  return false;
2589 }
2590 
2591 // Return true if we can make sure mask of N is all-ones mask.
2592 static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
2593  // Check that we're using V0 as a mask register.
2594  if (!isa<RegisterSDNode>(N->getOperand(MaskOpIdx)) ||
2595  cast<RegisterSDNode>(N->getOperand(MaskOpIdx))->getReg() != RISCV::V0)
2596  return false;
2597 
2598  // The glued user defines V0.
2599  const auto *Glued = N->getGluedNode();
2600 
2601  if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
2602  return false;
2603 
2604  // Check that we're defining V0 as a mask register.
2605  if (!isa<RegisterSDNode>(Glued->getOperand(1)) ||
2606  cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0)
2607  return false;
2608 
2609  // Check the instruction defining V0; it needs to be a VMSET pseudo.
2610  SDValue MaskSetter = Glued->getOperand(2);
2611 
2612  const auto IsVMSet = [](unsigned Opc) {
2613  return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
2614  Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
2615  Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
2616  Opc == RISCV::PseudoVMSET_M_B8;
2617  };
2618 
2619  // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
2620  // undefined behaviour if it's the wrong bitwidth, so we could choose to
2621  // assume that it's all-ones? Same applies to its VL.
2622  return MaskSetter->isMachineOpcode() &&
2623  IsVMSet(MaskSetter.getMachineOpcode());
2624 }
2625 
2626 // Optimize masked RVV pseudo instructions with a known all-ones mask to their
2627 // corresponding "unmasked" pseudo versions. The mask we're interested in will
2628 // take the form of a V0 physical register operand, with a glued
2629 // register-setting instruction.
2630 bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(SDNode *N) {
2632  RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
2633  if (!I)
2634  return false;
2635 
2636  unsigned MaskOpIdx = I->MaskOpIdx;
2637 
2638  if (!usesAllOnesMask(N, MaskOpIdx))
2639  return false;
2640 
2641  // Retrieve the tail policy operand index, if any.
2642  std::optional<unsigned> TailPolicyOpIdx;
2643  const RISCVInstrInfo &TII = *Subtarget->getInstrInfo();
2644  const MCInstrDesc &MaskedMCID = TII.get(N->getMachineOpcode());
2645 
2646  bool IsTA = true;
2647  if (RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags)) {
2648  TailPolicyOpIdx = getVecPolicyOpIdx(N, MaskedMCID);
2649  if (!(N->getConstantOperandVal(*TailPolicyOpIdx) &
2651  // Keep the true-masked instruction when there is no unmasked TU
2652  // instruction
2653  if (I->UnmaskedTUPseudo == I->MaskedPseudo && !N->getOperand(0).isUndef())
2654  return false;
2655  // We can't use TA if the tie-operand is not IMPLICIT_DEF
2656  if (!N->getOperand(0).isUndef())
2657  IsTA = false;
2658  }
2659  }
2660 
2661  unsigned Opc = IsTA ? I->UnmaskedPseudo : I->UnmaskedTUPseudo;
2662 
2663  // Check that we're dropping the mask operand and any policy operand
2664  // when we transform to this unmasked pseudo. Additionally, if this insturtion
2665  // is tail agnostic, the unmasked instruction should not have a merge op.
2666  uint64_t TSFlags = TII.get(Opc).TSFlags;
2667  assert((IsTA != RISCVII::hasMergeOp(TSFlags)) &&
2670  "Unexpected pseudo to transform to");
2671  (void)TSFlags;
2672 
2674  // Skip the merge operand at index 0 if IsTA
2675  for (unsigned I = IsTA, E = N->getNumOperands(); I != E; I++) {
2676  // Skip the mask, the policy, and the Glue.
2677  SDValue Op = N->getOperand(I);
2678  if (I == MaskOpIdx || I == TailPolicyOpIdx ||
2679  Op.getValueType() == MVT::Glue)
2680  continue;
2681  Ops.push_back(Op);
2682  }
2683 
2684  // Transitively apply any node glued to our new node.
2685  const auto *Glued = N->getGluedNode();
2686  if (auto *TGlued = Glued->getGluedNode())
2687  Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1));
2688 
2689  SDNode *Result = CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
2690  Result->setFlags(N->getFlags());
2691  ReplaceUses(N, Result);
2692 
2693  return true;
2694 }
2695 
2696 // Try to fold VMERGE_VVM with unmasked intrinsic to masked intrinsic. The
2697 // peephole only deals with VMERGE_VVM which is TU and has false operand same as
2698 // its true operand now. E.g. (VMERGE_VVM_M1_TU False, False, (VADD_M1 ...),
2699 // ...) -> (VADD_VV_M1_MASK)
2700 bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N, bool IsTA) {
2701  unsigned Offset = IsTA ? 0 : 1;
2702  uint64_t Policy = IsTA ? RISCVII::TAIL_AGNOSTIC : /*TUMU*/ 0;
2703 
2704  SDValue False = N->getOperand(0 + Offset);
2705  SDValue True = N->getOperand(1 + Offset);
2706  SDValue Mask = N->getOperand(2 + Offset);
2707  SDValue VL = N->getOperand(3 + Offset);
2708 
2709  assert(True.getResNo() == 0 &&
2710  "Expect True is the first output of an instruction.");
2711 
2712  // Need N is the exactly one using True.
2713  if (!True.hasOneUse())
2714  return false;
2715 
2716  if (!True.isMachineOpcode())
2717  return false;
2718 
2719  unsigned TrueOpc = True.getMachineOpcode();
2720 
2721  // Skip if True has merge operand.
2722  // TODO: Deal with True having same merge operand with N.
2723  if (RISCVII::hasMergeOp(TII->get(TrueOpc).TSFlags))
2724  return false;
2725 
2726  // Skip if True has side effect.
2727  // TODO: Support velff and vlsegff.
2728  if (TII->get(TrueOpc).hasUnmodeledSideEffects())
2729  return false;
2730 
2731  // Only deal with True when True is unmasked intrinsic now.
2733  RISCV::lookupMaskedIntrinsicByUnmaskedTA(TrueOpc);
2734 
2735  if (!Info)
2736  return false;
2737 
2738  // The last operand of unmasked intrinsic should be sew or chain.
2739  bool HasChainOp =
2740  True.getOperand(True.getNumOperands() - 1).getValueType() == MVT::Other;
2741 
2742  if (HasChainOp) {
2743  // Avoid creating cycles in the DAG. We must ensure that none of the other
2744  // operands depend on True through it's Chain.
2745  SmallVector<const SDNode *, 4> LoopWorklist;
2747  LoopWorklist.push_back(False.getNode());
2748  LoopWorklist.push_back(Mask.getNode());
2749  LoopWorklist.push_back(VL.getNode());
2750  if (SDNode *Glued = N->getGluedNode())
2751  LoopWorklist.push_back(Glued);
2752  if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
2753  return false;
2754  }
2755 
2756  // Need True has same VL with N.
2757  unsigned TrueVLIndex = True.getNumOperands() - HasChainOp - 2;
2758  SDValue TrueVL = True.getOperand(TrueVLIndex);
2759 
2760  auto IsNoFPExcept = [this](SDValue N) {
2761  return !this->mayRaiseFPException(N.getNode()) ||
2762  N->getFlags().hasNoFPExcept();
2763  };
2764 
2765  // Allow the peephole for non-exception True with VLMAX vector length, since
2766  // all the values after VL of N are dependent on Merge. VLMAX should be
2767  // lowered to (XLenVT -1).
2768  if (TrueVL != VL && !(IsNoFPExcept(True) && isAllOnesConstant(TrueVL)))
2769  return false;
2770 
2771  SDLoc DL(N);
2772  unsigned MaskedOpc = Info->MaskedPseudo;
2773  assert(RISCVII::hasVecPolicyOp(TII->get(MaskedOpc).TSFlags) &&
2774  "Expected instructions with mask have policy operand.");
2775  assert(RISCVII::hasMergeOp(TII->get(MaskedOpc).TSFlags) &&
2776  "Expected instructions with mask have merge operand.");
2777 
2779  Ops.push_back(False);
2780  Ops.append(True->op_begin(), True->op_begin() + TrueVLIndex);
2781  Ops.append({Mask, VL, /* SEW */ True.getOperand(TrueVLIndex + 1)});
2782  Ops.push_back(CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT()));
2783 
2784  // Result node should have chain operand of True.
2785  if (HasChainOp)
2786  Ops.push_back(True.getOperand(True.getNumOperands() - 1));
2787 
2788  // Result node should take over glued node of N.
2789  if (N->getGluedNode())
2790  Ops.push_back(N->getOperand(N->getNumOperands() - 1));
2791 
2792  SDNode *Result =
2793  CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
2794  Result->setFlags(True->getFlags());
2795 
2796  // Replace vmerge.vvm node by Result.
2797  ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
2798 
2799  // Replace another value of True. E.g. chain and VL.
2800  for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
2801  ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
2802 
2803  // Try to transform Result to unmasked intrinsic.
2804  doPeepholeMaskedRVV(Result);
2805  return true;
2806 }
2807 
2808 // Transform (VMERGE_VVM_<LMUL>_TU false, false, true, allones, vl, sew) to
2809 // (VADD_VI_<LMUL>_TU false, true, 0, vl, sew). It may decrease uses of VMSET.
2810 bool RISCVDAGToDAGISel::performVMergeToVAdd(SDNode *N) {
2811  unsigned NewOpc;
2812  switch (N->getMachineOpcode()) {
2813  default:
2814  llvm_unreachable("Expected VMERGE_VVM_<LMUL>_TU instruction.");
2815  case RISCV::PseudoVMERGE_VVM_MF8_TU:
2816  NewOpc = RISCV::PseudoVADD_VI_MF8_TU;
2817  break;
2818  case RISCV::PseudoVMERGE_VVM_MF4_TU:
2819  NewOpc = RISCV::PseudoVADD_VI_MF4_TU;
2820  break;
2821  case RISCV::PseudoVMERGE_VVM_MF2_TU:
2822  NewOpc = RISCV::PseudoVADD_VI_MF2_TU;
2823  break;
2824  case RISCV::PseudoVMERGE_VVM_M1_TU:
2825  NewOpc = RISCV::PseudoVADD_VI_M1_TU;
2826  break;
2827  case RISCV::PseudoVMERGE_VVM_M2_TU:
2828  NewOpc = RISCV::PseudoVADD_VI_M2_TU;
2829  break;
2830  case RISCV::PseudoVMERGE_VVM_M4_TU:
2831  NewOpc = RISCV::PseudoVADD_VI_M4_TU;
2832  break;
2833  case RISCV::PseudoVMERGE_VVM_M8_TU:
2834  NewOpc = RISCV::PseudoVADD_VI_M8_TU;
2835  break;
2836  }
2837 
2838  if (!usesAllOnesMask(N, /* MaskOpIdx */ 3))
2839  return false;
2840 
2841  SDLoc DL(N);
2842  EVT VT = N->getValueType(0);
2843  SDValue Ops[] = {N->getOperand(1), N->getOperand(2),
2844  CurDAG->getTargetConstant(0, DL, Subtarget->getXLenVT()),
2845  N->getOperand(4), N->getOperand(5)};
2846  SDNode *Result = CurDAG->getMachineNode(NewOpc, DL, VT, Ops);
2847  ReplaceUses(N, Result);
2848  return true;
2849 }
2850 
2851 bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
2852  bool MadeChange = false;
2854 
2855  while (Position != CurDAG->allnodes_begin()) {
2856  SDNode *N = &*--Position;
2857  if (N->use_empty() || !N->isMachineOpcode())
2858  continue;
2859 
2860  auto IsVMergeTU = [](unsigned Opcode) {
2861  return Opcode == RISCV::PseudoVMERGE_VVM_MF8_TU ||
2862  Opcode == RISCV::PseudoVMERGE_VVM_MF4_TU ||
2863  Opcode == RISCV::PseudoVMERGE_VVM_MF2_TU ||
2864  Opcode == RISCV::PseudoVMERGE_VVM_M1_TU ||
2865  Opcode == RISCV::PseudoVMERGE_VVM_M2_TU ||
2866  Opcode == RISCV::PseudoVMERGE_VVM_M4_TU ||
2867  Opcode == RISCV::PseudoVMERGE_VVM_M8_TU;
2868  };
2869 
2870  auto IsVMergeTA = [](unsigned Opcode) {
2871  return Opcode == RISCV::PseudoVMERGE_VVM_MF8 ||
2872  Opcode == RISCV::PseudoVMERGE_VVM_MF4 ||
2873  Opcode == RISCV::PseudoVMERGE_VVM_MF2 ||
2874  Opcode == RISCV::PseudoVMERGE_VVM_M1 ||
2875  Opcode == RISCV::PseudoVMERGE_VVM_M2 ||
2876  Opcode == RISCV::PseudoVMERGE_VVM_M4 ||
2877  Opcode == RISCV::PseudoVMERGE_VVM_M8;
2878  };
2879 
2880  unsigned Opc = N->getMachineOpcode();
2881  // The following optimizations require that the merge operand of N is same
2882  // as the false operand of N.
2883  if ((IsVMergeTU(Opc) && N->getOperand(0) == N->getOperand(1)) ||
2884  IsVMergeTA(Opc))
2885  MadeChange |= performCombineVMergeAndVOps(N, IsVMergeTA(Opc));
2886  if (IsVMergeTU(Opc) && N->getOperand(0) == N->getOperand(1))
2887  MadeChange |= performVMergeToVAdd(N);
2888  }
2889  return MadeChange;
2890 }
2891 
2892 // This pass converts a legalized DAG into a RISCV-specific DAG, ready
2893 // for instruction scheduling.
2895  CodeGenOpt::Level OptLevel) {
2896  return new RISCVDAGToDAGISel(TM, OptLevel);
2897 }
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:240
llvm::RISCVII::LMUL_1
@ LMUL_1
Definition: RISCVBaseInfo.h:109
llvm::TargetMachine::getOptLevel
CodeGenOpt::Level getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
Definition: TargetMachine.cpp:182
llvm::RISCVISD::VFMV_S_F_VL
@ VFMV_S_F_VL
Definition: RISCVISelLowering.h:146
llvm::RISCVMatInt::Inst
Definition: RISCVMatInt.h:28
llvm::ConstantSDNode
Definition: SelectionDAGNodes.h:1582
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:542
llvm::MVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: MachineValueType.h:1148
llvm::ISD::INTRINSIC_VOID
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
llvm::RISCVDAGToDAGISel::selectVLXSEG
void selectVLXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
Definition: RISCVISelDAGToDAG.cpp:401
MathExtras.h
Merge
R600 Clause Merge
Definition: R600ClauseMergePass.cpp:70
llvm::SelectionDAGISel::TLI
const TargetLowering * TLI
Definition: SelectionDAGISel.h:56
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::RISCVISD::SLLW
@ SLLW
Definition: RISCVISelLowering.h:65
llvm::SelectionDAGISel::TM
TargetMachine & TM
Definition: SelectionDAGISel.h:43
llvm::EVT::isScalarInteger
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
llvm::RISCV::VLSEGPseudo
Definition: RISCVISelDAGToDAG.h:143
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::MVT::isFixedLengthVector
bool isFixedLengthVector() const
Definition: MachineValueType.h:398
llvm::RISCVDAGToDAGISel::selectVSplatSimm5Plus1
bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2483
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:667
llvm::MVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: MachineValueType.h:370
llvm::RISCVDAGToDAGISel::PreprocessISelDAG
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
Definition: RISCVISelDAGToDAG.cpp:60
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:886
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::RISCVSubtarget::getTargetLowering
const RISCVTargetLowering * getTargetLowering() const override
Definition: RISCVSubtarget.h:144
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:767
llvm::RISCV::VLXSEGPseudo
Definition: RISCVISelDAGToDAG.h:154
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::isOneConstant
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Definition: SelectionDAG.cpp:10863
llvm::SelectionDAG::allnodes_end
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:526
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::RISCVISD::FMV_H_X
@ FMV_H_X
Definition: RISCVISelLowering.h:97
llvm::SDNode::isUndef
bool isUndef() const
Return true if the type of the node type undefined.
Definition: SelectionDAGNodes.h:667
llvm::KnownBits::Zero
APInt Zero
Definition: KnownBits.h:24
C1
instcombine should handle this C2 when C1
Definition: README.txt:263
llvm::RISCVSubtarget::hasOptimizedZeroStrideLoad
bool hasOptimizedZeroStrideLoad() const
Definition: RISCVSubtarget.h:210
llvm::RISCVISD::DIVUW
@ DIVUW
Definition: RISCVISelLowering.h:72
llvm::MVT::bitsLT
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
Definition: MachineValueType.h:1210
llvm::SelectionDAG::getFrameIndex
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Definition: SelectionDAG.cpp:1740
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::RISCVDAGToDAGISel::selectVSETVLI
void selectVSETVLI(SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:529
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9439
llvm::MachineSDNode
An SDNode that represents everything that will be needed to construct a MachineInstr.
Definition: SelectionDAGNodes.h:2897
llvm::RISCVSubtarget::hasVInstructions
bool hasVInstructions() const
Definition: RISCVSubtarget.h:243
llvm::RISCVDAGToDAGISel::SelectFrameAddrRegImm
bool SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1882
llvm::SelectionDAG::allnodes_begin
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:525
llvm::SelectionDAG::getRoot
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:545
llvm::RISCVDAGToDAGISel::selectSHXADD_UWOp
bool selectSHXADD_UWOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD_UW.
Definition: RISCVISelDAGToDAG.cpp:2243
llvm::HandleSDNode
This class is used to form a handle around another node that is persistent and is updated across invo...
Definition: SelectionDAGNodes.h:1233
llvm::RISCVMatInt::generateInstSeq
InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures)
Definition: RISCVMatInt.cpp:174
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::RISCVTargetMachine
Definition: RISCVTargetMachine.h:24
llvm::RISCVDAGToDAGISel::selectVSplat
bool selectVSplat(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2435
llvm::RISCVII::LMUL_8
@ LMUL_8
Definition: RISCVBaseInfo.h:112
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:282
llvm::MemOp
Definition: TargetLowering.h:111
llvm::RISCVDAGToDAGISel
Definition: RISCVISelDAGToDAG.h:23
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6874
llvm::RISCVDAGToDAGISel::selectZExtBits
bool selectZExtBits(SDValue N, unsigned Bits, SDValue &Val)
Definition: RISCVISelDAGToDAG.cpp:2127
llvm::RISCVSubtarget::hasStdExtZbs
bool hasStdExtZbs() const
Definition: RISCVSubtarget.h:170
Shift
bool Shift
Definition: README.txt:468
llvm::APInt::getBitWidth
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1431
llvm::tgtok::Bits
@ Bits
Definition: TGLexer.h:50
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8099
llvm::SelectionDAG::isBaseWithConstantOffset
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
Definition: SelectionDAG.cpp:4770
llvm::RISCV::VLX_VSXPseudo
Definition: RISCVISelDAGToDAG.h:202
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::SelectionDAG::RemoveDeadNodes
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
Definition: SelectionDAG.cpp:944
llvm::RISCVTargetLowering::getRegClassIDForVecVT
static unsigned getRegClassIDForVecVT(MVT VT)
Definition: RISCVISelLowering.cpp:1624
llvm::RISCV::VLMaxSentinel
static constexpr int64_t VLMaxSentinel
Definition: RISCVInstrInfo.h:238
Log2SEW
unsigned Log2SEW
Definition: RISCVInsertVSETVLI.cpp:641
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:458
llvm::countLeadingOnes
unsigned countLeadingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the most significant bit to the first zero bit.
Definition: MathExtras.h:476
llvm::RISCVDAGToDAGISel::selectVSSEG
void selectVSSEG(SDNode *Node, bool IsMasked, bool IsStrided)
Definition: RISCVISelDAGToDAG.cpp:455
llvm::RISCVII::hasVecPolicyOp
static bool hasVecPolicyOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:159
RISCVMatInt.h
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::commonAlignment
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:213
llvm::RISCVDAGToDAGISel::selectVSplatSimm5Plus1NonZero
bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2489
llvm::SDNode::getVTList
SDVTList getVTList() const
Definition: SelectionDAGNodes.h:949
llvm::MCInstrDesc::hasUnmodeledSideEffects
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by other flags.
Definition: MCInstrDesc.h:460
KnownBits.h
llvm::MVT::isScalableVector
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
Definition: MachineValueType.h:393
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2122
llvm::InlineAsm::Constraint_A
@ Constraint_A
Definition: InlineAsm.h:262
llvm::RISCVDAGToDAGISel::SelectAddrFrameIndex
bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1870
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
usesAllOnesMask
static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx)
Definition: RISCVISelDAGToDAG.cpp:2592
llvm::RISCVSubtarget::is64Bit
bool is64Bit() const
Definition: RISCVSubtarget.h:200
llvm::RISCV::VSSEGPseudo
Definition: RISCVISelDAGToDAG.h:165
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::RISCVII::LMUL_4
@ LMUL_4
Definition: RISCVBaseInfo.h:111
llvm::RISCV::RISCVMaskedPseudoInfo
Definition: RISCVISelDAGToDAG.h:212
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1734
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:362
llvm::RISCVII::TAIL_AGNOSTIC
@ TAIL_AGNOSTIC
Definition: RISCVBaseInfo.h:120
llvm::MCInstrDesc::TSFlags
uint64_t TSFlags
Definition: MCInstrDesc.h:205
llvm::RISCVDAGToDAGISel::selectShiftMask
bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt)
Definition: RISCVISelDAGToDAG.cpp:2065
llvm::SelectionDAG::getTargetFrameIndex
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:720
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::ISD::Constant
@ Constant
Definition: ISDOpcodes.h:76
getReg
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
Definition: MipsDisassembler.cpp:517
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::MachineFunction::getInfo
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Definition: MachineFunction.h:755
llvm::User
Definition: User.h:44
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1023
llvm::ISD::CopyToReg
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:781
createTuple
static SDValue createTuple(SelectionDAG &CurDAG, ArrayRef< SDValue > Regs, unsigned NF, RISCVII::VLMUL LMUL)
Definition: RISCVISelDAGToDAG.cpp:212
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:475
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::MVT::f64
@ f64
Definition: MachineValueType.h:58
llvm::isShiftedMask_64
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition: MathExtras.h:452
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3506
llvm::MVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: MachineValueType.h:1138
llvm::SelectionDAG::MaskedValueIsZero
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
Definition: SelectionDAG.cpp:2533
llvm::RISCVTargetLowering::getSubregIndexByMVT
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
Definition: RISCVISelLowering.cpp:1601
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ISD::SRA
@ SRA
Definition: ISDOpcodes.h:692
llvm::RISCVSubtarget::getXLenVT
MVT getXLenVT() const
Definition: RISCVSubtarget.h:211
RISCVISelDAGToDAG.h
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::SelectionDAGISel::ReplaceNode
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
Definition: SelectionDAGISel.h:231
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:441
llvm::Log2_32
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:547
llvm::MCInstrDesc
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:197
llvm::SelectionDAG::setRoot
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:554
llvm::RISCVVType::decodeVSEW
static unsigned decodeVSEW(unsigned VSEW)
Definition: RISCVBaseInfo.h:437
RISCVMCTargetDesc.h
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::ISD::ATOMIC_STORE
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1163
llvm::RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand
bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
Definition: RISCVISelDAGToDAG.cpp:1852
llvm::SelectionDAG::getMemIntrinsicNode
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
Definition: SelectionDAG.cpp:7824
llvm::RISCVSubtarget::hasStdExtZbb
bool hasStdExtZbb() const
Definition: RISCVSubtarget.h:168
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:666
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
llvm::RISCVSubtarget::getInstrInfo
const RISCVInstrInfo * getInstrInfo() const override
Definition: RISCVSubtarget.h:140
CASE_VMSLT_OPCODES
#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b)
Align
uint64_t Align
Definition: ELFObjHandler.cpp:82
llvm::ISD::SPLAT_VECTOR
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:613
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SDValue::getConstantOperandVal
uint64_t getConstantOperandVal(unsigned i) const
Definition: SelectionDAGNodes.h:1153
isWorthFoldingAdd
static bool isWorthFoldingAdd(SDValue Add)
Definition: RISCVISelDAGToDAG.cpp:1953
llvm::AArch64_AM::ROR
@ ROR
Definition: AArch64AddressingModes.h:38
llvm::RISCVISD::DIVW
@ DIVW
Definition: RISCVISelLowering.h:71
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:715
llvm::RISCVISD::CLZW
@ CLZW
Definition: RISCVISelLowering.h:80
llvm::RISCVDAGToDAGISel::SelectAddrRegImm
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1975
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:74
llvm::APInt::isSubsetOf
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1227
llvm::InlineAsm::Constraint_m
@ Constraint_m
Definition: InlineAsm.h:259
TSFlags
uint64_t TSFlags
Definition: RISCVInsertVSETVLI.cpp:595
llvm::createRISCVISelDag
FunctionPass * createRISCVISelDag(RISCVTargetMachine &TM, CodeGenOpt::Level OptLevel)
Definition: RISCVISelDAGToDAG.cpp:2894
llvm::RISCVISD::VMV_S_X_VL
@ VMV_S_X_VL
Definition: RISCVISelLowering.h:144
llvm::SDValue::getNumOperands
unsigned getNumOperands() const
Definition: SelectionDAGNodes.h:1145
llvm::SelectionDAG::RemoveDeadNode
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
Definition: SelectionDAG.cpp:998
llvm::RISCV::VSEPseudo
Definition: RISCVISelDAGToDAG.h:194
llvm::RISCVDAGToDAGISel::selectVLOp
bool selectVLOp(SDValue N, SDValue &VL)
Definition: RISCVISelDAGToDAG.cpp:2411
llvm::RISCVDAGToDAGISel::selectVSXSEG
void selectVSXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
Definition: RISCVISelDAGToDAG.cpp:487
llvm::SelectionDAGISel::IsProfitableToFold
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
Definition: SelectionDAGISel.cpp:2072
llvm::AMDGPU::Hwreg::Offset
Offset
Definition: SIDefines.h:419
selectVSplatSimmHelper
static bool selectVSplatSimmHelper(SDValue N, SDValue &SplatVal, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, ValidateFn ValidateImm)
Definition: RISCVISelDAGToDAG.cpp:2445
uint64_t
llvm::RISCVDAGToDAGISel::selectVSplatUimm5
bool selectVSplatUimm5(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2497
llvm::SelectionDAGISel::TII
const TargetInstrInfo * TII
Definition: SelectionDAGISel.h:55
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:966
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:79
llvm::SelectionDAGISel::FuncInfo
std::unique_ptr< FunctionLoweringInfo > FuncInfo
Definition: SelectionDAGISel.h:45
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:793
llvm::SelectionDAGISel::IsLegalToFold
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOpt::Level OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
Definition: SelectionDAGISel.cpp:2080
llvm::SelectionDAGISel::mayRaiseFPException
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
Definition: SelectionDAGISel.cpp:3727
llvm::SDNode::getOperand
const SDValue & getOperand(unsigned Num) const
Definition: SelectionDAGNodes.h:921
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9133
llvm::countTrailingOnes
unsigned countTrailingOnes(T Value, ZeroBehavior ZB=ZB_Width)
Count the number of ones from the least significant bit to the first zero bit.
Definition: MathExtras.h:491
llvm::RISCVISD::ROLW
@ ROLW
Definition: RISCVISelLowering.h:76
llvm::RISCVMachineFunctionInfo
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
Definition: RISCVMachineFunctionInfo.h:47
llvm::RISCVSubtarget
Definition: RISCVSubtarget.h:35
llvm::AVRISD::ROL
@ ROL
Bit rotate left.
Definition: AVRISelLowering.h:51
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::RISCVDAGToDAGISel::selectVLSEG
void selectVLSEG(SDNode *Node, bool IsMasked, bool IsStrided)
Definition: RISCVISelDAGToDAG.cpp:313
llvm::RISCVISD::ADD_LO
@ ADD_LO
Definition: RISCVISelLowering.h:48
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::MVT::getSizeInBits
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
Definition: MachineValueType.h:919
llvm::ConstantSDNode::getZExtValue
uint64_t getZExtValue() const
Definition: SelectionDAGNodes.h:1597
llvm::SDValue::getResNo
unsigned getResNo() const
get the index which selects a specific result in the SDNode
Definition: SelectionDAGNodes.h:156
llvm::RISCVMatInt::RegX0
@ RegX0
Definition: RISCVMatInt.h:25
llvm::SelectionDAGISel::CurDAG
SelectionDAG * CurDAG
Definition: SelectionDAGISel.h:49
llvm::RISCVDAGToDAGISel::hasAllWUsers
bool hasAllWUsers(SDNode *Node) const
Definition: RISCVISelDAGToDAG.h:80
llvm::SelectionDAG::getMachineNode
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
Definition: SelectionDAG.cpp:9877
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::RISCVISD::SRAW
@ SRAW
Definition: RISCVISelLowering.h:66
llvm::RISCVDAGToDAGISel::selectVSplatSimm5
bool selectVSplatSimm5(SDValue N, SDValue &SplatVal)
Definition: RISCVISelDAGToDAG.cpp:2478
llvm::SelectionDAG::setNodeMemRefs
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
Definition: SelectionDAG.cpp:9645
llvm::RISCVSubtarget::hasStdExtZba
bool hasStdExtZba() const
Definition: RISCVSubtarget.h:167
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:79
llvm::SDNode::isMachineOpcode
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
Definition: SelectionDAGNodes.h:699
llvm::RISCVII::hasMergeOp
static bool hasMergeOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:147
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
Definition: RISCVISelLowering.cpp:1636
llvm::RISCVISD::REMUW
@ REMUW
Definition: RISCVISelLowering.h:73
llvm::NVPTXISD::Dummy
@ Dummy
Definition: NVPTXISelLowering.h:60
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::RISCVInstrInfo
Definition: RISCVInstrInfo.h:44
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::countTrailingZeros
unsigned countTrailingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: MathExtras.h:152
llvm::SelectionDAG::getTargetInsertSubreg
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
Definition: SelectionDAG.cpp:10005
llvm::RISCVISD::VMV_V_X_VL
@ VMV_V_X_VL
Definition: RISCVISelLowering.h:135
llvm::RISCVSubtarget::getRegisterInfo
const RISCVRegisterInfo * getRegisterInfo() const override
Definition: RISCVSubtarget.h:141
llvm::SDValue::getMachineOpcode
unsigned getMachineOpcode() const
Definition: SelectionDAGNodes.h:1173
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::RISCVII::LMUL_2
@ LMUL_2
Definition: RISCVBaseInfo.h:110
llvm::SelectionDAG::ReplaceAllUsesOfValueWith
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
Definition: SelectionDAG.cpp:10472
llvm::SDValue::getOperand
const SDValue & getOperand(unsigned i) const
Definition: SelectionDAGNodes.h:1149
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ConstantSDNode::getSExtValue
int64_t getSExtValue() const
Definition: SelectionDAGNodes.h:1598
llvm::SDValue::hasOneUse
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
Definition: SelectionDAGNodes.h:1185
llvm::SDValue::getSimpleValueType
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:190
llvm::CodeGenOpt::Level
Level
Definition: CodeGen.h:52
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::SignExtend64
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:718
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
llvm::ISD::INTRINSIC_WO_CHAIN
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:668
llvm::MVT::getVectorElementCount
ElementCount getVectorElementCount() const
Definition: MachineValueType.h:901
llvm::RISCVISD::RORW
@ RORW
Definition: RISCVISelLowering.h:77
llvm::ISD::INSERT_SUBVECTOR
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:558
llvm::SelectionDAGISel::MF
MachineFunction * MF
Definition: SelectionDAGISel.h:47
llvm::RISCV
Definition: TargetParser.h:158
llvm::isAllOnesConstant
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
Definition: SelectionDAG.cpp:10858
CASE_VMSLT_VMNAND_VMSET_OPCODES
#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b)
Alignment.h
selectImm
static SDNode * selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, int64_t Imm, const RISCVSubtarget &Subtarget)
Definition: RISCVISelDAGToDAG.cpp:204
selectImmSeq
static SDNode * selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, RISCVMatInt::InstSeq &Seq)
Definition: RISCVISelDAGToDAG.cpp:175
llvm::SelectionDAG::computeKnownBits
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
Definition: SelectionDAG.cpp:2911
llvm::KnownBits
Definition: KnownBits.h:23
llvm::RISCVISD::SRLW
@ SRLW
Definition: RISCVISelLowering.h:67
llvm::ISD::EXTRACT_SUBVECTOR
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:572
llvm::isNullConstant
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
Definition: SelectionDAG.cpp:10848
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:351
llvm::RISCV::VSXSEGPseudo
Definition: RISCVISelDAGToDAG.h:174
RISCVISelLowering.h
llvm::RISCVDAGToDAGISel::PostprocessISelDAG
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
Definition: RISCVISelDAGToDAG.cpp:152
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:548
llvm::RISCVVType::encodeVTYPE
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
Definition: RISCVBaseInfo.cpp:133
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:469
llvm::RISCVDAGToDAGISel::selectVLSEGFF
void selectVLSEGFF(SDNode *Node, bool IsMasked)
Definition: RISCVISelDAGToDAG.cpp:356
llvm::SelectionDAG::getTargetExtractSubreg
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
Definition: SelectionDAG.cpp:9995
llvm::SelectionDAGISel::ReplaceUses
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
Definition: SelectionDAGISel.h:210
selectConstantAddr
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, const RISCVSubtarget *Subtarget, SDValue Addr, SDValue &Base, SDValue &Offset)
Definition: RISCVISelDAGToDAG.cpp:1905
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::SystemZISD::PACK
@ PACK
Definition: SystemZISelLowering.h:205
llvm::RISCVSubtarget::getXLen
unsigned getXLen() const
Definition: RISCVSubtarget.h:212
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:983
llvm::RISCVTargetLowering
Definition: RISCVISelLowering.h:335
llvm::RISCVMatInt::RegReg
@ RegReg
Definition: RISCVMatInt.h:24
llvm::XCoreISD::LMUL
@ LMUL
Definition: XCoreISelLowering.h:59
llvm::countLeadingZeros
unsigned countLeadingZeros(T Val, ZeroBehavior ZB=ZB_Width)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: MathExtras.h:220
llvm::ISD::STORE
@ STORE
Definition: ISDOpcodes.h:967
llvm::RISCVTargetLowering::getLMUL
static RISCVII::VLMUL getLMUL(MVT VT)
Definition: RISCVISelLowering.cpp:1557
llvm::AMDGPU::Hwreg::Width
Width
Definition: SIDefines.h:436
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::RISCVISD::VFMV_V_F_VL
@ VFMV_V_F_VL
Definition: RISCVISelLowering.h:139
llvm::SDValue::isUndef
bool isUndef() const
Definition: SelectionDAGNodes.h:1177
VLMul
RISCVII::VLMUL VLMul
Definition: RISCVInsertVSETVLI.cpp:639
llvm::RISCVMatInt::Imm
@ Imm
Definition: RISCVMatInt.h:23
llvm::RISCVDAGToDAGISel::tryShrinkShlLogicImm
bool tryShrinkShlLogicImm(SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:599
llvm::SDNode::op_begin
op_iterator op_begin() const
Definition: SelectionDAGNodes.h:928
SEW
unsigned SEW
Definition: RISCVInsertVSETVLI.cpp:643
llvm::RISCVII::LMUL_F8
@ LMUL_F8
Definition: RISCVBaseInfo.h:114
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:691
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1019
llvm::RISCVMatInt::RegImm
@ RegImm
Definition: RISCVMatInt.h:22
llvm::ISD::MUL
@ MUL
Definition: ISDOpcodes.h:241
llvm::SDValue::getConstantOperandAPInt
const APInt & getConstantOperandAPInt(unsigned i) const
Definition: SelectionDAGNodes.h:1157
llvm::MVT::f16
@ f16
Definition: MachineValueType.h:56
llvm::SDNode::hasPredecessorHelper
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
Definition: SelectionDAGNodes.h:849
N
#define N
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:693
RISCVMachineFunctionInfo.h
llvm::RISCVDAGToDAGISel::selectRVVSimm5
bool selectRVVSimm5(SDValue N, unsigned Width, SDValue &Imm)
Definition: RISCVISelDAGToDAG.cpp:2514
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:164
llvm::RISCVII::LMUL_F4
@ LMUL_F4
Definition: RISCVBaseInfo.h:115
llvm::RISCVDAGToDAGISel::Select
void Select(SDNode *Node) override
Main hook for targets to transform nodes into machine nodes.
Definition: RISCVISelDAGToDAG.cpp:677
llvm::RISCVII::VLMUL
VLMUL
Definition: RISCVBaseInfo.h:108
llvm::MVT::Untyped
@ Untyped
Definition: MachineValueType.h:286
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:637
llvm::SDValue::getOpcode
unsigned getOpcode() const
Definition: SelectionDAGNodes.h:1137
llvm::SelectionDAG::getTargetConstant
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:669
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
getLastNonGlueOrChainOpIdx
static unsigned getLastNonGlueOrChainOpIdx(const SDNode *Node)
Definition: RISCVISelDAGToDAG.cpp:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::ISD::INTRINSIC_W_CHAIN
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
CASE_VMXOR_VMANDN_VMOR_OPCODES
#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix)
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:466
llvm::SelectionDAG::ComputeNumSignBits
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
Definition: SelectionDAG.cpp:3968
llvm::RISCVII::hasDummyMaskOp
static bool hasDummyMaskOp(uint64_t TSFlags)
Definition: RISCVBaseInfo.h:139
llvm::MCInstrInfo::get
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
llvm::isMask_64
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition: MathExtras.h:440
llvm::RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL
@ SPLAT_VECTOR_SPLIT_I64_VL
Definition: RISCVISelLowering.h:150
llvm::APInt::getBitsSetFrom
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:269
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::M1
unsigned M1(unsigned Val)
Definition: VE.h:466
raw_ostream.h
llvm::SDValue::isMachineOpcode
bool isMachineOpcode() const
Definition: SelectionDAGNodes.h:1169
llvm::RISCVDAGToDAGISel::selectSHXADDOp
bool selectSHXADDOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD.
Definition: RISCVISelDAGToDAG.cpp:2148
llvm::SDNode::getFlags
SDNodeFlags getFlags() const
Definition: SelectionDAGNodes.h:972
llvm::RISCV::VLEPseudo
Definition: RISCVISelDAGToDAG.h:184
llvm::MVT::f32
@ f32
Definition: MachineValueType.h:57
llvm::RISCVDAGToDAGISel::hasAllHUsers
bool hasAllHUsers(SDNode *Node) const
Definition: RISCVISelDAGToDAG.h:79
ValidateFn
bool(*)(int64_t) ValidateFn
Definition: RISCVISelDAGToDAG.cpp:2443
llvm::RISCVISD::CTZW
@ CTZW
Definition: RISCVISelLowering.h:81
llvm::RISCVDAGToDAGISel::hasAllNBitUsers
bool hasAllNBitUsers(SDNode *Node, unsigned Bits) const
Definition: RISCVISelDAGToDAG.cpp:2288
Debug.h
llvm::ISD::ATOMIC_LOAD
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1159
llvm::RISCVDAGToDAGISel::selectSExti32
bool selectSExti32(SDValue N, SDValue &Val)
Definition: RISCVISelDAGToDAG.cpp:2112
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:356
getVecPolicyOpIdx
static unsigned getVecPolicyOpIdx(const SDNode *Node, const MCInstrDesc &MCID)
Definition: RISCVISelDAGToDAG.cpp:54
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::RISCVII::LMUL_F2
@ LMUL_F2
Definition: RISCVBaseInfo.h:116
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::RISCVDAGToDAGISel::addVectorLoadStoreOperands
void addVectorLoadStoreOperands(SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp, bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl< SDValue > &Operands, bool IsLoad=false, MVT *IndexVT=nullptr)
Definition: RISCVISelDAGToDAG.cpp:266
isAllUndef
static bool isAllUndef(ArrayRef< SDValue > Values)
Definition: RISCVISelDAGToDAG.cpp:309