LLVM 18.0.0git
RISCVISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
17#include "RISCVISelLowering.h"
20#include "llvm/IR/IntrinsicsRISCV.h"
22#include "llvm/Support/Debug.h"
25#include <optional>
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
33 "riscv-use-rematerializable-movimm", cl::Hidden,
34 cl::desc("Use a rematerializable pseudoinstruction for 2 instruction "
35 "constant materialization"),
36 cl::init(false));
37
38namespace llvm::RISCV {
39#define GET_RISCVVSSEGTable_IMPL
40#define GET_RISCVVLSEGTable_IMPL
41#define GET_RISCVVLXSEGTable_IMPL
42#define GET_RISCVVSXSEGTable_IMPL
43#define GET_RISCVVLETable_IMPL
44#define GET_RISCVVSETable_IMPL
45#define GET_RISCVVLXTable_IMPL
46#define GET_RISCVVSXTable_IMPL
47#define GET_RISCVMaskedPseudosTable_IMPL
48#include "RISCVGenSearchableTables.inc"
49} // namespace llvm::RISCV
50
53
54 bool MadeChange = false;
55 while (Position != CurDAG->allnodes_begin()) {
56 SDNode *N = &*--Position;
57 if (N->use_empty())
58 continue;
59
60 SDValue Result;
61 switch (N->getOpcode()) {
62 case ISD::SPLAT_VECTOR: {
63 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
64 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
65 MVT VT = N->getSimpleValueType(0);
66 unsigned Opc =
68 SDLoc DL(N);
69 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
70 SDValue Src = N->getOperand(0);
71 if (VT.isInteger())
72 Src = CurDAG->getNode(ISD::ANY_EXTEND, DL, Subtarget->getXLenVT(),
73 N->getOperand(0));
74 Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), Src, VL);
75 break;
76 }
78 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
79 // load. Done after lowering and combining so that we have a chance to
80 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
81 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
82 MVT VT = N->getSimpleValueType(0);
83 SDValue Passthru = N->getOperand(0);
84 SDValue Lo = N->getOperand(1);
85 SDValue Hi = N->getOperand(2);
86 SDValue VL = N->getOperand(3);
87 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
88 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
89 "Unexpected VTs!");
91 SDLoc DL(N);
92
93 // Create temporary stack for each expanding node.
94 SDValue StackSlot =
96 int FI = cast<FrameIndexSDNode>(StackSlot.getNode())->getIndex();
98
99 SDValue Chain = CurDAG->getEntryNode();
100 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
101
102 SDValue OffsetSlot =
104 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
105 Align(8));
106
107 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
108
109 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
110 SDValue IntID =
111 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
112 SDValue Ops[] = {Chain,
113 IntID,
114 Passthru,
115 StackSlot,
116 CurDAG->getRegister(RISCV::X0, MVT::i64),
117 VL};
118
120 MVT::i64, MPI, Align(8),
122 break;
123 }
124 }
125
126 if (Result) {
127 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
128 LLVM_DEBUG(N->dump(CurDAG));
129 LLVM_DEBUG(dbgs() << "\nNew: ");
130 LLVM_DEBUG(Result->dump(CurDAG));
131 LLVM_DEBUG(dbgs() << "\n");
132
134 MadeChange = true;
135 }
136 }
137
138 if (MadeChange)
140}
141
143 HandleSDNode Dummy(CurDAG->getRoot());
145
146 bool MadeChange = false;
147 while (Position != CurDAG->allnodes_begin()) {
148 SDNode *N = &*--Position;
149 // Skip dead nodes and any non-machine opcodes.
150 if (N->use_empty() || !N->isMachineOpcode())
151 continue;
152
153 MadeChange |= doPeepholeSExtW(N);
154 MadeChange |= doPeepholeMaskedRVV(cast<MachineSDNode>(N));
155 }
156
157 CurDAG->setRoot(Dummy.getValue());
158
159 MadeChange |= doPeepholeMergeVVMFold();
160
161 // After we're done with everything else, convert IMPLICIT_DEF
162 // passthru operands to NoRegister. This is required to workaround
163 // an optimization deficiency in MachineCSE. This really should
164 // be merged back into each of the patterns (i.e. there's no good
165 // reason not to go directly to NoReg), but is being done this way
166 // to allow easy backporting.
167 MadeChange |= doPeepholeNoRegPassThru();
168
169 if (MadeChange)
171}
172
173static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
175 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
176 for (const RISCVMatInt::Inst &Inst : Seq) {
177 SDValue SDImm = CurDAG->getTargetConstant(Inst.getImm(), DL, VT);
178 SDNode *Result = nullptr;
179 switch (Inst.getOpndKind()) {
180 case RISCVMatInt::Imm:
181 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SDImm);
182 break;
184 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg,
185 CurDAG->getRegister(RISCV::X0, VT));
186 break;
188 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SrcReg);
189 break;
191 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SDImm);
192 break;
193 }
194
195 // Only the first instruction has X0 as its source.
196 SrcReg = SDValue(Result, 0);
197 }
198
199 return SrcReg;
200}
201
202static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
203 int64_t Imm, const RISCVSubtarget &Subtarget) {
205
206 // Use a rematerializable pseudo instruction for short sequences if enabled.
207 if (Seq.size() == 2 && UsePseudoMovImm)
208 return SDValue(
209 CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
210 CurDAG->getTargetConstant(Imm, DL, VT)),
211 0);
212
213 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
214 // worst an LUI+ADDIW. This will require an extra register, but avoids a
215 // constant pool.
216 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
217 // low and high 32 bits are the same and bit 31 and 63 are set.
218 if (Seq.size() > 3) {
219 unsigned ShiftAmt, AddOpc;
221 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
222 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
223 SDValue Lo = selectImmSeq(CurDAG, DL, VT, SeqLo);
224
225 SDValue SLLI = SDValue(
226 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, Lo,
227 CurDAG->getTargetConstant(ShiftAmt, DL, VT)),
228 0);
229 return SDValue(CurDAG->getMachineNode(AddOpc, DL, VT, Lo, SLLI), 0);
230 }
231 }
232
233 // Otherwise, use the original sequence.
234 return selectImmSeq(CurDAG, DL, VT, Seq);
235}
236
238 unsigned NF, RISCVII::VLMUL LMUL) {
239 static const unsigned M1TupleRegClassIDs[] = {
240 RISCV::VRN2M1RegClassID, RISCV::VRN3M1RegClassID, RISCV::VRN4M1RegClassID,
241 RISCV::VRN5M1RegClassID, RISCV::VRN6M1RegClassID, RISCV::VRN7M1RegClassID,
242 RISCV::VRN8M1RegClassID};
243 static const unsigned M2TupleRegClassIDs[] = {RISCV::VRN2M2RegClassID,
244 RISCV::VRN3M2RegClassID,
245 RISCV::VRN4M2RegClassID};
246
247 assert(Regs.size() >= 2 && Regs.size() <= 8);
248
249 unsigned RegClassID;
250 unsigned SubReg0;
251 switch (LMUL) {
252 default:
253 llvm_unreachable("Invalid LMUL.");
258 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
259 "Unexpected subreg numbering");
260 SubReg0 = RISCV::sub_vrm1_0;
261 RegClassID = M1TupleRegClassIDs[NF - 2];
262 break;
264 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
265 "Unexpected subreg numbering");
266 SubReg0 = RISCV::sub_vrm2_0;
267 RegClassID = M2TupleRegClassIDs[NF - 2];
268 break;
270 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
271 "Unexpected subreg numbering");
272 SubReg0 = RISCV::sub_vrm4_0;
273 RegClassID = RISCV::VRN2M4RegClassID;
274 break;
275 }
276
277 SDLoc DL(Regs[0]);
279
280 Ops.push_back(CurDAG.getTargetConstant(RegClassID, DL, MVT::i32));
281
282 for (unsigned I = 0; I < Regs.size(); ++I) {
283 Ops.push_back(Regs[I]);
284 Ops.push_back(CurDAG.getTargetConstant(SubReg0 + I, DL, MVT::i32));
285 }
286 SDNode *N =
287 CurDAG.getMachineNode(TargetOpcode::REG_SEQUENCE, DL, MVT::Untyped, Ops);
288 return SDValue(N, 0);
289}
290
292 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
293 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
294 bool IsLoad, MVT *IndexVT) {
295 SDValue Chain = Node->getOperand(0);
296 SDValue Glue;
297
298 Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
299
300 if (IsStridedOrIndexed) {
301 Operands.push_back(Node->getOperand(CurOp++)); // Index.
302 if (IndexVT)
303 *IndexVT = Operands.back()->getSimpleValueType(0);
304 }
305
306 if (IsMasked) {
307 // Mask needs to be copied to V0.
308 SDValue Mask = Node->getOperand(CurOp++);
309 Chain = CurDAG->getCopyToReg(Chain, DL, RISCV::V0, Mask, SDValue());
310 Glue = Chain.getValue(1);
311 Operands.push_back(CurDAG->getRegister(RISCV::V0, Mask.getValueType()));
312 }
313 SDValue VL;
314 selectVLOp(Node->getOperand(CurOp++), VL);
315 Operands.push_back(VL);
316
317 MVT XLenVT = Subtarget->getXLenVT();
318 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
319 Operands.push_back(SEWOp);
320
321 // At the IR layer, all the masked load intrinsics have policy operands,
322 // none of the others do. All have passthru operands. For our pseudos,
323 // all loads have policy operands.
324 if (IsLoad) {
326 if (IsMasked)
327 Policy = Node->getConstantOperandVal(CurOp++);
328 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
329 Operands.push_back(PolicyOp);
330 }
331
332 Operands.push_back(Chain); // Chain.
333 if (Glue)
334 Operands.push_back(Glue);
335}
336
337void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, bool IsMasked,
338 bool IsStrided) {
339 SDLoc DL(Node);
340 unsigned NF = Node->getNumValues() - 1;
341 MVT VT = Node->getSimpleValueType(0);
342 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
344
345 unsigned CurOp = 2;
347
348 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
349 Node->op_begin() + CurOp + NF);
350 SDValue Merge = createTuple(*CurDAG, Regs, NF, LMUL);
351 Operands.push_back(Merge);
352 CurOp += NF;
353
354 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
355 Operands, /*IsLoad=*/true);
356
357 const RISCV::VLSEGPseudo *P =
358 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW,
359 static_cast<unsigned>(LMUL));
360 MachineSDNode *Load =
361 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
362
363 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
364 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
365
366 SDValue SuperReg = SDValue(Load, 0);
367 for (unsigned I = 0; I < NF; ++I) {
368 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
369 ReplaceUses(SDValue(Node, I),
370 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
371 }
372
373 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
374 CurDAG->RemoveDeadNode(Node);
375}
376
377void RISCVDAGToDAGISel::selectVLSEGFF(SDNode *Node, bool IsMasked) {
378 SDLoc DL(Node);
379 unsigned NF = Node->getNumValues() - 2; // Do not count VL and Chain.
380 MVT VT = Node->getSimpleValueType(0);
381 MVT XLenVT = Subtarget->getXLenVT();
382 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
384
385 unsigned CurOp = 2;
387
388 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
389 Node->op_begin() + CurOp + NF);
390 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
391 Operands.push_back(MaskedOff);
392 CurOp += NF;
393
394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
395 /*IsStridedOrIndexed*/ false, Operands,
396 /*IsLoad=*/true);
397
398 const RISCV::VLSEGPseudo *P =
399 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
400 Log2SEW, static_cast<unsigned>(LMUL));
401 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
402 XLenVT, MVT::Other, Operands);
403
404 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
405 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
406
407 SDValue SuperReg = SDValue(Load, 0);
408 for (unsigned I = 0; I < NF; ++I) {
409 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
410 ReplaceUses(SDValue(Node, I),
411 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
412 }
413
414 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1)); // VL
415 ReplaceUses(SDValue(Node, NF + 1), SDValue(Load, 2)); // Chain
416 CurDAG->RemoveDeadNode(Node);
417}
418
419void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, bool IsMasked,
420 bool IsOrdered) {
421 SDLoc DL(Node);
422 unsigned NF = Node->getNumValues() - 1;
423 MVT VT = Node->getSimpleValueType(0);
424 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
426
427 unsigned CurOp = 2;
429
430 SmallVector<SDValue, 8> Regs(Node->op_begin() + CurOp,
431 Node->op_begin() + CurOp + NF);
432 SDValue MaskedOff = createTuple(*CurDAG, Regs, NF, LMUL);
433 Operands.push_back(MaskedOff);
434 CurOp += NF;
435
436 MVT IndexVT;
437 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
438 /*IsStridedOrIndexed*/ true, Operands,
439 /*IsLoad=*/true, &IndexVT);
440
442 "Element count mismatch");
443
444 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
445 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
446 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
447 report_fatal_error("The V extension does not support EEW=64 for index "
448 "values when XLEN=32");
449 }
450 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
451 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
452 static_cast<unsigned>(IndexLMUL));
453 MachineSDNode *Load =
454 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
455
456 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
457 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
458
459 SDValue SuperReg = SDValue(Load, 0);
460 for (unsigned I = 0; I < NF; ++I) {
461 unsigned SubRegIdx = RISCVTargetLowering::getSubregIndexByMVT(VT, I);
462 ReplaceUses(SDValue(Node, I),
463 CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, SuperReg));
464 }
465
466 ReplaceUses(SDValue(Node, NF), SDValue(Load, 1));
467 CurDAG->RemoveDeadNode(Node);
468}
469
470void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, bool IsMasked,
471 bool IsStrided) {
472 SDLoc DL(Node);
473 unsigned NF = Node->getNumOperands() - 4;
474 if (IsStrided)
475 NF--;
476 if (IsMasked)
477 NF--;
478 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
479 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
481 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
482 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
483
485 Operands.push_back(StoreVal);
486 unsigned CurOp = 2 + NF;
487
488 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
489 Operands);
490
491 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
492 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
493 MachineSDNode *Store =
494 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
495
496 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
497 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
498
499 ReplaceNode(Node, Store);
500}
501
502void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, bool IsMasked,
503 bool IsOrdered) {
504 SDLoc DL(Node);
505 unsigned NF = Node->getNumOperands() - 5;
506 if (IsMasked)
507 --NF;
508 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
509 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
511 SmallVector<SDValue, 8> Regs(Node->op_begin() + 2, Node->op_begin() + 2 + NF);
512 SDValue StoreVal = createTuple(*CurDAG, Regs, NF, LMUL);
513
515 Operands.push_back(StoreVal);
516 unsigned CurOp = 2 + NF;
517
518 MVT IndexVT;
519 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
520 /*IsStridedOrIndexed*/ true, Operands,
521 /*IsLoad=*/false, &IndexVT);
522
524 "Element count mismatch");
525
526 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
527 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
528 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
529 report_fatal_error("The V extension does not support EEW=64 for index "
530 "values when XLEN=32");
531 }
532 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
533 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
534 static_cast<unsigned>(IndexLMUL));
535 MachineSDNode *Store =
536 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
537
538 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
539 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
540
541 ReplaceNode(Node, Store);
542}
543
545 if (!Subtarget->hasVInstructions())
546 return;
547
548 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
549
550 SDLoc DL(Node);
551 MVT XLenVT = Subtarget->getXLenVT();
552
553 unsigned IntNo = Node->getConstantOperandVal(0);
554
555 assert((IntNo == Intrinsic::riscv_vsetvli ||
556 IntNo == Intrinsic::riscv_vsetvlimax) &&
557 "Unexpected vsetvli intrinsic");
558
559 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
560 unsigned Offset = (VLMax ? 1 : 2);
561
562 assert(Node->getNumOperands() == Offset + 2 &&
563 "Unexpected number of operands");
564
565 unsigned SEW =
566 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
567 RISCVII::VLMUL VLMul = static_cast<RISCVII::VLMUL>(
568 Node->getConstantOperandVal(Offset + 1) & 0x7);
569
570 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
571 /*MaskAgnostic*/ true);
572 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
573
574 SDValue VLOperand;
575 unsigned Opcode = RISCV::PseudoVSETVLI;
576 if (auto *C = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
577 const unsigned VLEN = Subtarget->getRealMinVLen();
578 if (VLEN == Subtarget->getRealMaxVLen())
579 if (VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
580 VLMax = true;
581 }
582 if (VLMax || isAllOnesConstant(Node->getOperand(1))) {
583 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
584 Opcode = RISCV::PseudoVSETVLIX0;
585 } else {
586 VLOperand = Node->getOperand(1);
587
588 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
589 uint64_t AVL = C->getZExtValue();
590 if (isUInt<5>(AVL)) {
591 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
592 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL,
593 XLenVT, VLImm, VTypeIOp));
594 return;
595 }
596 }
597 }
598
599 ReplaceNode(Node,
600 CurDAG->getMachineNode(Opcode, DL, XLenVT, VLOperand, VTypeIOp));
601}
602
604 MVT VT = Node->getSimpleValueType(0);
605 unsigned Opcode = Node->getOpcode();
607 "Unexpected opcode");
608 SDLoc DL(Node);
609
610 // For operations of the form (x << C1) op C2, check if we can use
611 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
612 SDValue N0 = Node->getOperand(0);
613 SDValue N1 = Node->getOperand(1);
614
615 ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N1);
616 if (!Cst)
617 return false;
618
619 int64_t Val = Cst->getSExtValue();
620
621 // Check if immediate can already use ANDI/ORI/XORI.
622 if (isInt<12>(Val))
623 return false;
624
625 SDValue Shift = N0;
626
627 // If Val is simm32 and we have a sext_inreg from i32, then the binop
628 // produces at least 33 sign bits. We can peek through the sext_inreg and use
629 // a SLLIW at the end.
630 bool SignExt = false;
631 if (isInt<32>(Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
632 N0.hasOneUse() && cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32) {
633 SignExt = true;
634 Shift = N0.getOperand(0);
635 }
636
637 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
638 return false;
639
640 ConstantSDNode *ShlCst = dyn_cast<ConstantSDNode>(Shift.getOperand(1));
641 if (!ShlCst)
642 return false;
643
644 uint64_t ShAmt = ShlCst->getZExtValue();
645
646 // Make sure that we don't change the operation by removing bits.
647 // This only matters for OR and XOR, AND is unaffected.
648 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(ShAmt);
649 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
650 return false;
651
652 int64_t ShiftedVal = Val >> ShAmt;
653 if (!isInt<12>(ShiftedVal))
654 return false;
655
656 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
657 if (SignExt && ShAmt >= 32)
658 return false;
659
660 // Ok, we can reorder to get a smaller immediate.
661 unsigned BinOpc;
662 switch (Opcode) {
663 default: llvm_unreachable("Unexpected opcode");
664 case ISD::AND: BinOpc = RISCV::ANDI; break;
665 case ISD::OR: BinOpc = RISCV::ORI; break;
666 case ISD::XOR: BinOpc = RISCV::XORI; break;
667 }
668
669 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
670
671 SDNode *BinOp =
672 CurDAG->getMachineNode(BinOpc, DL, VT, Shift.getOperand(0),
673 CurDAG->getTargetConstant(ShiftedVal, DL, VT));
674 SDNode *SLLI =
675 CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
676 CurDAG->getTargetConstant(ShAmt, DL, VT));
677 ReplaceNode(Node, SLLI);
678 return true;
679}
680
682 // Only supported with XTHeadBb at the moment.
683 if (!Subtarget->hasVendorXTHeadBb())
684 return false;
685
686 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
687 if (!N1C)
688 return false;
689
690 SDValue N0 = Node->getOperand(0);
691 if (!N0.hasOneUse())
692 return false;
693
694 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb, SDLoc DL,
695 MVT VT) {
696 return CurDAG->getMachineNode(RISCV::TH_EXT, DL, VT, N0.getOperand(0),
697 CurDAG->getTargetConstant(Msb, DL, VT),
698 CurDAG->getTargetConstant(Lsb, DL, VT));
699 };
700
701 SDLoc DL(Node);
702 MVT VT = Node->getSimpleValueType(0);
703 const unsigned RightShAmt = N1C->getZExtValue();
704
705 // Transform (sra (shl X, C1) C2) with C1 < C2
706 // -> (TH.EXT X, msb, lsb)
707 if (N0.getOpcode() == ISD::SHL) {
708 auto *N01C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
709 if (!N01C)
710 return false;
711
712 const unsigned LeftShAmt = N01C->getZExtValue();
713 // Make sure that this is a bitfield extraction (i.e., the shift-right
714 // amount can not be less than the left-shift).
715 if (LeftShAmt > RightShAmt)
716 return false;
717
718 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
719 const unsigned Msb = MsbPlusOne - 1;
720 const unsigned Lsb = RightShAmt - LeftShAmt;
721
722 SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
723 ReplaceNode(Node, TH_EXT);
724 return true;
725 }
726
727 // Transform (sra (sext_inreg X, _), C) ->
728 // (TH.EXT X, msb, lsb)
729 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
730 unsigned ExtSize =
731 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
732
733 // ExtSize of 32 should use sraiw via tablegen pattern.
734 if (ExtSize == 32)
735 return false;
736
737 const unsigned Msb = ExtSize - 1;
738 const unsigned Lsb = RightShAmt;
739
740 SDNode *TH_EXT = BitfieldExtract(N0, Msb, Lsb, DL, VT);
741 ReplaceNode(Node, TH_EXT);
742 return true;
743 }
744
745 return false;
746}
747
749 // Target does not support indexed loads.
750 if (!Subtarget->hasVendorXTHeadMemIdx())
751 return false;
752
753 LoadSDNode *Ld = cast<LoadSDNode>(Node);
755 if (AM == ISD::UNINDEXED)
756 return false;
757
758 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Ld->getOffset());
759 if (!C)
760 return false;
761
762 EVT LoadVT = Ld->getMemoryVT();
763 bool IsPre = (AM == ISD::PRE_INC || AM == ISD::PRE_DEC);
764 bool IsPost = (AM == ISD::POST_INC || AM == ISD::POST_DEC);
765 int64_t Offset = C->getSExtValue();
766
767 // Convert decrements to increments by a negative quantity.
768 if (AM == ISD::PRE_DEC || AM == ISD::POST_DEC)
769 Offset = -Offset;
770
771 // The constants that can be encoded in the THeadMemIdx instructions
772 // are of the form (sign_extend(imm5) << imm2).
773 int64_t Shift;
774 for (Shift = 0; Shift < 4; Shift++)
775 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
776 break;
777
778 // Constant cannot be encoded.
779 if (Shift == 4)
780 return false;
781
782 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
783 unsigned Opcode;
784 if (LoadVT == MVT::i8 && IsPre)
785 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
786 else if (LoadVT == MVT::i8 && IsPost)
787 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
788 else if (LoadVT == MVT::i16 && IsPre)
789 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
790 else if (LoadVT == MVT::i16 && IsPost)
791 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
792 else if (LoadVT == MVT::i32 && IsPre)
793 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
794 else if (LoadVT == MVT::i32 && IsPost)
795 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
796 else if (LoadVT == MVT::i64 && IsPre)
797 Opcode = RISCV::TH_LDIB;
798 else if (LoadVT == MVT::i64 && IsPost)
799 Opcode = RISCV::TH_LDIA;
800 else
801 return false;
802
803 EVT Ty = Ld->getOffset().getValueType();
804 SDValue Ops[] = {Ld->getBasePtr(),
805 CurDAG->getTargetConstant(Offset >> Shift, SDLoc(Node), Ty),
806 CurDAG->getTargetConstant(Shift, SDLoc(Node), Ty),
807 Ld->getChain()};
808 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(Node), Ld->getValueType(0),
809 Ld->getValueType(1), MVT::Other, Ops);
810
811 MachineMemOperand *MemOp = cast<MemSDNode>(Node)->getMemOperand();
812 CurDAG->setNodeMemRefs(cast<MachineSDNode>(New), {MemOp});
813
814 ReplaceNode(Node, New);
815
816 return true;
817}
818
820 // If we have a custom node, we have already selected.
821 if (Node->isMachineOpcode()) {
822 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
823 Node->setNodeId(-1);
824 return;
825 }
826
827 // Instruction Selection not handled by the auto-generated tablegen selection
828 // should be handled here.
829 unsigned Opcode = Node->getOpcode();
830 MVT XLenVT = Subtarget->getXLenVT();
831 SDLoc DL(Node);
832 MVT VT = Node->getSimpleValueType(0);
833
834 bool HasBitTest = Subtarget->hasStdExtZbs() || Subtarget->hasVendorXTHeadBs();
835
836 switch (Opcode) {
837 case ISD::Constant: {
838 assert((VT == Subtarget->getXLenVT() || VT == MVT::i32) && "Unexpected VT");
839 auto *ConstNode = cast<ConstantSDNode>(Node);
840 if (ConstNode->isZero()) {
841 SDValue New =
842 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT);
843 ReplaceNode(Node, New.getNode());
844 return;
845 }
846 int64_t Imm = ConstNode->getSExtValue();
847 // If the upper XLen-16 bits are not used, try to convert this to a simm12
848 // by sign extending bit 15.
849 if (isUInt<16>(Imm) && isInt<12>(SignExtend64<16>(Imm)) &&
850 hasAllHUsers(Node))
851 Imm = SignExtend64<16>(Imm);
852 // If the upper 32-bits are not used try to convert this into a simm32 by
853 // sign extending bit 32.
854 if (!isInt<32>(Imm) && isUInt<32>(Imm) && hasAllWUsers(Node))
855 Imm = SignExtend64<32>(Imm);
856
857 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget).getNode());
858 return;
859 }
860 case ISD::ConstantFP: {
861 const APFloat &APF = cast<ConstantFPSDNode>(Node)->getValueAPF();
862 auto [FPImm, NeedsFNeg] =
863 static_cast<const RISCVTargetLowering *>(TLI)->getLegalZfaFPImm(APF,
864 VT);
865 if (FPImm >= 0) {
866 unsigned Opc;
867 unsigned FNegOpc;
868 switch (VT.SimpleTy) {
869 default:
870 llvm_unreachable("Unexpected size");
871 case MVT::f16:
872 Opc = RISCV::FLI_H;
873 FNegOpc = RISCV::FSGNJN_H;
874 break;
875 case MVT::f32:
876 Opc = RISCV::FLI_S;
877 FNegOpc = RISCV::FSGNJN_S;
878 break;
879 case MVT::f64:
880 Opc = RISCV::FLI_D;
881 FNegOpc = RISCV::FSGNJN_D;
882 break;
883 }
885 Opc, DL, VT, CurDAG->getTargetConstant(FPImm, DL, XLenVT));
886 if (NeedsFNeg)
887 Res = CurDAG->getMachineNode(FNegOpc, DL, VT, SDValue(Res, 0),
888 SDValue(Res, 0));
889
890 ReplaceNode(Node, Res);
891 return;
892 }
893
894 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
895 SDValue Imm;
896 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
897 // create an integer immediate.
898 if (APF.isPosZero() || NegZeroF64)
899 Imm = CurDAG->getRegister(RISCV::X0, XLenVT);
900 else
901 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
902 *Subtarget);
903
904 bool HasZdinx = Subtarget->hasStdExtZdinx();
905 bool Is64Bit = Subtarget->is64Bit();
906 unsigned Opc;
907 switch (VT.SimpleTy) {
908 default:
909 llvm_unreachable("Unexpected size");
910 case MVT::bf16:
911 assert(Subtarget->hasStdExtZfbfmin());
912 Opc = RISCV::FMV_H_X;
913 break;
914 case MVT::f16:
915 Opc =
916 Subtarget->hasStdExtZhinxOrZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
917 break;
918 case MVT::f32:
919 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
920 break;
921 case MVT::f64:
922 // For RV32, we can't move from a GPR, we need to convert instead. This
923 // should only happen for +0.0 and -0.0.
924 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
925 if (Is64Bit)
926 Opc = HasZdinx ? RISCV::COPY : RISCV::FMV_D_X;
927 else
928 Opc = HasZdinx ? RISCV::FCVT_D_W_IN32X : RISCV::FCVT_D_W;
929 break;
930 }
931
932 SDNode *Res;
933 if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
934 Res = CurDAG->getMachineNode(
935 Opc, DL, VT, Imm,
937 else
938 Res = CurDAG->getMachineNode(Opc, DL, VT, Imm);
939
940 // For f64 -0.0, we need to insert a fneg.d idiom.
941 if (NegZeroF64) {
942 Opc = RISCV::FSGNJN_D;
943 if (HasZdinx)
944 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
945 Res =
946 CurDAG->getMachineNode(Opc, DL, VT, SDValue(Res, 0), SDValue(Res, 0));
947 }
948
949 ReplaceNode(Node, Res);
950 return;
951 }
952 case RISCVISD::SplitF64: {
953 if (!Subtarget->hasStdExtZfa())
954 break;
955 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
956 "Unexpected subtarget");
957
958 // With Zfa, lower to fmv.x.w and fmvh.x.d.
959 if (!SDValue(Node, 0).use_empty()) {
960 SDNode *Lo = CurDAG->getMachineNode(RISCV::FMV_X_W_FPR64, DL, VT,
961 Node->getOperand(0));
962 ReplaceUses(SDValue(Node, 0), SDValue(Lo, 0));
963 }
964 if (!SDValue(Node, 1).use_empty()) {
965 SDNode *Hi = CurDAG->getMachineNode(RISCV::FMVH_X_D, DL, VT,
966 Node->getOperand(0));
967 ReplaceUses(SDValue(Node, 1), SDValue(Hi, 0));
968 }
969
970 CurDAG->RemoveDeadNode(Node);
971 return;
972 }
973 case ISD::SHL: {
974 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
975 if (!N1C)
976 break;
977 SDValue N0 = Node->getOperand(0);
978 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
979 !isa<ConstantSDNode>(N0.getOperand(1)))
980 break;
981 unsigned ShAmt = N1C->getZExtValue();
982 uint64_t Mask = N0.getConstantOperandVal(1);
983
984 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C) where C2 has
985 // 32 leading zeros and C3 trailing zeros.
986 if (ShAmt <= 32 && isShiftedMask_64(Mask)) {
987 unsigned XLen = Subtarget->getXLen();
988 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
989 unsigned TrailingZeros = llvm::countr_zero(Mask);
990 if (TrailingZeros > 0 && LeadingZeros == 32) {
991 SDNode *SRLIW = CurDAG->getMachineNode(
992 RISCV::SRLIW, DL, VT, N0->getOperand(0),
993 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
995 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
996 CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT));
997 ReplaceNode(Node, SLLI);
998 return;
999 }
1000 }
1001 break;
1002 }
1003 case ISD::SRL: {
1004 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1005 if (!N1C)
1006 break;
1007 SDValue N0 = Node->getOperand(0);
1008 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1009 break;
1010 unsigned ShAmt = N1C->getZExtValue();
1011 uint64_t Mask = N0.getConstantOperandVal(1);
1012
1013 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1014 // 32 leading zeros and C3 trailing zeros.
1015 if (isShiftedMask_64(Mask) && N0.hasOneUse()) {
1016 unsigned XLen = Subtarget->getXLen();
1017 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
1018 unsigned TrailingZeros = llvm::countr_zero(Mask);
1019 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1020 SDNode *SRLIW = CurDAG->getMachineNode(
1021 RISCV::SRLIW, DL, VT, N0->getOperand(0),
1022 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
1023 SDNode *SLLI = CurDAG->getMachineNode(
1024 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1025 CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT));
1026 ReplaceNode(Node, SLLI);
1027 return;
1028 }
1029 }
1030
1031 // Optimize (srl (and X, C2), C) ->
1032 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1033 // Where C2 is a mask with C3 trailing ones.
1034 // Taking into account that the C2 may have had lower bits unset by
1035 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1036 // This pattern occurs when type legalizing right shifts for types with
1037 // less than XLen bits.
1038 Mask |= maskTrailingOnes<uint64_t>(ShAmt);
1039 if (!isMask_64(Mask))
1040 break;
1041 unsigned TrailingOnes = llvm::countr_one(Mask);
1042 if (ShAmt >= TrailingOnes)
1043 break;
1044 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1045 if (TrailingOnes == 32) {
1046 SDNode *SRLI = CurDAG->getMachineNode(
1047 Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, DL, VT,
1048 N0->getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
1049 ReplaceNode(Node, SRLI);
1050 return;
1051 }
1052
1053 // Only do the remaining transforms if the AND has one use.
1054 if (!N0.hasOneUse())
1055 break;
1056
1057 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1058 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1059 SDNode *BEXTI = CurDAG->getMachineNode(
1060 Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, DL, VT,
1061 N0->getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
1062 ReplaceNode(Node, BEXTI);
1063 return;
1064 }
1065
1066 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1067 SDNode *SLLI =
1068 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
1069 CurDAG->getTargetConstant(LShAmt, DL, VT));
1070 SDNode *SRLI = CurDAG->getMachineNode(
1071 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1072 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1073 ReplaceNode(Node, SRLI);
1074 return;
1075 }
1076 case ISD::SRA: {
1077 if (trySignedBitfieldExtract(Node))
1078 return;
1079
1080 // Optimize (sra (sext_inreg X, i16), C) ->
1081 // (srai (slli X, (XLen-16), (XLen-16) + C)
1082 // And (sra (sext_inreg X, i8), C) ->
1083 // (srai (slli X, (XLen-8), (XLen-8) + C)
1084 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1085 // This transform matches the code we get without Zbb. The shifts are more
1086 // compressible, and this can help expose CSE opportunities in the sdiv by
1087 // constant optimization.
1088 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1089 if (!N1C)
1090 break;
1091 SDValue N0 = Node->getOperand(0);
1092 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1093 break;
1094 unsigned ShAmt = N1C->getZExtValue();
1095 unsigned ExtSize =
1096 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
1097 // ExtSize of 32 should use sraiw via tablegen pattern.
1098 if (ExtSize >= 32 || ShAmt >= ExtSize)
1099 break;
1100 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1101 SDNode *SLLI =
1102 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0->getOperand(0),
1103 CurDAG->getTargetConstant(LShAmt, DL, VT));
1104 SDNode *SRAI = CurDAG->getMachineNode(
1105 RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
1106 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1107 ReplaceNode(Node, SRAI);
1108 return;
1109 }
1110 case ISD::OR:
1111 case ISD::XOR:
1112 if (tryShrinkShlLogicImm(Node))
1113 return;
1114
1115 break;
1116 case ISD::AND: {
1117 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1118 if (!N1C)
1119 break;
1120 uint64_t C1 = N1C->getZExtValue();
1121 const bool isC1Mask = isMask_64(C1);
1122 const bool isC1ANDI = isInt<12>(C1);
1123
1124 SDValue N0 = Node->getOperand(0);
1125
1126 auto tryUnsignedBitfieldExtract = [&](SDNode *Node, SDLoc DL, MVT VT,
1127 SDValue X, unsigned Msb,
1128 unsigned Lsb) {
1129 if (!Subtarget->hasVendorXTHeadBb())
1130 return false;
1131
1132 SDNode *TH_EXTU = CurDAG->getMachineNode(
1133 RISCV::TH_EXTU, DL, VT, X, CurDAG->getTargetConstant(Msb, DL, VT),
1134 CurDAG->getTargetConstant(Lsb, DL, VT));
1135 ReplaceNode(Node, TH_EXTU);
1136 return true;
1137 };
1138
1139 bool LeftShift = N0.getOpcode() == ISD::SHL;
1140 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1141 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
1142 if (!C)
1143 break;
1144 unsigned C2 = C->getZExtValue();
1145 unsigned XLen = Subtarget->getXLen();
1146 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1147
1148 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1149 // shift pair might offer more compression opportunities.
1150 // TODO: We could check for C extension here, but we don't have many lit
1151 // tests with the C extension enabled so not checking gets better
1152 // coverage.
1153 // TODO: What if ANDI faster than shift?
1154 bool IsCANDI = isInt<6>(N1C->getSExtValue());
1155
1156 // Clear irrelevant bits in the mask.
1157 if (LeftShift)
1158 C1 &= maskTrailingZeros<uint64_t>(C2);
1159 else
1160 C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
1161
1162 // Some transforms should only be done if the shift has a single use or
1163 // the AND would become (srli (slli X, 32), 32)
1164 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1165
1166 SDValue X = N0.getOperand(0);
1167
1168 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1169 // with c3 leading zeros.
1170 if (!LeftShift && isC1Mask) {
1171 unsigned Leading = XLen - llvm::bit_width(C1);
1172 if (C2 < Leading) {
1173 // If the number of leading zeros is C2+32 this can be SRLIW.
1174 if (C2 + 32 == Leading) {
1175 SDNode *SRLIW = CurDAG->getMachineNode(
1176 RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT));
1177 ReplaceNode(Node, SRLIW);
1178 return;
1179 }
1180
1181 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1182 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1183 //
1184 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1185 // legalized and goes through DAG combine.
1186 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1187 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1188 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) {
1189 SDNode *SRAIW =
1190 CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0),
1191 CurDAG->getTargetConstant(31, DL, VT));
1192 SDNode *SRLIW = CurDAG->getMachineNode(
1193 RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0),
1194 CurDAG->getTargetConstant(Leading - 32, DL, VT));
1195 ReplaceNode(Node, SRLIW);
1196 return;
1197 }
1198
1199 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1200 // available.
1201 // Transform (and (srl x, C2), C1)
1202 // -> (<bfextract> x, msb, lsb)
1203 //
1204 // Make sure to keep this below the SRLIW cases, as we always want to
1205 // prefer the more common instruction.
1206 const unsigned Msb = llvm::bit_width(C1) + C2 - 1;
1207 const unsigned Lsb = C2;
1208 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1209 return;
1210
1211 // (srli (slli x, c3-c2), c3).
1212 // Skip if we could use (zext.w (sraiw X, C2)).
1213 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1214 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1215 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32;
1216 // Also Skip if we can use bexti or th.tst.
1217 Skip |= HasBitTest && Leading == XLen - 1;
1218 if (OneUseOrZExtW && !Skip) {
1219 SDNode *SLLI = CurDAG->getMachineNode(
1220 RISCV::SLLI, DL, VT, X,
1221 CurDAG->getTargetConstant(Leading - C2, DL, VT));
1222 SDNode *SRLI = CurDAG->getMachineNode(
1223 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1224 CurDAG->getTargetConstant(Leading, DL, VT));
1225 ReplaceNode(Node, SRLI);
1226 return;
1227 }
1228 }
1229 }
1230
1231 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1232 // shifted by c2 bits with c3 leading zeros.
1233 if (LeftShift && isShiftedMask_64(C1)) {
1234 unsigned Leading = XLen - llvm::bit_width(C1);
1235
1236 if (C2 + Leading < XLen &&
1237 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) {
1238 // Use slli.uw when possible.
1239 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1240 SDNode *SLLI_UW =
1241 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, VT, X,
1242 CurDAG->getTargetConstant(C2, DL, VT));
1243 ReplaceNode(Node, SLLI_UW);
1244 return;
1245 }
1246
1247 // (srli (slli c2+c3), c3)
1248 if (OneUseOrZExtW && !IsCANDI) {
1249 SDNode *SLLI = CurDAG->getMachineNode(
1250 RISCV::SLLI, DL, VT, X,
1251 CurDAG->getTargetConstant(C2 + Leading, DL, VT));
1252 SDNode *SRLI = CurDAG->getMachineNode(
1253 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1254 CurDAG->getTargetConstant(Leading, DL, VT));
1255 ReplaceNode(Node, SRLI);
1256 return;
1257 }
1258 }
1259 }
1260
1261 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1262 // shifted mask with c2 leading zeros and c3 trailing zeros.
1263 if (!LeftShift && isShiftedMask_64(C1)) {
1264 unsigned Leading = XLen - llvm::bit_width(C1);
1265 unsigned Trailing = llvm::countr_zero(C1);
1266 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1267 !IsCANDI) {
1268 unsigned SrliOpc = RISCV::SRLI;
1269 // If the input is zexti32 we should use SRLIW.
1270 if (X.getOpcode() == ISD::AND &&
1271 isa<ConstantSDNode>(X.getOperand(1)) &&
1272 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) {
1273 SrliOpc = RISCV::SRLIW;
1274 X = X.getOperand(0);
1275 }
1276 SDNode *SRLI = CurDAG->getMachineNode(
1277 SrliOpc, DL, VT, X,
1278 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1279 SDNode *SLLI = CurDAG->getMachineNode(
1280 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1281 CurDAG->getTargetConstant(Trailing, DL, VT));
1282 ReplaceNode(Node, SLLI);
1283 return;
1284 }
1285 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1286 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1287 OneUseOrZExtW && !IsCANDI) {
1288 SDNode *SRLIW = CurDAG->getMachineNode(
1289 RISCV::SRLIW, DL, VT, X,
1290 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1291 SDNode *SLLI = CurDAG->getMachineNode(
1292 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1293 CurDAG->getTargetConstant(Trailing, DL, VT));
1294 ReplaceNode(Node, SLLI);
1295 return;
1296 }
1297 }
1298
1299 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1300 // shifted mask with no leading zeros and c3 trailing zeros.
1301 if (LeftShift && isShiftedMask_64(C1)) {
1302 unsigned Leading = XLen - llvm::bit_width(C1);
1303 unsigned Trailing = llvm::countr_zero(C1);
1304 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1305 SDNode *SRLI = CurDAG->getMachineNode(
1306 RISCV::SRLI, DL, VT, X,
1307 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1308 SDNode *SLLI = CurDAG->getMachineNode(
1309 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1310 CurDAG->getTargetConstant(Trailing, DL, VT));
1311 ReplaceNode(Node, SLLI);
1312 return;
1313 }
1314 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1315 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1316 SDNode *SRLIW = CurDAG->getMachineNode(
1317 RISCV::SRLIW, DL, VT, X,
1318 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1319 SDNode *SLLI = CurDAG->getMachineNode(
1320 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1321 CurDAG->getTargetConstant(Trailing, DL, VT));
1322 ReplaceNode(Node, SLLI);
1323 return;
1324 }
1325 }
1326 }
1327
1328 // If C1 masks off the upper bits only (but can't be formed as an
1329 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1330 // available.
1331 // Transform (and x, C1)
1332 // -> (<bfextract> x, msb, lsb)
1333 if (isC1Mask && !isC1ANDI) {
1334 const unsigned Msb = llvm::bit_width(C1) - 1;
1335 if (tryUnsignedBitfieldExtract(Node, DL, VT, N0, Msb, 0))
1336 return;
1337 }
1338
1339 if (tryShrinkShlLogicImm(Node))
1340 return;
1341
1342 break;
1343 }
1344 case ISD::MUL: {
1345 // Special case for calculating (mul (and X, C2), C1) where the full product
1346 // fits in XLen bits. We can shift X left by the number of leading zeros in
1347 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1348 // product has XLen trailing zeros, putting it in the output of MULHU. This
1349 // can avoid materializing a constant in a register for C2.
1350
1351 // RHS should be a constant.
1352 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1353 if (!N1C || !N1C->hasOneUse())
1354 break;
1355
1356 // LHS should be an AND with constant.
1357 SDValue N0 = Node->getOperand(0);
1358 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1359 break;
1360
1361 uint64_t C2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
1362
1363 // Constant should be a mask.
1364 if (!isMask_64(C2))
1365 break;
1366
1367 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1368 // multiple users or the constant is a simm12. This prevents inserting a
1369 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1370 // make it more costly to materialize. Otherwise, using a SLLI might allow
1371 // it to be compressed.
1372 bool IsANDIOrZExt =
1373 isInt<12>(C2) ||
1374 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1375 // With XTHeadBb, we can use TH.EXTU.
1376 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1377 if (IsANDIOrZExt && (isInt<12>(N1C->getSExtValue()) || !N0.hasOneUse()))
1378 break;
1379 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1380 // the constant is a simm32.
1381 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1382 // With XTHeadBb, we can use TH.EXTU.
1383 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1384 if (IsZExtW && (isInt<32>(N1C->getSExtValue()) || !N0.hasOneUse()))
1385 break;
1386
1387 // We need to shift left the AND input and C1 by a total of XLen bits.
1388
1389 // How far left do we need to shift the AND input?
1390 unsigned XLen = Subtarget->getXLen();
1391 unsigned LeadingZeros = XLen - llvm::bit_width(C2);
1392
1393 // The constant gets shifted by the remaining amount unless that would
1394 // shift bits out.
1395 uint64_t C1 = N1C->getZExtValue();
1396 unsigned ConstantShift = XLen - LeadingZeros;
1397 if (ConstantShift > (XLen - llvm::bit_width(C1)))
1398 break;
1399
1400 uint64_t ShiftedC1 = C1 << ConstantShift;
1401 // If this RV32, we need to sign extend the constant.
1402 if (XLen == 32)
1403 ShiftedC1 = SignExtend64<32>(ShiftedC1);
1404
1405 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1406 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget).getNode();
1407 SDNode *SLLI =
1408 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1409 CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1410 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT,
1411 SDValue(SLLI, 0), SDValue(Imm, 0));
1412 ReplaceNode(Node, MULHU);
1413 return;
1414 }
1415 case ISD::LOAD: {
1416 if (tryIndexedLoad(Node))
1417 return;
1418 break;
1419 }
1421 unsigned IntNo = Node->getConstantOperandVal(0);
1422 switch (IntNo) {
1423 // By default we do not custom select any intrinsic.
1424 default:
1425 break;
1426 case Intrinsic::riscv_vmsgeu:
1427 case Intrinsic::riscv_vmsge: {
1428 SDValue Src1 = Node->getOperand(1);
1429 SDValue Src2 = Node->getOperand(2);
1430 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
1431 bool IsCmpUnsignedZero = false;
1432 // Only custom select scalar second operand.
1433 if (Src2.getValueType() != XLenVT)
1434 break;
1435 // Small constants are handled with patterns.
1436 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1437 int64_t CVal = C->getSExtValue();
1438 if (CVal >= -15 && CVal <= 16) {
1439 if (!IsUnsigned || CVal != 0)
1440 break;
1441 IsCmpUnsignedZero = true;
1442 }
1443 }
1444 MVT Src1VT = Src1.getSimpleValueType();
1445 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode;
1446 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1447 default:
1448 llvm_unreachable("Unexpected LMUL!");
1449#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b) \
1450 case RISCVII::VLMUL::lmulenum: \
1451 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1452 : RISCV::PseudoVMSLT_VX_##suffix; \
1453 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
1454 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix_b; \
1455 break;
1456 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F8, MF8, B1)
1457 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F4, MF4, B2)
1458 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_F2, MF2, B4)
1460 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_2, M2, B16)
1461 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_4, M4, B32)
1462 CASE_VMSLT_VMNAND_VMSET_OPCODES(LMUL_8, M8, B64)
1463#undef CASE_VMSLT_VMNAND_VMSET_OPCODES
1464 }
1466 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1467 SDValue VL;
1468 selectVLOp(Node->getOperand(3), VL);
1469
1470 // If vmsgeu with 0 immediate, expand it to vmset.
1471 if (IsCmpUnsignedZero) {
1472 ReplaceNode(Node, CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, SEW));
1473 return;
1474 }
1475
1476 // Expand to
1477 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
1478 SDValue Cmp = SDValue(
1479 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1480 0);
1481 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
1482 {Cmp, Cmp, VL, SEW}));
1483 return;
1484 }
1485 case Intrinsic::riscv_vmsgeu_mask:
1486 case Intrinsic::riscv_vmsge_mask: {
1487 SDValue Src1 = Node->getOperand(2);
1488 SDValue Src2 = Node->getOperand(3);
1489 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
1490 bool IsCmpUnsignedZero = false;
1491 // Only custom select scalar second operand.
1492 if (Src2.getValueType() != XLenVT)
1493 break;
1494 // Small constants are handled with patterns.
1495 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
1496 int64_t CVal = C->getSExtValue();
1497 if (CVal >= -15 && CVal <= 16) {
1498 if (!IsUnsigned || CVal != 0)
1499 break;
1500 IsCmpUnsignedZero = true;
1501 }
1502 }
1503 MVT Src1VT = Src1.getSimpleValueType();
1504 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
1505 VMOROpcode;
1506 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
1507 default:
1508 llvm_unreachable("Unexpected LMUL!");
1509#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b) \
1510 case RISCVII::VLMUL::lmulenum: \
1511 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
1512 : RISCV::PseudoVMSLT_VX_##suffix; \
1513 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
1514 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
1515 break;
1516 CASE_VMSLT_OPCODES(LMUL_F8, MF8, B1)
1517 CASE_VMSLT_OPCODES(LMUL_F4, MF4, B2)
1518 CASE_VMSLT_OPCODES(LMUL_F2, MF2, B4)
1519 CASE_VMSLT_OPCODES(LMUL_1, M1, B8)
1520 CASE_VMSLT_OPCODES(LMUL_2, M2, B16)
1521 CASE_VMSLT_OPCODES(LMUL_4, M4, B32)
1522 CASE_VMSLT_OPCODES(LMUL_8, M8, B64)
1523#undef CASE_VMSLT_OPCODES
1524 }
1525 // Mask operations use the LMUL from the mask type.
1526 switch (RISCVTargetLowering::getLMUL(VT)) {
1527 default:
1528 llvm_unreachable("Unexpected LMUL!");
1529#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
1530 case RISCVII::VLMUL::lmulenum: \
1531 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
1532 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
1533 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
1534 break;
1535 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, MF8)
1536 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, MF4)
1537 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, MF2)
1542#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
1543 }
1545 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
1546 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
1547 SDValue VL;
1548 selectVLOp(Node->getOperand(5), VL);
1549 SDValue MaskedOff = Node->getOperand(1);
1550 SDValue Mask = Node->getOperand(4);
1551
1552 // If vmsgeu_mask with 0 immediate, expand it to vmor mask, maskedoff.
1553 if (IsCmpUnsignedZero) {
1554 // We don't need vmor if the MaskedOff and the Mask are the same
1555 // value.
1556 if (Mask == MaskedOff) {
1557 ReplaceUses(Node, Mask.getNode());
1558 return;
1559 }
1560 ReplaceNode(Node,
1561 CurDAG->getMachineNode(VMOROpcode, DL, VT,
1562 {Mask, MaskedOff, VL, MaskSEW}));
1563 return;
1564 }
1565
1566 // If the MaskedOff value and the Mask are the same value use
1567 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
1568 // This avoids needing to copy v0 to vd before starting the next sequence.
1569 if (Mask == MaskedOff) {
1570 SDValue Cmp = SDValue(
1571 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
1572 0);
1573 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT,
1574 {Mask, Cmp, VL, MaskSEW}));
1575 return;
1576 }
1577
1578 // Mask needs to be copied to V0.
1580 RISCV::V0, Mask, SDValue());
1581 SDValue Glue = Chain.getValue(1);
1582 SDValue V0 = CurDAG->getRegister(RISCV::V0, VT);
1583
1584 // Otherwise use
1585 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
1586 // The result is mask undisturbed.
1587 // We use the same instructions to emulate mask agnostic behavior, because
1588 // the agnostic result can be either undisturbed or all 1.
1589 SDValue Cmp = SDValue(
1590 CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
1591 {MaskedOff, Src1, Src2, V0, VL, SEW, Glue}),
1592 0);
1593 // vmxor.mm vd, vd, v0 is used to update active value.
1594 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
1595 {Cmp, Mask, VL, MaskSEW}));
1596 return;
1597 }
1598 case Intrinsic::riscv_vsetvli:
1599 case Intrinsic::riscv_vsetvlimax:
1600 return selectVSETVLI(Node);
1601 }
1602 break;
1603 }
1605 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1606 switch (IntNo) {
1607 // By default we do not custom select any intrinsic.
1608 default:
1609 break;
1610 case Intrinsic::riscv_vlseg2:
1611 case Intrinsic::riscv_vlseg3:
1612 case Intrinsic::riscv_vlseg4:
1613 case Intrinsic::riscv_vlseg5:
1614 case Intrinsic::riscv_vlseg6:
1615 case Intrinsic::riscv_vlseg7:
1616 case Intrinsic::riscv_vlseg8: {
1617 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1618 return;
1619 }
1620 case Intrinsic::riscv_vlseg2_mask:
1621 case Intrinsic::riscv_vlseg3_mask:
1622 case Intrinsic::riscv_vlseg4_mask:
1623 case Intrinsic::riscv_vlseg5_mask:
1624 case Intrinsic::riscv_vlseg6_mask:
1625 case Intrinsic::riscv_vlseg7_mask:
1626 case Intrinsic::riscv_vlseg8_mask: {
1627 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1628 return;
1629 }
1630 case Intrinsic::riscv_vlsseg2:
1631 case Intrinsic::riscv_vlsseg3:
1632 case Intrinsic::riscv_vlsseg4:
1633 case Intrinsic::riscv_vlsseg5:
1634 case Intrinsic::riscv_vlsseg6:
1635 case Intrinsic::riscv_vlsseg7:
1636 case Intrinsic::riscv_vlsseg8: {
1637 selectVLSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1638 return;
1639 }
1640 case Intrinsic::riscv_vlsseg2_mask:
1641 case Intrinsic::riscv_vlsseg3_mask:
1642 case Intrinsic::riscv_vlsseg4_mask:
1643 case Intrinsic::riscv_vlsseg5_mask:
1644 case Intrinsic::riscv_vlsseg6_mask:
1645 case Intrinsic::riscv_vlsseg7_mask:
1646 case Intrinsic::riscv_vlsseg8_mask: {
1647 selectVLSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1648 return;
1649 }
1650 case Intrinsic::riscv_vloxseg2:
1651 case Intrinsic::riscv_vloxseg3:
1652 case Intrinsic::riscv_vloxseg4:
1653 case Intrinsic::riscv_vloxseg5:
1654 case Intrinsic::riscv_vloxseg6:
1655 case Intrinsic::riscv_vloxseg7:
1656 case Intrinsic::riscv_vloxseg8:
1657 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1658 return;
1659 case Intrinsic::riscv_vluxseg2:
1660 case Intrinsic::riscv_vluxseg3:
1661 case Intrinsic::riscv_vluxseg4:
1662 case Intrinsic::riscv_vluxseg5:
1663 case Intrinsic::riscv_vluxseg6:
1664 case Intrinsic::riscv_vluxseg7:
1665 case Intrinsic::riscv_vluxseg8:
1666 selectVLXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1667 return;
1668 case Intrinsic::riscv_vloxseg2_mask:
1669 case Intrinsic::riscv_vloxseg3_mask:
1670 case Intrinsic::riscv_vloxseg4_mask:
1671 case Intrinsic::riscv_vloxseg5_mask:
1672 case Intrinsic::riscv_vloxseg6_mask:
1673 case Intrinsic::riscv_vloxseg7_mask:
1674 case Intrinsic::riscv_vloxseg8_mask:
1675 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1676 return;
1677 case Intrinsic::riscv_vluxseg2_mask:
1678 case Intrinsic::riscv_vluxseg3_mask:
1679 case Intrinsic::riscv_vluxseg4_mask:
1680 case Intrinsic::riscv_vluxseg5_mask:
1681 case Intrinsic::riscv_vluxseg6_mask:
1682 case Intrinsic::riscv_vluxseg7_mask:
1683 case Intrinsic::riscv_vluxseg8_mask:
1684 selectVLXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1685 return;
1686 case Intrinsic::riscv_vlseg8ff:
1687 case Intrinsic::riscv_vlseg7ff:
1688 case Intrinsic::riscv_vlseg6ff:
1689 case Intrinsic::riscv_vlseg5ff:
1690 case Intrinsic::riscv_vlseg4ff:
1691 case Intrinsic::riscv_vlseg3ff:
1692 case Intrinsic::riscv_vlseg2ff: {
1693 selectVLSEGFF(Node, /*IsMasked*/ false);
1694 return;
1695 }
1696 case Intrinsic::riscv_vlseg8ff_mask:
1697 case Intrinsic::riscv_vlseg7ff_mask:
1698 case Intrinsic::riscv_vlseg6ff_mask:
1699 case Intrinsic::riscv_vlseg5ff_mask:
1700 case Intrinsic::riscv_vlseg4ff_mask:
1701 case Intrinsic::riscv_vlseg3ff_mask:
1702 case Intrinsic::riscv_vlseg2ff_mask: {
1703 selectVLSEGFF(Node, /*IsMasked*/ true);
1704 return;
1705 }
1706 case Intrinsic::riscv_vloxei:
1707 case Intrinsic::riscv_vloxei_mask:
1708 case Intrinsic::riscv_vluxei:
1709 case Intrinsic::riscv_vluxei_mask: {
1710 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
1711 IntNo == Intrinsic::riscv_vluxei_mask;
1712 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
1713 IntNo == Intrinsic::riscv_vloxei_mask;
1714
1715 MVT VT = Node->getSimpleValueType(0);
1716 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1717
1718 unsigned CurOp = 2;
1720 Operands.push_back(Node->getOperand(CurOp++));
1721
1722 MVT IndexVT;
1723 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1724 /*IsStridedOrIndexed*/ true, Operands,
1725 /*IsLoad=*/true, &IndexVT);
1726
1728 "Element count mismatch");
1729
1731 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1732 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1733 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1734 report_fatal_error("The V extension does not support EEW=64 for index "
1735 "values when XLEN=32");
1736 }
1737 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1738 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1739 static_cast<unsigned>(IndexLMUL));
1740 MachineSDNode *Load =
1741 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1742
1743 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1744 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1745
1746 ReplaceNode(Node, Load);
1747 return;
1748 }
1749 case Intrinsic::riscv_vlm:
1750 case Intrinsic::riscv_vle:
1751 case Intrinsic::riscv_vle_mask:
1752 case Intrinsic::riscv_vlse:
1753 case Intrinsic::riscv_vlse_mask: {
1754 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
1755 IntNo == Intrinsic::riscv_vlse_mask;
1756 bool IsStrided =
1757 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
1758
1759 MVT VT = Node->getSimpleValueType(0);
1760 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1761
1762 // The riscv_vlm intrinsic are always tail agnostic and no passthru
1763 // operand at the IR level. In pseudos, they have both policy and
1764 // passthru operand. The passthru operand is needed to track the
1765 // "tail undefined" state, and the policy is there just for
1766 // for consistency - it will always be "don't care" for the
1767 // unmasked form.
1768 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
1769 unsigned CurOp = 2;
1771 if (HasPassthruOperand)
1772 Operands.push_back(Node->getOperand(CurOp++));
1773 else {
1774 // We eagerly lower to implicit_def (instead of undef), as we
1775 // otherwise fail to select nodes such as: nxv1i1 = undef
1776 SDNode *Passthru =
1777 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
1778 Operands.push_back(SDValue(Passthru, 0));
1779 }
1780 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1781 Operands, /*IsLoad=*/true);
1782
1784 const RISCV::VLEPseudo *P =
1785 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
1786 static_cast<unsigned>(LMUL));
1787 MachineSDNode *Load =
1788 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1789
1790 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1791 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1792
1793 ReplaceNode(Node, Load);
1794 return;
1795 }
1796 case Intrinsic::riscv_vleff:
1797 case Intrinsic::riscv_vleff_mask: {
1798 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
1799
1800 MVT VT = Node->getSimpleValueType(0);
1801 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1802
1803 unsigned CurOp = 2;
1805 Operands.push_back(Node->getOperand(CurOp++));
1806 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1807 /*IsStridedOrIndexed*/ false, Operands,
1808 /*IsLoad=*/true);
1809
1811 const RISCV::VLEPseudo *P =
1812 RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
1813 Log2SEW, static_cast<unsigned>(LMUL));
1815 P->Pseudo, DL, Node->getVTList(), Operands);
1816 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1817 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
1818
1819 ReplaceNode(Node, Load);
1820 return;
1821 }
1822 }
1823 break;
1824 }
1825 case ISD::INTRINSIC_VOID: {
1826 unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
1827 switch (IntNo) {
1828 case Intrinsic::riscv_vsseg2:
1829 case Intrinsic::riscv_vsseg3:
1830 case Intrinsic::riscv_vsseg4:
1831 case Intrinsic::riscv_vsseg5:
1832 case Intrinsic::riscv_vsseg6:
1833 case Intrinsic::riscv_vsseg7:
1834 case Intrinsic::riscv_vsseg8: {
1835 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ false);
1836 return;
1837 }
1838 case Intrinsic::riscv_vsseg2_mask:
1839 case Intrinsic::riscv_vsseg3_mask:
1840 case Intrinsic::riscv_vsseg4_mask:
1841 case Intrinsic::riscv_vsseg5_mask:
1842 case Intrinsic::riscv_vsseg6_mask:
1843 case Intrinsic::riscv_vsseg7_mask:
1844 case Intrinsic::riscv_vsseg8_mask: {
1845 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ false);
1846 return;
1847 }
1848 case Intrinsic::riscv_vssseg2:
1849 case Intrinsic::riscv_vssseg3:
1850 case Intrinsic::riscv_vssseg4:
1851 case Intrinsic::riscv_vssseg5:
1852 case Intrinsic::riscv_vssseg6:
1853 case Intrinsic::riscv_vssseg7:
1854 case Intrinsic::riscv_vssseg8: {
1855 selectVSSEG(Node, /*IsMasked*/ false, /*IsStrided*/ true);
1856 return;
1857 }
1858 case Intrinsic::riscv_vssseg2_mask:
1859 case Intrinsic::riscv_vssseg3_mask:
1860 case Intrinsic::riscv_vssseg4_mask:
1861 case Intrinsic::riscv_vssseg5_mask:
1862 case Intrinsic::riscv_vssseg6_mask:
1863 case Intrinsic::riscv_vssseg7_mask:
1864 case Intrinsic::riscv_vssseg8_mask: {
1865 selectVSSEG(Node, /*IsMasked*/ true, /*IsStrided*/ true);
1866 return;
1867 }
1868 case Intrinsic::riscv_vsoxseg2:
1869 case Intrinsic::riscv_vsoxseg3:
1870 case Intrinsic::riscv_vsoxseg4:
1871 case Intrinsic::riscv_vsoxseg5:
1872 case Intrinsic::riscv_vsoxseg6:
1873 case Intrinsic::riscv_vsoxseg7:
1874 case Intrinsic::riscv_vsoxseg8:
1875 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ true);
1876 return;
1877 case Intrinsic::riscv_vsuxseg2:
1878 case Intrinsic::riscv_vsuxseg3:
1879 case Intrinsic::riscv_vsuxseg4:
1880 case Intrinsic::riscv_vsuxseg5:
1881 case Intrinsic::riscv_vsuxseg6:
1882 case Intrinsic::riscv_vsuxseg7:
1883 case Intrinsic::riscv_vsuxseg8:
1884 selectVSXSEG(Node, /*IsMasked*/ false, /*IsOrdered*/ false);
1885 return;
1886 case Intrinsic::riscv_vsoxseg2_mask:
1887 case Intrinsic::riscv_vsoxseg3_mask:
1888 case Intrinsic::riscv_vsoxseg4_mask:
1889 case Intrinsic::riscv_vsoxseg5_mask:
1890 case Intrinsic::riscv_vsoxseg6_mask:
1891 case Intrinsic::riscv_vsoxseg7_mask:
1892 case Intrinsic::riscv_vsoxseg8_mask:
1893 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ true);
1894 return;
1895 case Intrinsic::riscv_vsuxseg2_mask:
1896 case Intrinsic::riscv_vsuxseg3_mask:
1897 case Intrinsic::riscv_vsuxseg4_mask:
1898 case Intrinsic::riscv_vsuxseg5_mask:
1899 case Intrinsic::riscv_vsuxseg6_mask:
1900 case Intrinsic::riscv_vsuxseg7_mask:
1901 case Intrinsic::riscv_vsuxseg8_mask:
1902 selectVSXSEG(Node, /*IsMasked*/ true, /*IsOrdered*/ false);
1903 return;
1904 case Intrinsic::riscv_vsoxei:
1905 case Intrinsic::riscv_vsoxei_mask:
1906 case Intrinsic::riscv_vsuxei:
1907 case Intrinsic::riscv_vsuxei_mask: {
1908 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
1909 IntNo == Intrinsic::riscv_vsuxei_mask;
1910 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
1911 IntNo == Intrinsic::riscv_vsoxei_mask;
1912
1913 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1914 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1915
1916 unsigned CurOp = 2;
1918 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1919
1920 MVT IndexVT;
1921 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
1922 /*IsStridedOrIndexed*/ true, Operands,
1923 /*IsLoad=*/false, &IndexVT);
1924
1926 "Element count mismatch");
1927
1929 RISCVII::VLMUL IndexLMUL = RISCVTargetLowering::getLMUL(IndexVT);
1930 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1931 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1932 report_fatal_error("The V extension does not support EEW=64 for index "
1933 "values when XLEN=32");
1934 }
1935 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1936 IsMasked, IsOrdered, IndexLog2EEW,
1937 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL));
1938 MachineSDNode *Store =
1939 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1940
1941 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1942 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1943
1944 ReplaceNode(Node, Store);
1945 return;
1946 }
1947 case Intrinsic::riscv_vsm:
1948 case Intrinsic::riscv_vse:
1949 case Intrinsic::riscv_vse_mask:
1950 case Intrinsic::riscv_vsse:
1951 case Intrinsic::riscv_vsse_mask: {
1952 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
1953 IntNo == Intrinsic::riscv_vsse_mask;
1954 bool IsStrided =
1955 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
1956
1957 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
1958 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1959
1960 unsigned CurOp = 2;
1962 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
1963
1964 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
1965 Operands);
1966
1968 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1969 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1970 MachineSDNode *Store =
1971 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
1972 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
1973 CurDAG->setNodeMemRefs(Store, {MemOp->getMemOperand()});
1974
1975 ReplaceNode(Node, Store);
1976 return;
1977 }
1978 }
1979 break;
1980 }
1981 case ISD::BITCAST: {
1982 MVT SrcVT = Node->getOperand(0).getSimpleValueType();
1983 // Just drop bitcasts between vectors if both are fixed or both are
1984 // scalable.
1985 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
1986 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
1987 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
1988 CurDAG->RemoveDeadNode(Node);
1989 return;
1990 }
1991 break;
1992 }
1993 case ISD::INSERT_SUBVECTOR: {
1994 SDValue V = Node->getOperand(0);
1995 SDValue SubV = Node->getOperand(1);
1996 SDLoc DL(SubV);
1997 auto Idx = Node->getConstantOperandVal(2);
1998 MVT SubVecVT = SubV.getSimpleValueType();
1999
2000 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
2001 MVT SubVecContainerVT = SubVecVT;
2002 // Establish the correct scalable-vector types for any fixed-length type.
2003 if (SubVecVT.isFixedLengthVector())
2004 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
2005 if (VT.isFixedLengthVector())
2006 VT = TLI.getContainerForFixedLengthVector(VT);
2007
2008 const auto *TRI = Subtarget->getRegisterInfo();
2009 unsigned SubRegIdx;
2010 std::tie(SubRegIdx, Idx) =
2012 VT, SubVecContainerVT, Idx, TRI);
2013
2014 // If the Idx hasn't been completely eliminated then this is a subvector
2015 // insert which doesn't naturally align to a vector register. These must
2016 // be handled using instructions to manipulate the vector registers.
2017 if (Idx != 0)
2018 break;
2019
2020 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecContainerVT);
2021 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
2022 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
2023 SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
2024 (void)IsSubVecPartReg; // Silence unused variable warning without asserts.
2025 assert((!IsSubVecPartReg || V.isUndef()) &&
2026 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
2027 "the subvector is smaller than a full-sized register");
2028
2029 // If we haven't set a SubRegIdx, then we must be going between
2030 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
2031 if (SubRegIdx == RISCV::NoSubRegister) {
2032 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VT);
2034 InRegClassID &&
2035 "Unexpected subvector extraction");
2036 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
2037 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
2038 DL, VT, SubV, RC);
2039 ReplaceNode(Node, NewNode);
2040 return;
2041 }
2042
2043 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
2044 ReplaceNode(Node, Insert.getNode());
2045 return;
2046 }
2048 SDValue V = Node->getOperand(0);
2049 auto Idx = Node->getConstantOperandVal(1);
2050 MVT InVT = V.getSimpleValueType();
2051 SDLoc DL(V);
2052
2053 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
2054 MVT SubVecContainerVT = VT;
2055 // Establish the correct scalable-vector types for any fixed-length type.
2056 if (VT.isFixedLengthVector())
2057 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
2058 if (InVT.isFixedLengthVector())
2059 InVT = TLI.getContainerForFixedLengthVector(InVT);
2060
2061 const auto *TRI = Subtarget->getRegisterInfo();
2062 unsigned SubRegIdx;
2063 std::tie(SubRegIdx, Idx) =
2065 InVT, SubVecContainerVT, Idx, TRI);
2066
2067 // If the Idx hasn't been completely eliminated then this is a subvector
2068 // extract which doesn't naturally align to a vector register. These must
2069 // be handled using instructions to manipulate the vector registers.
2070 if (Idx != 0)
2071 break;
2072
2073 // If we haven't set a SubRegIdx, then we must be going between
2074 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
2075 if (SubRegIdx == RISCV::NoSubRegister) {
2076 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
2078 InRegClassID &&
2079 "Unexpected subvector extraction");
2080 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
2081 SDNode *NewNode =
2082 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
2083 ReplaceNode(Node, NewNode);
2084 return;
2085 }
2086
2087 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
2088 ReplaceNode(Node, Extract.getNode());
2089 return;
2090 }
2094 case RISCVISD::VFMV_V_F_VL: {
2095 // Try to match splat of a scalar load to a strided load with stride of x0.
2096 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
2097 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
2098 if (!Node->getOperand(0).isUndef())
2099 break;
2100 SDValue Src = Node->getOperand(1);
2101 auto *Ld = dyn_cast<LoadSDNode>(Src);
2102 // Can't fold load update node because the second
2103 // output is used so that load update node can't be removed.
2104 if (!Ld || Ld->isIndexed())
2105 break;
2106 EVT MemVT = Ld->getMemoryVT();
2107 // The memory VT should be the same size as the element type.
2108 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
2109 break;
2110 if (!IsProfitableToFold(Src, Node, Node) ||
2111 !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
2112 break;
2113
2114 SDValue VL;
2115 if (IsScalarMove) {
2116 // We could deal with more VL if we update the VSETVLI insert pass to
2117 // avoid introducing more VSETVLI.
2118 if (!isOneConstant(Node->getOperand(2)))
2119 break;
2120 selectVLOp(Node->getOperand(2), VL);
2121 } else
2122 selectVLOp(Node->getOperand(2), VL);
2123
2124 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2126
2127 // If VL=1, then we don't need to do a strided load and can just do a
2128 // regular load.
2129 bool IsStrided = !isOneConstant(VL);
2130
2131 // Only do a strided load if we have optimized zero-stride vector load.
2132 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
2133 break;
2134
2136 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT), 0),
2137 Ld->getBasePtr()};
2138 if (IsStrided)
2139 Operands.push_back(CurDAG->getRegister(RISCV::X0, XLenVT));
2141 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
2142 Operands.append({VL, SEW, PolicyOp, Ld->getChain()});
2143
2145 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
2146 /*IsMasked*/ false, IsStrided, /*FF*/ false,
2147 Log2SEW, static_cast<unsigned>(LMUL));
2148 MachineSDNode *Load =
2149 CurDAG->getMachineNode(P->Pseudo, DL, {VT, MVT::Other}, Operands);
2150 // Update the chain.
2151 ReplaceUses(Src.getValue(1), SDValue(Load, 1));
2152 // Record the mem-refs
2153 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
2154 // Replace the splat with the vlse.
2155 ReplaceNode(Node, Load);
2156 return;
2157 }
2158 case ISD::PREFETCH:
2159 unsigned Locality = Node->getConstantOperandVal(3);
2160 if (Locality > 2)
2161 break;
2162
2163 if (auto *LoadStoreMem = dyn_cast<MemSDNode>(Node)) {
2164 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
2166
2167 int NontemporalLevel = 0;
2168 switch (Locality) {
2169 case 0:
2170 NontemporalLevel = 3; // NTL.ALL
2171 break;
2172 case 1:
2173 NontemporalLevel = 1; // NTL.PALL
2174 break;
2175 case 2:
2176 NontemporalLevel = 0; // NTL.P1
2177 break;
2178 default:
2179 llvm_unreachable("unexpected locality value.");
2180 }
2181
2182 if (NontemporalLevel & 0b1)
2184 if (NontemporalLevel & 0b10)
2186 }
2187 break;
2188 }
2189
2190 // Select the default instruction.
2191 SelectCode(Node);
2192}
2193
2195 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
2196 std::vector<SDValue> &OutOps) {
2197 // Always produce a register and immediate operand, as expected by
2198 // RISCVAsmPrinter::PrintAsmMemoryOperand.
2199 switch (ConstraintID) {
2202 SDValue Op0, Op1;
2203 bool Found = SelectAddrRegImm(Op, Op0, Op1);
2204 assert(Found && "SelectAddrRegImm should always succeed");
2205 (void)Found;
2206 OutOps.push_back(Op0);
2207 OutOps.push_back(Op1);
2208 return false;
2209 }
2211 OutOps.push_back(Op);
2212 OutOps.push_back(
2213 CurDAG->getTargetConstant(0, SDLoc(Op), Subtarget->getXLenVT()));
2214 return false;
2215 default:
2216 report_fatal_error("Unexpected asm memory constraint " +
2217 InlineAsm::getMemConstraintName(ConstraintID));
2218 }
2219
2220 return true;
2221}
2222
2224 SDValue &Offset) {
2225 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
2226 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
2227 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT());
2228 return true;
2229 }
2230
2231 return false;
2232}
2233
2234// Select a frame index and an optional immediate offset from an ADD or OR.
2236 SDValue &Offset) {
2238 return true;
2239
2241 return false;
2242
2243 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr.getOperand(0))) {
2244 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2245 if (isInt<12>(CVal)) {
2246 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(),
2247 Subtarget->getXLenVT());
2249 Subtarget->getXLenVT());
2250 return true;
2251 }
2252 }
2253
2254 return false;
2255}
2256
2257// Fold constant addresses.
2258static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
2259 const MVT VT, const RISCVSubtarget *Subtarget,
2261 bool IsPrefetch = false) {
2262 if (!isa<ConstantSDNode>(Addr))
2263 return false;
2264
2265 int64_t CVal = cast<ConstantSDNode>(Addr)->getSExtValue();
2266
2267 // If the constant is a simm12, we can fold the whole constant and use X0 as
2268 // the base. If the constant can be materialized with LUI+simm12, use LUI as
2269 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
2270 int64_t Lo12 = SignExtend64<12>(CVal);
2271 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
2272 if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
2273 if (IsPrefetch && (Lo12 & 0b11111) != 0)
2274 return false;
2275
2276 if (Hi) {
2277 int64_t Hi20 = (Hi >> 12) & 0xfffff;
2278 Base = SDValue(
2279 CurDAG->getMachineNode(RISCV::LUI, DL, VT,
2280 CurDAG->getTargetConstant(Hi20, DL, VT)),
2281 0);
2282 } else {
2283 Base = CurDAG->getRegister(RISCV::X0, VT);
2284 }
2285 Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
2286 return true;
2287 }
2288
2289 // Ask how constant materialization would handle this constant.
2290 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(CVal, *Subtarget);
2291
2292 // If the last instruction would be an ADDI, we can fold its immediate and
2293 // emit the rest of the sequence as the base.
2294 if (Seq.back().getOpcode() != RISCV::ADDI)
2295 return false;
2296 Lo12 = Seq.back().getImm();
2297 if (IsPrefetch && (Lo12 & 0b11111) != 0)
2298 return false;
2299
2300 // Drop the last instruction.
2301 Seq.pop_back();
2302 assert(!Seq.empty() && "Expected more instructions in sequence");
2303
2304 Base = selectImmSeq(CurDAG, DL, VT, Seq);
2305 Offset = CurDAG->getTargetConstant(Lo12, DL, VT);
2306 return true;
2307}
2308
2309// Is this ADD instruction only used as the base pointer of scalar loads and
2310// stores?
2312 for (auto *Use : Add->uses()) {
2313 if (Use->getOpcode() != ISD::LOAD && Use->getOpcode() != ISD::STORE &&
2314 Use->getOpcode() != ISD::ATOMIC_LOAD &&
2315 Use->getOpcode() != ISD::ATOMIC_STORE)
2316 return false;
2317 EVT VT = cast<MemSDNode>(Use)->getMemoryVT();
2318 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
2319 VT != MVT::f64)
2320 return false;
2321 // Don't allow stores of the value. It must be used as the address.
2322 if (Use->getOpcode() == ISD::STORE &&
2323 cast<StoreSDNode>(Use)->getValue() == Add)
2324 return false;
2325 if (Use->getOpcode() == ISD::ATOMIC_STORE &&
2326 cast<AtomicSDNode>(Use)->getVal() == Add)
2327 return false;
2328 }
2329
2330 return true;
2331}
2332
2334 unsigned MaxShiftAmount,
2336 SDValue &Scale) {
2337 EVT VT = Addr.getSimpleValueType();
2338 auto UnwrapShl = [this, VT, MaxShiftAmount](SDValue N, SDValue &Index,
2339 SDValue &Shift) {
2340 uint64_t ShiftAmt = 0;
2341 Index = N;
2342
2343 if (N.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N.getOperand(1))) {
2344 // Only match shifts by a value in range [0, MaxShiftAmount].
2345 if (N.getConstantOperandVal(1) <= MaxShiftAmount) {
2346 Index = N.getOperand(0);
2347 ShiftAmt = N.getConstantOperandVal(1);
2348 }
2349 }
2350
2351 Shift = CurDAG->getTargetConstant(ShiftAmt, SDLoc(N), VT);
2352 return ShiftAmt != 0;
2353 };
2354
2355 if (Addr.getOpcode() == ISD::ADD) {
2356 if (auto *C1 = dyn_cast<ConstantSDNode>(Addr.getOperand(1))) {
2357 SDValue AddrB = Addr.getOperand(0);
2358 if (AddrB.getOpcode() == ISD::ADD &&
2359 UnwrapShl(AddrB.getOperand(0), Index, Scale) &&
2360 !isa<ConstantSDNode>(AddrB.getOperand(1)) &&
2361 isInt<12>(C1->getSExtValue())) {
2362 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
2363 SDValue C1Val =
2364 CurDAG->getTargetConstant(C1->getZExtValue(), SDLoc(Addr), VT);
2365 Base = SDValue(CurDAG->getMachineNode(RISCV::ADDI, SDLoc(Addr), VT,
2366 AddrB.getOperand(1), C1Val),
2367 0);
2368 return true;
2369 }
2370 } else if (UnwrapShl(Addr.getOperand(0), Index, Scale)) {
2371 Base = Addr.getOperand(1);
2372 return true;
2373 } else {
2374 UnwrapShl(Addr.getOperand(1), Index, Scale);
2375 Base = Addr.getOperand(0);
2376 return true;
2377 }
2378 } else if (UnwrapShl(Addr, Index, Scale)) {
2379 EVT VT = Addr.getValueType();
2380 Base = CurDAG->getRegister(RISCV::X0, VT);
2381 return true;
2382 }
2383
2384 return false;
2385}
2386
2388 SDValue &Offset, bool IsINX) {
2390 return true;
2391
2392 SDLoc DL(Addr);
2393 MVT VT = Addr.getSimpleValueType();
2394
2395 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
2396 Base = Addr.getOperand(0);
2397 Offset = Addr.getOperand(1);
2398 return true;
2399 }
2400
2401 int64_t RV32ZdinxRange = IsINX ? 4 : 0;
2403 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2404 if (isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) {
2405 Base = Addr.getOperand(0);
2406 if (Base.getOpcode() == RISCVISD::ADD_LO) {
2407 SDValue LoOperand = Base.getOperand(1);
2408 if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) {
2409 // If the Lo in (ADD_LO hi, lo) is a global variable's address
2410 // (its low part, really), then we can rely on the alignment of that
2411 // variable to provide a margin of safety before low part can overflow
2412 // the 12 bits of the load/store offset. Check if CVal falls within
2413 // that margin; if so (low part + CVal) can't overflow.
2414 const DataLayout &DL = CurDAG->getDataLayout();
2415 Align Alignment = commonAlignment(
2416 GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
2417 if (CVal == 0 || Alignment > CVal) {
2418 int64_t CombinedOffset = CVal + GA->getOffset();
2419 Base = Base.getOperand(0);
2421 GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(),
2422 CombinedOffset, GA->getTargetFlags());
2423 return true;
2424 }
2425 }
2426 }
2427
2428 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
2429 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
2430 Offset = CurDAG->getTargetConstant(CVal, DL, VT);
2431 return true;
2432 }
2433 }
2434
2435 // Handle ADD with large immediates.
2436 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
2437 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2438 assert(!(isInt<12>(CVal) && isInt<12>(CVal + RV32ZdinxRange)) &&
2439 "simm12 not already handled?");
2440
2441 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
2442 // an ADDI for part of the offset and fold the rest into the load/store.
2443 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
2444 if (isInt<12>(CVal / 2) && isInt<12>(CVal - CVal / 2)) {
2445 int64_t Adj = CVal < 0 ? -2048 : 2047;
2446 Base = SDValue(
2447 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
2448 CurDAG->getTargetConstant(Adj, DL, VT)),
2449 0);
2450 Offset = CurDAG->getTargetConstant(CVal - Adj, DL, VT);
2451 return true;
2452 }
2453
2454 // For larger immediates, we might be able to save one instruction from
2455 // constant materialization by folding the Lo12 bits of the immediate into
2456 // the address. We should only do this if the ADD is only used by loads and
2457 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
2458 // separately with the full materialized immediate creating extra
2459 // instructions.
2460 if (isWorthFoldingAdd(Addr) &&
2461 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
2462 Offset)) {
2463 // Insert an ADD instruction with the materialized Hi52 bits.
2464 Base = SDValue(
2465 CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
2466 0);
2467 return true;
2468 }
2469 }
2470
2471 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset))
2472 return true;
2473
2474 Base = Addr;
2475 Offset = CurDAG->getTargetConstant(0, DL, VT);
2476 return true;
2477}
2478
2479/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
2480/// Offset shoule be all zeros.
2482 SDValue &Offset) {
2484 return true;
2485
2486 SDLoc DL(Addr);
2487 MVT VT = Addr.getSimpleValueType();
2488
2490 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2491 if (isInt<12>(CVal)) {
2492 Base = Addr.getOperand(0);
2493
2494 // Early-out if not a valid offset.
2495 if ((CVal & 0b11111) != 0) {
2496 Base = Addr;
2497 Offset = CurDAG->getTargetConstant(0, DL, VT);
2498 return true;
2499 }
2500
2501 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
2502 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
2503 Offset = CurDAG->getTargetConstant(CVal, DL, VT);
2504 return true;
2505 }
2506 }
2507
2508 // Handle ADD with large immediates.
2509 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
2510 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
2511 assert(!(isInt<12>(CVal) && isInt<12>(CVal)) &&
2512 "simm12 not already handled?");
2513
2514 // Handle immediates in the range [-4096,-2049] or [2017, 4065]. We can save
2515 // one instruction by folding adjustment (-2048 or 2016) into the address.
2516 if ((-2049 >= CVal && CVal >= -4096) || (4065 >= CVal && CVal >= 2017)) {
2517 int64_t Adj = CVal < 0 ? -2048 : 2016;
2518 int64_t AdjustedOffset = CVal - Adj;
2520 RISCV::ADDI, DL, VT, Addr.getOperand(0),
2521 CurDAG->getTargetConstant(AdjustedOffset, DL, VT)),
2522 0);
2523 Offset = CurDAG->getTargetConstant(Adj, DL, VT);
2524 return true;
2525 }
2526
2527 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
2528 Offset, true)) {
2529 // Insert an ADD instruction with the materialized Hi52 bits.
2530 Base = SDValue(
2531 CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
2532 0);
2533 return true;
2534 }
2535 }
2536
2537 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset, true))
2538 return true;
2539
2540 Base = Addr;
2541 Offset = CurDAG->getTargetConstant(0, DL, VT);
2542 return true;
2543}
2544
2546 SDValue &ShAmt) {
2547 ShAmt = N;
2548
2549 // Peek through zext.
2550 if (ShAmt->getOpcode() == ISD::ZERO_EXTEND)
2551 ShAmt = ShAmt.getOperand(0);
2552
2553 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
2554 // amount. If there is an AND on the shift amount, we can bypass it if it
2555 // doesn't affect any of those bits.
2556 if (ShAmt.getOpcode() == ISD::AND &&
2557 isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2558 const APInt &AndMask = ShAmt.getConstantOperandAPInt(1);
2559
2560 // Since the max shift amount is a power of 2 we can subtract 1 to make a
2561 // mask that covers the bits needed to represent all shift amounts.
2562 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
2563 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
2564
2565 if (ShMask.isSubsetOf(AndMask)) {
2566 ShAmt = ShAmt.getOperand(0);
2567 } else {
2568 // SimplifyDemandedBits may have optimized the mask so try restoring any
2569 // bits that are known zero.
2570 KnownBits Known = CurDAG->computeKnownBits(ShAmt.getOperand(0));
2571 if (!ShMask.isSubsetOf(AndMask | Known.Zero))
2572 return true;
2573 ShAmt = ShAmt.getOperand(0);
2574 }
2575 }
2576
2577 if (ShAmt.getOpcode() == ISD::ADD &&
2578 isa<ConstantSDNode>(ShAmt.getOperand(1))) {
2579 uint64_t Imm = ShAmt.getConstantOperandVal(1);
2580 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
2581 // to avoid the ADD.
2582 if (Imm != 0 && Imm % ShiftWidth == 0) {
2583 ShAmt = ShAmt.getOperand(0);
2584 return true;
2585 }
2586 } else if (ShAmt.getOpcode() == ISD::SUB &&
2587 isa<ConstantSDNode>(ShAmt.getOperand(0))) {
2588 uint64_t Imm = ShAmt.getConstantOperandVal(0);
2589 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
2590 // generate a NEG instead of a SUB of a constant.
2591 if (Imm != 0 && Imm % ShiftWidth == 0) {
2592 SDLoc DL(ShAmt);
2593 EVT VT = ShAmt.getValueType();
2594 SDValue Zero = CurDAG->getRegister(RISCV::X0, VT);
2595 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
2596 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero,
2597 ShAmt.getOperand(1));
2598 ShAmt = SDValue(Neg, 0);
2599 return true;
2600 }
2601 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
2602 // to generate a NOT instead of a SUB of a constant.
2603 if (Imm % ShiftWidth == ShiftWidth - 1) {
2604 SDLoc DL(ShAmt);
2605 EVT VT = ShAmt.getValueType();
2606 MachineSDNode *Not =
2607 CurDAG->getMachineNode(RISCV::XORI, DL, VT, ShAmt.getOperand(1),
2608 CurDAG->getTargetConstant(-1, DL, VT));
2609 ShAmt = SDValue(Not, 0);
2610 return true;
2611 }
2612 }
2613
2614 return true;
2615}
2616
2617/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
2618/// check for equality with 0. This function emits instructions that convert the
2619/// seteq/setne into something that can be compared with 0.
2620/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
2621/// ISD::SETNE).
2623 SDValue &Val) {
2624 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
2625 "Unexpected condition code!");
2626
2627 // We're looking for a setcc.
2628 if (N->getOpcode() != ISD::SETCC)
2629 return false;
2630
2631 // Must be an equality comparison.
2632 ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
2633 if (CCVal != ExpectedCCVal)
2634 return false;
2635
2636 SDValue LHS = N->getOperand(0);
2637 SDValue RHS = N->getOperand(1);
2638
2639 if (!LHS.getValueType().isScalarInteger())
2640 return false;
2641
2642 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
2643 if (isNullConstant(RHS)) {
2644 Val = LHS;
2645 return true;
2646 }
2647
2648 SDLoc DL(N);
2649
2650 if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
2651 int64_t CVal = C->getSExtValue();
2652 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
2653 // non-zero otherwise.
2654 if (CVal == -2048) {
2655 Val =
2657 RISCV::XORI, DL, N->getValueType(0), LHS,
2658 CurDAG->getTargetConstant(CVal, DL, N->getValueType(0))),
2659 0);
2660 return true;
2661 }
2662 // If the RHS is [-2047,2048], we can use addi with -RHS to produce 0 if the
2663 // LHS is equal to the RHS and non-zero otherwise.
2664 if (isInt<12>(CVal) || CVal == 2048) {
2665 Val =
2667 RISCV::ADDI, DL, N->getValueType(0), LHS,
2668 CurDAG->getTargetConstant(-CVal, DL, N->getValueType(0))),
2669 0);
2670 return true;
2671 }
2672 }
2673
2674 // If nothing else we can XOR the LHS and RHS to produce zero if they are
2675 // equal and a non-zero value if they aren't.
2676 Val = SDValue(
2677 CurDAG->getMachineNode(RISCV::XOR, DL, N->getValueType(0), LHS, RHS), 0);
2678 return true;
2679}
2680
2682 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
2683 cast<VTSDNode>(N.getOperand(1))->getVT().getSizeInBits() == Bits) {
2684 Val = N.getOperand(0);
2685 return true;
2686 }
2687
2688 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
2689 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(N.getOperand(1)))
2690 return N;
2691
2692 SDValue N0 = N.getOperand(0);
2693 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2694 N.getConstantOperandVal(1) == ShiftAmt &&
2695 N0.getConstantOperandVal(1) == ShiftAmt)
2696 return N0.getOperand(0);
2697
2698 return N;
2699 };
2700
2701 MVT VT = N.getSimpleValueType();
2702 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - Bits)) {
2703 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
2704 return true;
2705 }
2706
2707 return false;
2708}
2709
2711 if (N.getOpcode() == ISD::AND) {
2712 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
2713 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
2714 Val = N.getOperand(0);
2715 return true;
2716 }
2717 }
2718 MVT VT = N.getSimpleValueType();
2719 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), Bits);
2720 if (CurDAG->MaskedValueIsZero(N, Mask)) {
2721 Val = N;
2722 return true;
2723 }
2724
2725 return false;
2726}
2727
2728/// Look for various patterns that can be done with a SHL that can be folded
2729/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
2730/// SHXADD we are trying to match.
2732 SDValue &Val) {
2733 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
2734 SDValue N0 = N.getOperand(0);
2735
2736 bool LeftShift = N0.getOpcode() == ISD::SHL;
2737 if ((LeftShift || N0.getOpcode() == ISD::SRL) &&
2738 isa<ConstantSDNode>(N0.getOperand(1))) {
2739 uint64_t Mask = N.getConstantOperandVal(1);
2740 unsigned C2 = N0.getConstantOperandVal(1);
2741
2742 unsigned XLen = Subtarget->getXLen();
2743 if (LeftShift)
2744 Mask &= maskTrailingZeros<uint64_t>(C2);
2745 else
2746 Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
2747
2748 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
2749 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
2750 // followed by a SHXADD with c3 for the X amount.
2751 if (isShiftedMask_64(Mask)) {
2752 unsigned Leading = XLen - llvm::bit_width(Mask);
2753 unsigned Trailing = llvm::countr_zero(Mask);
2754 if (LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
2755 SDLoc DL(N);
2756 EVT VT = N.getValueType();
2758 RISCV::SRLI, DL, VT, N0.getOperand(0),
2759 CurDAG->getTargetConstant(Trailing - C2, DL, VT)),
2760 0);
2761 return true;
2762 }
2763 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
2764 // leading zeros and c3 trailing zeros. We can use an SRLI by C3
2765 // followed by a SHXADD using c3 for the X amount.
2766 if (!LeftShift && Leading == C2 && Trailing == ShAmt) {
2767 SDLoc DL(N);
2768 EVT VT = N.getValueType();
2769 Val = SDValue(
2771 RISCV::SRLI, DL, VT, N0.getOperand(0),
2772 CurDAG->getTargetConstant(Leading + Trailing, DL, VT)),
2773 0);
2774 return true;
2775 }
2776 }
2777 }
2778 }
2779
2780 bool LeftShift = N.getOpcode() == ISD::SHL;
2781 if ((LeftShift || N.getOpcode() == ISD::SRL) &&
2782 isa<ConstantSDNode>(N.getOperand(1))) {
2783 SDValue N0 = N.getOperand(0);
2784 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
2785 isa<ConstantSDNode>(N0.getOperand(1))) {
2786 uint64_t Mask = N0.getConstantOperandVal(1);
2787 if (isShiftedMask_64(Mask)) {
2788 unsigned C1 = N.getConstantOperandVal(1);
2789 unsigned XLen = Subtarget->getXLen();
2790 unsigned Leading = XLen - llvm::bit_width(Mask);
2791 unsigned Trailing = llvm::countr_zero(Mask);
2792 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
2793 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
2794 if (LeftShift && Leading == 32 && Trailing > 0 &&
2795 (Trailing + C1) == ShAmt) {
2796 SDLoc DL(N);
2797 EVT VT = N.getValueType();
2799 RISCV::SRLIW, DL, VT, N0.getOperand(0),
2800 CurDAG->getTargetConstant(Trailing, DL, VT)),
2801 0);
2802 return true;
2803 }
2804 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
2805 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
2806 if (!LeftShift && Leading == 32 && Trailing > C1 &&
2807 (Trailing - C1) == ShAmt) {
2808 SDLoc DL(N);
2809 EVT VT = N.getValueType();
2811 RISCV::SRLIW, DL, VT, N0.getOperand(0),
2812 CurDAG->getTargetConstant(Trailing, DL, VT)),
2813 0);
2814 return true;
2815 }
2816 }
2817 }
2818 }
2819
2820 return false;
2821}
2822
2823/// Look for various patterns that can be done with a SHL that can be folded
2824/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
2825/// SHXADD_UW we are trying to match.
2827 SDValue &Val) {
2828 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
2829 N.hasOneUse()) {
2830 SDValue N0 = N.getOperand(0);
2831 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
2832 N0.hasOneUse()) {
2833 uint64_t Mask = N.getConstantOperandVal(1);
2834 unsigned C2 = N0.getConstantOperandVal(1);
2835
2836 Mask &= maskTrailingZeros<uint64_t>(C2);
2837
2838 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
2839 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
2840 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
2841 if (isShiftedMask_64(Mask)) {
2842 unsigned Leading = llvm::countl_zero(Mask);
2843 unsigned Trailing = llvm::countr_zero(Mask);
2844 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
2845 SDLoc DL(N);
2846 EVT VT = N.getValueType();
2848 RISCV::SLLI, DL, VT, N0.getOperand(0),
2849 CurDAG->getTargetConstant(C2 - ShAmt, DL, VT)),
2850 0);
2851 return true;
2852 }
2853 }
2854 }
2855 }
2856
2857 return false;
2858}
2859
2860static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
2861 unsigned Bits,
2862 const TargetInstrInfo *TII) {
2863 unsigned MCOpcode = RISCV::getRVVMCOpcode(User->getMachineOpcode());
2864
2865 if (!MCOpcode)
2866 return false;
2867
2868 const MCInstrDesc &MCID = TII->get(User->getMachineOpcode());
2869 const uint64_t TSFlags = MCID.TSFlags;
2871 return false;
2873
2874 bool HasGlueOp = User->getGluedNode() != nullptr;
2875 unsigned ChainOpIdx = User->getNumOperands() - HasGlueOp - 1;
2876 bool HasChainOp = User->getOperand(ChainOpIdx).getValueType() == MVT::Other;
2877 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
2878 unsigned VLIdx =
2879 User->getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2;
2880 const unsigned Log2SEW = User->getConstantOperandVal(VLIdx + 1);
2881
2882 if (UserOpNo == VLIdx)
2883 return false;
2884
2885 auto NumDemandedBits =
2887 return NumDemandedBits && Bits >= *NumDemandedBits;
2888}
2889
2890// Return true if all users of this SDNode* only consume the lower \p Bits.
2891// This can be used to form W instructions for add/sub/mul/shl even when the
2892// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
2893// SimplifyDemandedBits has made it so some users see a sext_inreg and some
2894// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
2895// the add/sub/mul/shl to become non-W instructions. By checking the users we
2896// may be able to use a W instruction and CSE with the other instruction if
2897// this has happened. We could try to detect that the CSE opportunity exists
2898// before doing this, but that would be more complicated.
2900 const unsigned Depth) const {
2901 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
2902 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
2903 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
2904 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
2905 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
2906 isa<ConstantSDNode>(Node) || Depth != 0) &&
2907 "Unexpected opcode");
2908
2910 return false;
2911
2912 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
2913 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
2914 if (Depth == 0 && !Node->getValueType(0).isScalarInteger())
2915 return false;
2916
2917 for (auto UI = Node->use_begin(), UE = Node->use_end(); UI != UE; ++UI) {
2918 SDNode *User = *UI;
2919 // Users of this node should have already been instruction selected
2920 if (!User->isMachineOpcode())
2921 return false;
2922
2923 // TODO: Add more opcodes?
2924 switch (User->getMachineOpcode()) {
2925 default:
2926 if (vectorPseudoHasAllNBitUsers(User, UI.getOperandNo(), Bits, TII))
2927 break;
2928 return false;
2929 case RISCV::ADDW:
2930 case RISCV::ADDIW:
2931 case RISCV::SUBW:
2932 case RISCV::MULW:
2933 case RISCV::SLLW:
2934 case RISCV::SLLIW:
2935 case RISCV::SRAW:
2936 case RISCV::SRAIW:
2937 case RISCV::SRLW:
2938 case RISCV::SRLIW:
2939 case RISCV::DIVW:
2940 case RISCV::DIVUW:
2941 case RISCV::REMW:
2942 case RISCV::REMUW:
2943 case RISCV::ROLW:
2944 case RISCV::RORW:
2945 case RISCV::RORIW:
2946 case RISCV::CLZW:
2947 case RISCV::CTZW:
2948 case RISCV::CPOPW:
2949 case RISCV::SLLI_UW:
2950 case RISCV::FMV_W_X:
2951 case RISCV::FCVT_H_W:
2952 case RISCV::FCVT_H_WU:
2953 case RISCV::FCVT_S_W:
2954 case RISCV::FCVT_S_WU:
2955 case RISCV::FCVT_D_W:
2956 case RISCV::FCVT_D_WU:
2957 case RISCV::TH_REVW:
2958 case RISCV::TH_SRRIW:
2959 if (Bits < 32)
2960 return false;
2961 break;
2962 case RISCV::SLL:
2963 case RISCV::SRA:
2964 case RISCV::SRL:
2965 case RISCV::ROL:
2966 case RISCV::ROR:
2967 case RISCV::BSET:
2968 case RISCV::BCLR:
2969 case RISCV::BINV:
2970 // Shift amount operands only use log2(Xlen) bits.
2971 if (UI.getOperandNo() != 1 || Bits < Log2_32(Subtarget->getXLen()))
2972 return false;
2973 break;
2974 case RISCV::SLLI:
2975 // SLLI only uses the lower (XLen - ShAmt) bits.
2976 if (Bits < Subtarget->getXLen() - User->getConstantOperandVal(1))
2977 return false;
2978 break;
2979 case RISCV::ANDI:
2980 if (Bits >= (unsigned)llvm::bit_width(User->getConstantOperandVal(1)))
2981 break;
2982 goto RecCheck;
2983 case RISCV::ORI: {
2984 uint64_t Imm = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
2985 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(~Imm))
2986 break;
2987 [[fallthrough]];
2988 }
2989 case RISCV::AND:
2990 case RISCV::OR:
2991 case RISCV::XOR:
2992 case RISCV::XORI:
2993 case RISCV::ANDN:
2994 case RISCV::ORN:
2995 case RISCV::XNOR:
2996 case RISCV::SH1ADD:
2997 case RISCV::SH2ADD:
2998 case RISCV::SH3ADD:
2999 RecCheck:
3000 if (hasAllNBitUsers(User, Bits, Depth + 1))
3001 break;
3002 return false;
3003 case RISCV::SRLI: {
3004 unsigned ShAmt = User->getConstantOperandVal(1);
3005 // If we are shifting right by less than Bits, and users don't demand any
3006 // bits that were shifted into [Bits-1:0], then we can consider this as an
3007 // N-Bit user.
3008 if (Bits > ShAmt && hasAllNBitUsers(User, Bits - ShAmt, Depth + 1))
3009 break;
3010 return false;
3011 }
3012 case RISCV::SEXT_B:
3013 case RISCV::PACKH:
3014 if (Bits < 8)
3015 return false;
3016 break;
3017 case RISCV::SEXT_H:
3018 case RISCV::FMV_H_X:
3019 case RISCV::ZEXT_H_RV32:
3020 case RISCV::ZEXT_H_RV64:
3021 case RISCV::PACKW:
3022 if (Bits < 16)
3023 return false;
3024 break;
3025 case RISCV::PACK:
3026 if (Bits < (Subtarget->getXLen() / 2))
3027 return false;
3028 break;
3029 case RISCV::ADD_UW:
3030 case RISCV::SH1ADD_UW:
3031 case RISCV::SH2ADD_UW:
3032 case RISCV::SH3ADD_UW:
3033 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
3034 // 32 bits.
3035 if (UI.getOperandNo() != 0 || Bits < 32)
3036 return false;
3037 break;
3038 case RISCV::SB:
3039 if (UI.getOperandNo() != 0 || Bits < 8)
3040 return false;
3041 break;
3042 case RISCV::SH:
3043 if (UI.getOperandNo() != 0 || Bits < 16)
3044 return false;
3045 break;
3046 case RISCV::SW:
3047 if (UI.getOperandNo() != 0 || Bits < 32)
3048 return false;
3049 break;
3050 }
3051 }
3052
3053 return true;
3054}
3055
3056// Select a constant that can be represented as (sign_extend(imm5) << imm2).
3058 SDValue &Shl2) {
3059 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
3060 int64_t Offset = C->getSExtValue();
3061 int64_t Shift;
3062 for (Shift = 0; Shift < 4; Shift++)
3063 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
3064 break;
3065
3066 // Constant cannot be encoded.
3067 if (Shift == 4)
3068 return false;
3069
3070 EVT Ty = N->getValueType(0);
3071 Simm5 = CurDAG->getTargetConstant(Offset >> Shift, SDLoc(N), Ty);
3072 Shl2 = CurDAG->getTargetConstant(Shift, SDLoc(N), Ty);
3073 return true;
3074 }
3075
3076 return false;
3077}
3078
3079// Select VL as a 5 bit immediate or a value that will become a register. This
3080// allows us to choose betwen VSETIVLI or VSETVLI later.
3082 auto *C = dyn_cast<ConstantSDNode>(N);
3083 if (C && isUInt<5>(C->getZExtValue())) {
3084 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
3085 N->getValueType(0));
3086 } else if (C && C->isAllOnes()) {
3087 // Treat all ones as VLMax.
3089 N->getValueType(0));
3090 } else if (isa<RegisterSDNode>(N) &&
3091 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
3092 // All our VL operands use an operand that allows GPRNoX0 or an immediate
3093 // as the register class. Convert X0 to a special immediate to pass the
3094 // MachineVerifier. This is recognized specially by the vsetvli insertion
3095 // pass.
3097 N->getValueType(0));
3098 } else {
3099 VL = N;
3100 }
3101
3102 return true;
3103}
3104
3106 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
3107 if (!N.getOperand(0).isUndef())
3108 return SDValue();
3109 N = N.getOperand(1);
3110 }
3111 SDValue Splat = N;
3112 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
3113 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
3114 !Splat.getOperand(0).isUndef())
3115 return SDValue();
3116 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
3117 return Splat;
3118}
3119
3122 if (!Splat)
3123 return false;
3124
3125 SplatVal = Splat.getOperand(1);
3126 return true;
3127}
3128
3130 SelectionDAG &DAG,
3131 const RISCVSubtarget &Subtarget,
3132 std::function<bool(int64_t)> ValidateImm) {
3134 if (!Splat || !isa<ConstantSDNode>(Splat.getOperand(1)))
3135 return false;
3136
3137 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
3138 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
3139 "Unexpected splat operand type");
3140
3141 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
3142 // type is wider than the resulting vector element type: an implicit
3143 // truncation first takes place. Therefore, perform a manual
3144 // truncation/sign-extension in order to ignore any truncated bits and catch
3145 // any zero-extended immediate.
3146 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
3147 // sign-extending to (XLenVT -1).
3148 APInt SplatConst = Splat.getConstantOperandAPInt(1).sextOrTrunc(SplatEltSize);
3149
3150 int64_t SplatImm = SplatConst.getSExtValue();
3151
3152 if (!ValidateImm(SplatImm))
3153 return false;
3154
3155 SplatVal = DAG.getTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
3156 return true;
3157}
3158
3160 return selectVSplatImmHelper(N, SplatVal, *CurDAG, *Subtarget,
3161 [](int64_t Imm) { return isInt<5>(Imm); });
3162}
3163
3165 return selectVSplatImmHelper(
3166 N, SplatVal, *CurDAG, *Subtarget,
3167 [](int64_t Imm) { return (isInt<5>(Imm) && Imm != -16) || Imm == 16; });
3168}
3169
3171 SDValue &SplatVal) {
3172 return selectVSplatImmHelper(
3173 N, SplatVal, *CurDAG, *Subtarget, [](int64_t Imm) {
3174 return Imm != 0 && ((isInt<5>(Imm) && Imm != -16) || Imm == 16);
3175 });
3176}
3177
3179 SDValue &SplatVal) {
3180 return selectVSplatImmHelper(
3181 N, SplatVal, *CurDAG, *Subtarget,
3182 [Bits](int64_t Imm) { return isUIntN(Bits, Imm); });
3183}
3184
3186 // Truncates are custom lowered during legalization.
3187 auto IsTrunc = [this](SDValue N) {
3188 if (N->getOpcode() != RISCVISD::TRUNCATE_VECTOR_VL)
3189 return false;
3190 SDValue VL;
3191 selectVLOp(N->getOperand(2), VL);
3192 // Any vmset_vl is ok, since any bits past VL are undefined and we can
3193 // assume they are set.
3194 return N->getOperand(1).getOpcode() == RISCVISD::VMSET_VL &&
3195 isa<ConstantSDNode>(VL) &&
3196 cast<ConstantSDNode>(VL)->getSExtValue() == RISCV::VLMaxSentinel;
3197 };
3198
3199 // We can have multiple nested truncates, so unravel them all if needed.
3200 while (N->getOpcode() == ISD::SIGN_EXTEND ||
3201 N->getOpcode() == ISD::ZERO_EXTEND || IsTrunc(N)) {
3202 if (!N.hasOneUse() ||
3203 N.getValueType().getSizeInBits().getKnownMinValue() < 8)
3204 return false;
3205 N = N->getOperand(0);
3206 }
3207
3208 return selectVSplat(N, SplatVal);
3209}
3210
3212 ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N.getNode());
3213 if (!CFP)
3214 return false;
3215 const APFloat &APF = CFP->getValueAPF();
3216 // td can handle +0.0 already.
3217 if (APF.isPosZero())
3218 return false;
3219
3220 MVT VT = CFP->getSimpleValueType(0);
3221
3222 // Even if this FPImm requires an additional FNEG (i.e. the second element of
3223 // the returned pair is true) we still prefer FLI + FNEG over immediate
3224 // materialization as the latter might generate a longer instruction sequence.
3225 if (static_cast<const RISCVTargetLowering *>(TLI)
3226 ->getLegalZfaFPImm(APF, VT)
3227 .first >= 0)
3228 return false;
3229
3230 MVT XLenVT = Subtarget->getXLenVT();
3231 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
3232 assert(APF.isNegZero() && "Unexpected constant.");
3233 return false;
3234 }
3235 SDLoc DL(N);
3236 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
3237 *Subtarget);
3238 return true;
3239}
3240
3242 SDValue &Imm) {
3243 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
3244 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
3245
3246 if (!isInt<5>(ImmVal))
3247 return false;
3248
3249 Imm = CurDAG->getTargetConstant(ImmVal, SDLoc(N), Subtarget->getXLenVT());
3250 return true;
3251 }
3252
3253 return false;
3254}
3255
3256// Try to remove sext.w if the input is a W instruction or can be made into
3257// a W instruction cheaply.
3258bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
3259 // Look for the sext.w pattern, addiw rd, rs1, 0.
3260 if (N->getMachineOpcode() != RISCV::ADDIW ||
3261 !isNullConstant(N->getOperand(1)))
3262 return false;
3263
3264 SDValue N0 = N->getOperand(0);
3265 if (!N0.isMachineOpcode())
3266 return false;
3267
3268 switch (N0.getMachineOpcode()) {
3269 default:
3270 break;
3271 case RISCV::ADD:
3272 case RISCV::ADDI:
3273 case RISCV::SUB:
3274 case RISCV::MUL:
3275 case RISCV::SLLI: {
3276 // Convert sext.w+add/sub/mul to their W instructions. This will create
3277 // a new independent instruction. This improves latency.
3278 unsigned Opc;
3279 switch (N0.getMachineOpcode()) {
3280 default:
3281 llvm_unreachable("Unexpected opcode!");
3282 case RISCV::ADD: Opc = RISCV::ADDW; break;
3283 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
3284 case RISCV::SUB: Opc = RISCV::SUBW; break;
3285 case RISCV::MUL: Opc = RISCV::MULW; break;
3286 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
3287 }
3288
3289 SDValue N00 = N0.getOperand(0);
3290 SDValue N01 = N0.getOperand(1);
3291
3292 // Shift amount needs to be uimm5.
3293 if (N0.getMachineOpcode() == RISCV::SLLI &&
3294 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue()))
3295 break;
3296
3297 SDNode *Result =
3298 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
3299 N00, N01);
3300 ReplaceUses(N, Result);
3301 return true;
3302 }
3303 case RISCV::ADDW:
3304 case RISCV::ADDIW:
3305 case RISCV::SUBW:
3306 case RISCV::MULW:
3307 case RISCV::SLLIW:
3308 case RISCV::PACKW:
3309 case RISCV::TH_MULAW:
3310 case RISCV::TH_MULAH:
3311 case RISCV::TH_MULSW:
3312 case RISCV::TH_MULSH:
3313 if (N0.getValueType() == MVT::i32)
3314 break;
3315
3316 // Result is already sign extended just remove the sext.w.
3317 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
3318 ReplaceUses(N, N0.getNode());
3319 return true;
3320 }
3321
3322 return false;
3323}
3324
3325static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp) {
3326 // Check that we're using V0 as a mask register.
3327 if (!isa<RegisterSDNode>(MaskOp) ||
3328 cast<RegisterSDNode>(MaskOp)->getReg() != RISCV::V0)
3329 return false;
3330
3331 // The glued user defines V0.
3332 const auto *Glued = GlueOp.getNode();
3333
3334 if (!Glued || Glued->getOpcode() != ISD::CopyToReg)
3335 return false;
3336
3337 // Check that we're defining V0 as a mask register.
3338 if (!isa<RegisterSDNode>(Glued->getOperand(1)) ||
3339 cast<RegisterSDNode>(Glued->getOperand(1))->getReg() != RISCV::V0)
3340 return false;
3341
3342 // Check the instruction defining V0; it needs to be a VMSET pseudo.
3343 SDValue MaskSetter = Glued->getOperand(2);
3344
3345 // Sometimes the VMSET is wrapped in a COPY_TO_REGCLASS, e.g. if the mask came
3346 // from an extract_subvector or insert_subvector.
3347 if (MaskSetter->isMachineOpcode() &&
3348 MaskSetter->getMachineOpcode() == RISCV::COPY_TO_REGCLASS)
3349 MaskSetter = MaskSetter->getOperand(0);
3350
3351 const auto IsVMSet = [](unsigned Opc) {
3352 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
3353 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
3354 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
3355 Opc == RISCV::PseudoVMSET_M_B8;
3356 };
3357
3358 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
3359 // undefined behaviour if it's the wrong bitwidth, so we could choose to
3360 // assume that it's all-ones? Same applies to its VL.
3361 return MaskSetter->isMachineOpcode() &&
3362 IsVMSet(MaskSetter.getMachineOpcode());
3363}
3364
3365// Return true if we can make sure mask of N is all-ones mask.
3366static bool usesAllOnesMask(SDNode *N, unsigned MaskOpIdx) {
3367 return usesAllOnesMask(N->getOperand(MaskOpIdx),
3368 N->getOperand(N->getNumOperands() - 1));
3369}
3370
3371static bool isImplicitDef(SDValue V) {
3372 return V.isMachineOpcode() &&
3373 V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
3374}
3375
3376// Optimize masked RVV pseudo instructions with a known all-ones mask to their
3377// corresponding "unmasked" pseudo versions. The mask we're interested in will
3378// take the form of a V0 physical register operand, with a glued
3379// register-setting instruction.
3380bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
3382 RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
3383 if (!I)
3384 return false;
3385
3386 unsigned MaskOpIdx = I->MaskOpIdx;
3387 if (!usesAllOnesMask(N, MaskOpIdx))
3388 return false;
3389
3390 // There are two classes of pseudos in the table - compares and
3391 // everything else. See the comment on RISCVMaskedPseudo for details.
3392 const unsigned Opc = I->UnmaskedPseudo;
3393 const MCInstrDesc &MCID = TII->get(Opc);
3394 const bool UseTUPseudo = RISCVII::hasVecPolicyOp(MCID.TSFlags);
3395#ifndef NDEBUG
3396 const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode());
3399 "Masked and unmasked pseudos are inconsistent");
3400 const bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(MCID);
3401 assert(UseTUPseudo == HasTiedDest && "Unexpected pseudo structure");
3402#endif
3403
3405 // Skip the merge operand at index 0 if !UseTUPseudo.
3406 for (unsigned I = !UseTUPseudo, E = N->getNumOperands(); I != E; I++) {
3407 // Skip the mask, and the Glue.
3408 SDValue Op = N->getOperand(I);
3409 if (I == MaskOpIdx || Op.getValueType() == MVT::Glue)
3410 continue;
3411 Ops.push_back(Op);
3412 }
3413
3414 // Transitively apply any node glued to our new node.
3415 const auto *Glued = N->getGluedNode();
3416 if (auto *TGlued = Glued->getGluedNode())
3417 Ops.push_back(SDValue(TGlued, TGlued->getNumValues() - 1));
3418
3420 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3421
3422 if (!N->memoperands_empty())
3423 CurDAG->setNodeMemRefs(Result, N->memoperands());
3424
3425 Result->setFlags(N->getFlags());
3426 ReplaceUses(N, Result);
3427
3428 return true;
3429}
3430
3431static bool IsVMerge(SDNode *N) {
3432 return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMERGE_VVM;
3433}
3434
3435static bool IsVMv(SDNode *N) {
3436 return RISCV::getRVVMCOpcode(N->getMachineOpcode()) == RISCV::VMV_V_V;
3437}
3438
3439static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL) {
3440 switch (LMUL) {
3441 case RISCVII::LMUL_F8:
3442 return RISCV::PseudoVMSET_M_B1;
3443 case RISCVII::LMUL_F4:
3444 return RISCV::PseudoVMSET_M_B2;
3445 case RISCVII::LMUL_F2:
3446 return RISCV::PseudoVMSET_M_B4;
3447 case RISCVII::LMUL_1:
3448 return RISCV::PseudoVMSET_M_B8;
3449 case RISCVII::LMUL_2:
3450 return RISCV::PseudoVMSET_M_B16;
3451 case RISCVII::LMUL_4:
3452 return RISCV::PseudoVMSET_M_B32;
3453 case RISCVII::LMUL_8:
3454 return RISCV::PseudoVMSET_M_B64;
3456 llvm_unreachable("Unexpected LMUL");
3457 }
3458 llvm_unreachable("Unknown VLMUL enum");
3459}
3460
3461// Try to fold away VMERGE_VVM instructions. We handle these cases:
3462// -Masked TU VMERGE_VVM combined with an unmasked TA instruction instruction
3463// folds to a masked TU instruction. VMERGE_VVM must have have merge operand
3464// same as false operand.
3465// -Masked TA VMERGE_VVM combined with an unmasked TA instruction fold to a
3466// masked TA instruction.
3467// -Unmasked TU VMERGE_VVM combined with a masked MU TA instruction folds to
3468// masked TU instruction. Both instructions must have the same merge operand.
3469// VMERGE_VVM must have have merge operand same as false operand.
3470// Note: The VMERGE_VVM forms above (TA, and TU) refer to the policy implied,
3471// not the pseudo name. That is, a TA VMERGE_VVM can be either the _TU pseudo
3472// form with an IMPLICIT_DEF passthrough operand or the unsuffixed (TA) pseudo
3473// form.
3474bool RISCVDAGToDAGISel::performCombineVMergeAndVOps(SDNode *N) {
3475 SDValue Merge, False, True, VL, Mask, Glue;
3476 // A vmv.v.v is equivalent to a vmerge with an all-ones mask.
3477 if (IsVMv(N)) {
3478 Merge = N->getOperand(0);
3479 False = N->getOperand(0);
3480 True = N->getOperand(1);
3481 VL = N->getOperand(2);
3482 // A vmv.v.v won't have a Mask or Glue, instead we'll construct an all-ones
3483 // mask later below.
3484 } else {
3485 assert(IsVMerge(N));
3486 Merge = N->getOperand(0);
3487 False = N->getOperand(1);
3488 True = N->getOperand(2);
3489 Mask = N->getOperand(3);
3490 VL = N->getOperand(4);
3491 // We always have a glue node for the mask at v0.
3492 Glue = N->getOperand(N->getNumOperands() - 1);
3493 }
3494 assert(!Mask || cast<RegisterSDNode>(Mask)->getReg() == RISCV::V0);
3495 assert(!Glue || Glue.getValueType() == MVT::Glue);
3496
3497 // We require that either merge and false are the same, or that merge
3498 // is undefined.
3499 if (Merge != False && !isImplicitDef(Merge))
3500 return false;
3501
3502 assert(True.getResNo() == 0 &&
3503 "Expect True is the first output of an instruction.");
3504
3505 // Need N is the exactly one using True.
3506 if (!True.hasOneUse())
3507 return false;
3508
3509 if (!True.isMachineOpcode())
3510 return false;
3511
3512 unsigned TrueOpc = True.getMachineOpcode();
3513 const MCInstrDesc &TrueMCID = TII->get(TrueOpc);
3514 uint64_t TrueTSFlags = TrueMCID.TSFlags;
3515 bool HasTiedDest = RISCVII::isFirstDefTiedToFirstUse(TrueMCID);
3516
3517 bool IsMasked = false;
3519 RISCV::lookupMaskedIntrinsicByUnmasked(TrueOpc);
3520 if (!Info && HasTiedDest) {
3521 Info = RISCV::getMaskedPseudoInfo(TrueOpc);
3522 IsMasked = true;
3523 }
3524
3525 if (!Info)
3526 return false;
3527
3528 // When Mask is not a true mask, this transformation is illegal for some
3529 // operations whose results are affected by mask, like viota.m.
3530 if (Info->MaskAffectsResult && Mask && !usesAllOnesMask(Mask, Glue))
3531 return false;
3532
3533 if (HasTiedDest && !isImplicitDef(True->getOperand(0))) {
3534 // The vmerge instruction must be TU.
3535 // FIXME: This could be relaxed, but we need to handle the policy for the
3536 // resulting op correctly.
3537 if (isImplicitDef(Merge))
3538 return false;
3539 SDValue MergeOpTrue = True->getOperand(0);
3540 // Both the vmerge instruction and the True instruction must have the same
3541 // merge operand.
3542 if (False != MergeOpTrue)
3543 return false;
3544 }
3545
3546 if (IsMasked) {
3547 assert(HasTiedDest && "Expected tied dest");
3548 // The vmerge instruction must be TU.
3549 if (isImplicitDef(Merge))
3550 return false;
3551 // The vmerge instruction must have an all 1s mask since we're going to keep
3552 // the mask from the True instruction.
3553 // FIXME: Support mask agnostic True instruction which would have an
3554 // undef merge operand.
3555 if (Mask && !usesAllOnesMask(Mask, Glue))
3556 return false;
3557 }
3558
3559 // Skip if True has side effect.
3560 // TODO: Support vleff and vlsegff.
3561 if (TII->get(TrueOpc).hasUnmodeledSideEffects())
3562 return false;
3563
3564 // The last operand of a masked instruction may be glued.
3565 bool HasGlueOp = True->getGluedNode() != nullptr;
3566
3567 // The chain operand may exist either before the glued operands or in the last
3568 // position.
3569 unsigned TrueChainOpIdx = True.getNumOperands() - HasGlueOp - 1;
3570 bool HasChainOp =
3571 True.getOperand(TrueChainOpIdx).getValueType() == MVT::Other;
3572
3573 if (HasChainOp) {
3574 // Avoid creating cycles in the DAG. We must ensure that none of the other
3575 // operands depend on True through it's Chain.
3576 SmallVector<const SDNode *, 4> LoopWorklist;
3578 LoopWorklist.push_back(False.getNode());
3579 if (Mask)
3580 LoopWorklist.push_back(Mask.getNode());
3581 LoopWorklist.push_back(VL.getNode());
3582 if (Glue)
3583 LoopWorklist.push_back(Glue.getNode());
3584 if (SDNode::hasPredecessorHelper(True.getNode(), Visited, LoopWorklist))
3585 return false;
3586 }
3587
3588 // The vector policy operand may be present for masked intrinsics
3589 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TrueTSFlags);
3590 unsigned TrueVLIndex =
3591 True.getNumOperands() - HasVecPolicyOp - HasChainOp - HasGlueOp - 2;
3592 SDValue TrueVL = True.getOperand(TrueVLIndex);
3593 SDValue SEW = True.getOperand(TrueVLIndex + 1);
3594
3595 auto GetMinVL = [](SDValue LHS, SDValue RHS) {
3596 if (LHS == RHS)
3597 return LHS;
3598 if (isAllOnesConstant(LHS))
3599 return RHS;
3600 if (isAllOnesConstant(RHS))
3601 return LHS;
3602 auto *CLHS = dyn_cast<ConstantSDNode>(LHS);
3603 auto *CRHS = dyn_cast<ConstantSDNode>(RHS);
3604 if (!CLHS || !CRHS)
3605 return SDValue();
3606 return CLHS->getZExtValue() <= CRHS->getZExtValue() ? LHS : RHS;
3607 };
3608
3609 // Because N and True must have the same merge operand (or True's operand is
3610 // implicit_def), the "effective" body is the minimum of their VLs.
3611 SDValue OrigVL = VL;
3612 VL = GetMinVL(TrueVL, VL);
3613 if (!VL)
3614 return false;
3615
3616 // If we end up changing the VL or mask of True, then we need to make sure it
3617 // doesn't raise any observable fp exceptions, since changing the active
3618 // elements will affect how fflags is set.
3619 if (TrueVL != VL || !IsMasked)
3620 if (mayRaiseFPException(True.getNode()) &&
3621 !True->getFlags().hasNoFPExcept())
3622 return false;
3623
3624 SDLoc DL(N);
3625
3626 // From the preconditions we checked above, we know the mask and thus glue
3627 // for the result node will be taken from True.
3628 if (IsMasked) {
3629 Mask = True->getOperand(Info->MaskOpIdx);
3630 Glue = True->getOperand(True->getNumOperands() - 1);
3631 assert(Glue.getValueType() == MVT::Glue);
3632 }
3633 // If we end up using the vmerge mask the vmerge is actually a vmv.v.v, create
3634 // an all-ones mask to use.
3635 else if (IsVMv(N)) {
3636 unsigned TSFlags = TII->get(N->getMachineOpcode()).TSFlags;
3637 unsigned VMSetOpc = GetVMSetForLMul(RISCVII::getLMul(TSFlags));
3638 ElementCount EC = N->getValueType(0).getVectorElementCount();
3639 MVT MaskVT = MVT::getVectorVT(MVT::i1, EC);
3640
3641 SDValue AllOnesMask =
3642 SDValue(CurDAG->getMachineNode(VMSetOpc, DL, MaskVT, VL, SEW), 0);
3644 RISCV::V0, AllOnesMask, SDValue());
3645 Mask = CurDAG->getRegister(RISCV::V0, MaskVT);
3646 Glue = MaskCopy.getValue(1);
3647 }
3648
3649 unsigned MaskedOpc = Info->MaskedPseudo;
3650#ifndef NDEBUG
3651 const MCInstrDesc &MaskedMCID = TII->get(MaskedOpc);
3653 "Expected instructions with mask have policy operand.");
3654 assert(MaskedMCID.getOperandConstraint(MaskedMCID.getNumDefs(),
3655 MCOI::TIED_TO) == 0 &&
3656 "Expected instructions with mask have a tied dest.");
3657#endif
3658
3659 // Use a tumu policy, relaxing it to tail agnostic provided that the merge
3660 // operand is undefined.
3661 //
3662 // However, if the VL became smaller than what the vmerge had originally, then
3663 // elements past VL that were previously in the vmerge's body will have moved
3664 // to the tail. In that case we always need to use tail undisturbed to
3665 // preserve them.
3666 bool MergeVLShrunk = VL != OrigVL;
3667 uint64_t Policy = (isImplicitDef(Merge) && !MergeVLShrunk)
3669 : /*TUMU*/ 0;
3670 SDValue PolicyOp =
3671 CurDAG->getTargetConstant(Policy, DL, Subtarget->getXLenVT());
3672
3673
3675 Ops.push_back(False);
3676
3677 const bool HasRoundingMode = RISCVII::hasRoundModeOp(TrueTSFlags);
3678 const unsigned NormalOpsEnd = TrueVLIndex - IsMasked - HasRoundingMode;
3679 assert(!IsMasked || NormalOpsEnd == Info->MaskOpIdx);
3680 Ops.append(True->op_begin() + HasTiedDest, True->op_begin() + NormalOpsEnd);
3681
3682 Ops.push_back(Mask);
3683
3684 // For unmasked "VOp" with rounding mode operand, that is interfaces like
3685 // (..., rm, vl) or (..., rm, vl, policy).
3686 // Its masked version is (..., vm, rm, vl, policy).
3687 // Check the rounding mode pseudo nodes under RISCVInstrInfoVPseudos.td
3688 if (HasRoundingMode)
3689 Ops.push_back(True->getOperand(TrueVLIndex - 1));
3690
3691 Ops.append({VL, SEW, PolicyOp});
3692
3693 // Result node should have chain operand of True.
3694 if (HasChainOp)
3695 Ops.push_back(True.getOperand(TrueChainOpIdx));
3696
3697 // Add the glue for the CopyToReg of mask->v0.
3698 Ops.push_back(Glue);
3699
3701 CurDAG->getMachineNode(MaskedOpc, DL, True->getVTList(), Ops);
3702 Result->setFlags(True->getFlags());
3703
3704 if (!cast<MachineSDNode>(True)->memoperands_empty())
3705 CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(True)->memoperands());
3706
3707 // Replace vmerge.vvm node by Result.
3708 ReplaceUses(SDValue(N, 0), SDValue(Result, 0));
3709
3710 // Replace another value of True. E.g. chain and VL.
3711 for (unsigned Idx = 1; Idx < True->getNumValues(); ++Idx)
3712 ReplaceUses(True.getValue(Idx), SDValue(Result, Idx));
3713
3714 // Try to transform Result to unmasked intrinsic.
3715 doPeepholeMaskedRVV(Result);
3716 return true;
3717}
3718
3719bool RISCVDAGToDAGISel::doPeepholeMergeVVMFold() {
3720 bool MadeChange = false;
3722
3723 while (Position != CurDAG->allnodes_begin()) {
3724 SDNode *N = &*--Position;
3725 if (N->use_empty() || !N->isMachineOpcode())
3726 continue;
3727
3728 if (IsVMerge(N) || IsVMv(N))
3729 MadeChange |= performCombineVMergeAndVOps(N);
3730 }
3731 return MadeChange;
3732}
3733
3734/// If our passthru is an implicit_def, use noreg instead. This side
3735/// steps issues with MachineCSE not being able to CSE expressions with
3736/// IMPLICIT_DEF operands while preserving the semantic intent. See
3737/// pr64282 for context. Note that this transform is the last one
3738/// performed at ISEL DAG to DAG.
3739bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
3740 bool MadeChange = false;
3742
3743 while (Position != CurDAG->allnodes_begin()) {
3744 SDNode *N = &*--Position;
3745 if (N->use_empty() || !N->isMachineOpcode())
3746 continue;
3747
3748 const unsigned Opc = N->getMachineOpcode();
3749 if (!RISCVVPseudosTable::getPseudoInfo(Opc) ||
3751 !isImplicitDef(N->getOperand(0)))
3752 continue;
3753
3755 Ops.push_back(CurDAG->getRegister(RISCV::NoRegister, N->getValueType(0)));
3756 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
3757 SDValue Op = N->getOperand(I);
3758 Ops.push_back(Op);
3759 }
3760
3762 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
3763 Result->setFlags(N->getFlags());
3764 CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(N)->memoperands());
3765 ReplaceUses(N, Result);
3766 MadeChange = true;
3767 }
3768 return MadeChange;
3769}
3770
3771
3772// This pass converts a legalized DAG into a RISCV-specific DAG, ready
3773// for instruction scheduling.
3775 CodeGenOptLevel OptLevel) {
3776 return new RISCVDAGToDAGISel(TM, OptLevel);
3777}
3778
3779char RISCVDAGToDAGISel::ID = 0;
3780
static Register createTuple(ArrayRef< Register > Regs, const unsigned RegClassIDs[], const unsigned SubRegs[], MachineIRBuilder &MIB)
Create a REG_SEQUENCE instruction using the registers in Regs.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
#define DEBUG_TYPE
const HexagonInstrInfo * TII
#define I(x, y, z)
Definition: MD5.cpp:58
mir Rename Register Operands
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
const char LLVMTargetMachineRef TM
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
R600 Clause Merge
static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, int64_t Imm, const RISCVSubtarget &Subtarget)
#define CASE_VMSLT_OPCODES(lmulenum, suffix, suffix_b)
static bool isWorthFoldingAdd(SDValue Add)
static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, RISCVMatInt::InstSeq &Seq)
static bool isImplicitDef(SDValue V)
static unsigned GetVMSetForLMul(RISCVII::VLMUL LMUL)
#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix)
static bool usesAllOnesMask(SDValue MaskOp, SDValue GlueOp)
static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo, unsigned Bits, const TargetInstrInfo *TII)
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, const RISCVSubtarget *Subtarget, SDValue Addr, SDValue &Base, SDValue &Offset, bool IsPrefetch=false)
static bool IsVMv(SDNode *N)
static cl::opt< bool > UsePseudoMovImm("riscv-use-rematerializable-movimm", cl::Hidden, cl::desc("Use a rematerializable pseudoinstruction for 2 instruction " "constant materialization"), cl::init(false))
#define CASE_VMSLT_VMNAND_VMSET_OPCODES(lmulenum, suffix, suffix_b)
static SDValue findVSplat(SDValue N)
static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, std::function< bool(int64_t)> ValidateImm)
static bool IsVMerge(SDNode *N)
unsigned Log2SEW
RISCVII::VLMUL VLMul
unsigned SEW
uint64_t TSFlags
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
#define PASS_NAME
Value * RHS
Value * LHS
static constexpr uint32_t Opcode
Definition: aarch32.h:200
bool isZero() const
Definition: APFloat.h:1291
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
bool isPosZero() const
Definition: APFloat.h:1306
bool isNegZero() const
Definition: APFloat.h:1307
Class for arbitrary precision integers.
Definition: APInt.h:76
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1433
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1229
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1507
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
const APFloat & getValueAPF() const
uint64_t getZExtValue() const
int64_t getSExtValue() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
This class is used to form a handle around another node that is persistent and is updated across invo...
static StringRef getMemConstraintName(ConstraintCode C)
Definition: InlineAsm.h:467
ISD::MemIndexedMode getAddressingMode() const
Return the addressing mode for this load or store: unindexed, pre-inc, pre-dec, post-inc,...
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumDefs() const
Return the number of MachineOperands that are register definitions.
Definition: MCInstrDesc.h:248
int getOperandConstraint(unsigned OpNum, MCOI::OperandConstraint Constraint) const
Returns the value of the specified operand constraint if it is present.
Definition: MCInstrDesc.h:219
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by other flags.
Definition: MCInstrDesc.h:463
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition: MCInstrInfo.h:63
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
void setFlags(Flags f)
Bitwise OR the current flags with the given flags.
An SDNode that represents everything that will be needed to construct a MachineInstr.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
bool selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal, SDValue &Val)
RISC-V doesn't have general instructions for integer setne/seteq, but we can check for equality with ...
bool selectSExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectZExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectSHXADD_UWOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD_UW.
bool hasAllNBitUsers(SDNode *Node, unsigned Bits, const unsigned Depth=0) const
void selectVSSEG(SDNode *Node, bool IsMasked, bool IsStrided)
bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset)
Similar to SelectAddrRegImm, except that the least significant 5 bits of Offset shoule be all zeros.
bool SelectFrameAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
void selectVLSEGFF(SDNode *Node, bool IsMasked)
bool selectFPImm(SDValue N, SDValue &Imm)
bool selectSimm5Shl2(SDValue N, SDValue &Simm5, SDValue &Shl2)
bool selectLow8BitsVSplat(SDValue N, SDValue &SplatVal)
bool hasAllHUsers(SDNode *Node) const
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool selectVSplatSimm5(SDValue N, SDValue &SplatVal)
bool selectRVVSimm5(SDValue N, unsigned Width, SDValue &Imm)
bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset)
bool hasAllWUsers(SDNode *Node) const
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
void Select(SDNode *Node) override
Main hook for targets to transform nodes into machine nodes.
bool selectVSplat(SDValue N, SDValue &SplatVal)
void addVectorLoadStoreOperands(SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp, bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl< SDValue > &Operands, bool IsLoad=false, MVT *IndexVT=nullptr)
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
void selectVLXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
bool tryShrinkShlLogicImm(SDNode *Node)
void selectVSETVLI(SDNode *Node)
bool selectVLOp(SDValue N, SDValue &VL)
bool trySignedBitfieldExtract(SDNode *Node)
void selectVSXSEG(SDNode *Node, bool IsMasked, bool IsOrdered)
bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal)
bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal)
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset, bool IsINX=false)
void selectVLSEG(SDNode *Node, bool IsMasked, bool IsStrided)
bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt)
bool selectSHXADDOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD.
bool tryIndexedLoad(SDNode *Node)
bool SelectAddrRegRegScale(SDValue Addr, unsigned MaxShiftAmount, SDValue &Base, SDValue &Index, SDValue &Scale)
bool selectVSplatUimm(SDValue N, unsigned Bits, SDValue &SplatVal)
unsigned getRealMinVLen() const
unsigned getXLen() const
bool hasVInstructions() const
unsigned getRealMaxVLen() const
bool hasStdExtZhinxOrZhinxmin() const
const RISCVRegisterInfo * getRegisterInfo() const override
const RISCVTargetLowering * getTargetLowering() const override
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
static unsigned getSubregIndexByMVT(MVT VT, unsigned Index)
static unsigned getRegClassIDForVecVT(MVT VT)
static RISCVII::VLMUL getLMUL(MVT VT)
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDNodeFlags getFlags() const
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
unsigned getMachineOpcode() const
This may only be called if isMachineOpcode returns true.
SDVTList getVTList() const
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
const SDValue & getOperand(unsigned i) const
const APInt & getConstantOperandAPInt(unsigned i) const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
unsigned getNumOperands() const
const TargetLowering * TLI
MachineFunction * MF
const TargetInstrInfo * TII
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
bool mayRaiseFPException(SDNode *Node) const
Return whether the node may raise an FP exception.
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:720
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:532
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getTargetFrameIndex(int FI, EVT VT)
Definition: SelectionDAG.h:725
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getRegister(unsigned Reg, EVT VT)
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:771
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:674
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:797
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:687
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
TargetInstrInfo - Interface to description of machine instruction set.
CodeGenOptLevel getOptLevel() const
Returns the optimization level: None, Less, Default, or Aggressive.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:324
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1233
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1213
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1229
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition: ISDOpcodes.h:203
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1455
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1506
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
Definition: ISDOpcodes.h:1551
static bool hasRoundModeOp(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
static unsigned decodeVSEW(unsigned VSEW)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
std::optional< unsigned > getVectorLowDemandedScalarBits(uint16_t Opcode, unsigned Log2SEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
static constexpr int64_t VLMaxSentinel
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
static const MachineMemOperand::Flags MONontemporalBit1
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:306
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:316
static const MachineMemOperand::Flags MONontemporalBit0
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:214
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition: MathExtras.h:258
unsigned M1(unsigned Val)
Definition: VE.h:376
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition: bit.h:280
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition: MathExtras.h:246
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
@ Add
Sum of integers.
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
FunctionPass * createRISCVISelDag(RISCVTargetMachine &TM, CodeGenOptLevel OptLevel)
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:452
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:373
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
bool hasNoFPExcept() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.