LLVM 24.0.0git
RISCVISelDAGToDAG.cpp
Go to the documentation of this file.
1//===-- RISCVISelDAGToDAG.cpp - A dag to dag inst selector for RISC-V -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines an instruction selector for the RISC-V target.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVISelDAGToDAG.h"
17#include "RISCVISelLowering.h"
18#include "RISCVInstrInfo.h"
21#include "llvm/IR/IntrinsicsRISCV.h"
23#include "llvm/Support/Debug.h"
26
27using namespace llvm;
28
29#define DEBUG_TYPE "riscv-isel"
30#define PASS_NAME "RISC-V DAG->DAG Pattern Instruction Selection"
31
32#define GET_DAGISEL_BODY RISCVDAGToDAGISel
33#include "RISCVGenDAGISel.inc"
34
36 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
37
38 bool MadeChange = false;
39 while (Position != CurDAG->allnodes_begin()) {
40 SDNode *N = &*--Position;
41 if (N->use_empty())
42 continue;
43
44 SDValue Result;
45 switch (N->getOpcode()) {
46 case ISD::SPLAT_VECTOR: {
47 if (Subtarget->hasStdExtP())
48 break;
49 // Convert integer SPLAT_VECTOR to VMV_V_X_VL and floating-point
50 // SPLAT_VECTOR to VFMV_V_F_VL to reduce isel burden.
51 MVT VT = N->getSimpleValueType(0);
52 unsigned Opc =
53 VT.isInteger() ? RISCVISD::VMV_V_X_VL : RISCVISD::VFMV_V_F_VL;
54 SDLoc DL(N);
55 SDValue VL = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
56 SDValue Src = N->getOperand(0);
57 if (VT.isInteger())
58 Src = CurDAG->getNode(ISD::ANY_EXTEND, DL, Subtarget->getXLenVT(),
59 N->getOperand(0));
60 Result = CurDAG->getNode(Opc, DL, VT, CurDAG->getUNDEF(VT), Src, VL);
61 break;
62 }
63 case RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL: {
64 // Lower SPLAT_VECTOR_SPLIT_I64 to two scalar stores and a stride 0 vector
65 // load. Done after lowering and combining so that we have a chance to
66 // optimize this to VMV_V_X_VL when the upper bits aren't needed.
67 assert(N->getNumOperands() == 4 && "Unexpected number of operands");
68 MVT VT = N->getSimpleValueType(0);
69 SDValue Passthru = N->getOperand(0);
70 SDValue Lo = N->getOperand(1);
71 SDValue Hi = N->getOperand(2);
72 SDValue VL = N->getOperand(3);
73 assert(VT.getVectorElementType() == MVT::i64 && VT.isScalableVector() &&
74 Lo.getValueType() == MVT::i32 && Hi.getValueType() == MVT::i32 &&
75 "Unexpected VTs!");
76 MachineFunction &MF = CurDAG->getMachineFunction();
77 SDLoc DL(N);
78
79 // Create temporary stack for each expanding node.
80 SDValue StackSlot =
81 CurDAG->CreateStackTemporary(TypeSize::getFixed(8), Align(8));
82 int FI = cast<FrameIndexSDNode>(StackSlot.getNode())->getIndex();
84
85 SDValue Chain = CurDAG->getEntryNode();
86 Lo = CurDAG->getStore(Chain, DL, Lo, StackSlot, MPI, Align(8));
87
88 SDValue OffsetSlot =
89 CurDAG->getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), DL);
90 Hi = CurDAG->getStore(Chain, DL, Hi, OffsetSlot, MPI.getWithOffset(4),
91 Align(8));
92
93 Chain = CurDAG->getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
94
95 SDVTList VTs = CurDAG->getVTList({VT, MVT::Other});
96 SDValue IntID =
97 CurDAG->getTargetConstant(Intrinsic::riscv_vlse, DL, MVT::i64);
98 SDValue Ops[] = {Chain,
99 IntID,
100 Passthru,
101 StackSlot,
102 CurDAG->getRegister(RISCV::X0, MVT::i64),
103 VL};
104
105 Result = CurDAG->getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
106 MVT::i64, MPI, Align(8),
108 break;
109 }
110 case ISD::FP_EXTEND: {
111 // We only have vector patterns for riscv_fpextend_vl in isel.
112 SDLoc DL(N);
113 MVT VT = N->getSimpleValueType(0);
114 if (!VT.isVector())
115 break;
116 SDValue VLMAX = CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT());
117 SDValue TrueMask = CurDAG->getNode(
118 RISCVISD::VMSET_VL, DL, VT.changeVectorElementType(MVT::i1), VLMAX);
119 Result = CurDAG->getNode(RISCVISD::FP_EXTEND_VL, DL, VT, N->getOperand(0),
120 TrueMask, VLMAX);
121 break;
122 }
123 case ISD::ADD: {
124 // Turn (add X, C) into (sub X, -C) when a constant node holding -C
125 // already exists in the DAG, so both share one materialization. Do this
126 // before selection, while both are still ConstantSDNodes: by selection
127 // time -C may already have been selected into instructions.
128 //
129 // ADD is commutative, but getNode canonicalizes constants to the RHS, so
130 // the constant is always operand 1.
131 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
132 if (!N1C)
133 break;
134 MVT VT = N->getSimpleValueType(0);
135 if (VT != Subtarget->getXLenVT())
136 break;
137 int64_t Imm = N1C->getSExtValue();
138 // Only worthwhile for wide constants: values that fit in 32 bits take at
139 // most two instructions to materialize, matching the threshold used by
140 // selectNegImm. Skip INT64_MIN too, whose negation is itself.
141 if (isInt<32>(Imm) || Imm == INT64_MIN)
142 break;
143 // A constant is anchored if it has a user other than an ADD, i.e. it is
144 // materialized regardless of this fold. N1C is the (unique) node for Imm,
145 // so the positive side needs no search.
146 auto IsAnchored = [](const SDNode *C) {
147 return any_of(C->users(), [](const SDNode *U) {
148 return U->getOpcode() != ISD::ADD;
149 });
150 };
151 // If Imm is materialized anyway, keep the ADD so it reuses Imm; an ADD is
152 // also more compressible than a SUB. This also lets us skip the search
153 // for -Imm below.
154 if (IsAnchored(N1C))
155 break;
156 // Find the (unique) constant node for -Imm, if any.
157 const SDNode *NegC = nullptr;
158 for (const SDNode &Node : CurDAG->allnodes()) {
160 if (C && C->getSimpleValueType(0) == VT && C->getSExtValue() == -Imm) {
161 NegC = &Node;
162 break;
163 }
164 }
165 // Reuse is only free if -Imm is already in the DAG.
166 if (!NegC)
167 break;
168 // dyn_cast<ConstantSDNode> also matches TargetConstant, which is encoded
169 // into the instruction rather than materialized, so reusing it would not
170 // remove a materialization. No TargetConstant is this wide (the largest
171 // are intrinsic IDs, which fit in 32 bits), so assert it is a Constant.
172 assert(NegC->getOpcode() == ISD::Constant &&
173 "Unexpected wide TargetConstant");
174 // Pick which of Imm/-Imm should be the surviving constant, so exactly
175 // one of the pair is materialized and any ADDs of the other reuse it:
176 // - if -Imm is materialized anyway, reuse it (rewrite to SUB);
177 // - else keep the cheaper constant, breaking ties towards the positive
178 // value so both ADDs of a C/-C pair agree on the survivor.
179 bool Rewrite;
180 if (IsAnchored(NegC)) {
181 Rewrite = true;
182 } else {
183 int PosCost = RISCVMatInt::getIntMatCost(APInt(64, Imm), 64, *Subtarget,
184 /*CompressionCost=*/true);
185 int NegCost =
186 RISCVMatInt::getIntMatCost(APInt(64, -Imm), 64, *Subtarget,
187 /*CompressionCost=*/true);
188 Rewrite = NegCost != PosCost ? NegCost < PosCost : Imm < 0;
189 }
190 if (!Rewrite)
191 break;
192 SDLoc DL(N);
193 // getConstant uniques onto the existing -C node, so it is shared.
194 Result = CurDAG->getNode(ISD::SUB, DL, VT, N->getOperand(0),
195 CurDAG->getConstant(-Imm, DL, VT));
196 break;
197 }
198 }
199
200 if (Result) {
201 LLVM_DEBUG(dbgs() << "RISC-V DAG preprocessing replacing:\nOld: ");
202 LLVM_DEBUG(N->dump(CurDAG));
203 LLVM_DEBUG(dbgs() << "\nNew: ");
204 LLVM_DEBUG(Result->dump(CurDAG));
205 LLVM_DEBUG(dbgs() << "\n");
206
207 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Result);
208 MadeChange = true;
209 }
210 }
211
212 if (MadeChange)
213 CurDAG->RemoveDeadNodes();
214}
215
217 HandleSDNode Dummy(CurDAG->getRoot());
218 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
219
220 bool MadeChange = false;
221 while (Position != CurDAG->allnodes_begin()) {
222 SDNode *N = &*--Position;
223 // Skip dead nodes and any non-machine opcodes.
224 if (N->use_empty() || !N->isMachineOpcode())
225 continue;
226
227 MadeChange |= doPeepholeSExtW(N);
228
229 // FIXME: This is here only because the VMerge transform doesn't
230 // know how to handle masked true inputs. Once that has been moved
231 // to post-ISEL, this can be deleted as well.
232 MadeChange |= doPeepholeMaskedRVV(cast<MachineSDNode>(N));
233 }
234
235 CurDAG->setRoot(Dummy.getValue());
236
237 // After we're done with everything else, convert IMPLICIT_DEF
238 // passthru operands to NoRegister. This is required to workaround
239 // an optimization deficiency in MachineCSE. This really should
240 // be merged back into each of the patterns (i.e. there's no good
241 // reason not to go directly to NoReg), but is being done this way
242 // to allow easy backporting.
243 MadeChange |= doPeepholeNoRegPassThru();
244
245 if (MadeChange)
246 CurDAG->RemoveDeadNodes();
247}
248
249static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
251 SDValue SrcReg = CurDAG->getRegister(RISCV::X0, VT);
252 for (const RISCVMatInt::Inst &Inst : Seq) {
253 SDValue SDImm = CurDAG->getSignedTargetConstant(Inst.getImm(), DL, VT);
254 SDNode *Result = nullptr;
255 switch (Inst.getOpndKind()) {
256 case RISCVMatInt::Imm:
257 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SDImm);
258 break;
260 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg,
261 CurDAG->getRegister(RISCV::X0, VT));
262 break;
264 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SrcReg);
265 break;
267 Result = CurDAG->getMachineNode(Inst.getOpcode(), DL, VT, SrcReg, SDImm);
268 break;
269 }
270
271 // Only the first instruction has X0 as its source.
272 SrcReg = SDValue(Result, 0);
273 }
274
275 return SrcReg;
276}
277
278static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT,
279 int64_t Imm, const RISCVSubtarget &Subtarget) {
281
282 // Use a rematerializable pseudo instruction for short sequences if enabled.
283 if (Seq.size() == 2 && Subtarget.getCLOpts().use_rematerializable_movimm)
284 return SDValue(
285 CurDAG->getMachineNode(RISCV::PseudoMovImm, DL, VT,
286 CurDAG->getSignedTargetConstant(Imm, DL, VT)),
287 0);
288
289 // See if we can create this constant as (ADD (SLLI X, C), X) where X is at
290 // worst an LUI+ADDIW. This will require an extra register, but avoids a
291 // constant pool.
292 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
293 // low and high 32 bits are the same and bit 31 and 63 are set.
294 if (Seq.size() > 3) {
295 unsigned ShiftAmt, AddOpc;
298 if (!SeqLo.empty() && (SeqLo.size() + 2) < Seq.size()) {
299 SDValue Lo = selectImmSeq(CurDAG, DL, VT, SeqLo);
300
301 SDValue SLLI = SDValue(
302 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, Lo,
303 CurDAG->getTargetConstant(ShiftAmt, DL, VT)),
304 0);
305 return SDValue(CurDAG->getMachineNode(AddOpc, DL, VT, Lo, SLLI), 0);
306 }
307 }
308
309 // Otherwise, use the original sequence.
310 return selectImmSeq(CurDAG, DL, VT, Seq);
311}
312
314 SDNode *Node, unsigned Log2SEW, const SDLoc &DL, unsigned CurOp,
315 bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl<SDValue> &Operands,
316 bool IsLoad, MVT *IndexVT) {
317 SDValue Chain = Node->getOperand(0);
318
319 Operands.push_back(Node->getOperand(CurOp++)); // Base pointer.
320
321 if (IsStridedOrIndexed) {
322 Operands.push_back(Node->getOperand(CurOp++)); // Index.
323 if (IndexVT)
324 *IndexVT = Operands.back()->getSimpleValueType(0);
325 }
326
327 if (IsMasked) {
328 SDValue Mask = Node->getOperand(CurOp++);
329 Operands.push_back(Mask);
330 }
331 SDValue VL;
332 selectVLOp(Node->getOperand(CurOp++), VL);
333 Operands.push_back(VL);
334
335 MVT XLenVT = Subtarget->getXLenVT();
336 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
337 Operands.push_back(SEWOp);
338
339 // At the IR layer, all the masked load intrinsics have policy operands,
340 // none of the others do. All have passthru operands. For our pseudos,
341 // all loads have policy operands.
342 if (IsLoad) {
343 uint64_t Policy = RISCVVType::MASK_AGNOSTIC;
344 if (IsMasked)
345 Policy = Node->getConstantOperandVal(CurOp++);
346 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
347 Operands.push_back(PolicyOp);
348 }
349
350 Operands.push_back(Chain); // Chain.
351}
352
353void RISCVDAGToDAGISel::selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked,
354 bool IsStrided) {
355 SDLoc DL(Node);
356 MVT VT = Node->getSimpleValueType(0);
357 unsigned Log2SEW = Node->getConstantOperandVal(Node->getNumOperands() - 1);
359
360 unsigned CurOp = 2;
362
363 Operands.push_back(Node->getOperand(CurOp++));
364
365 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
366 Operands, /*IsLoad=*/true);
367
368 const RISCV::VLSEGPseudo *P =
369 RISCV::getVLSEGPseudo(NF, IsMasked, IsStrided, /*FF*/ false, Log2SEW,
370 static_cast<unsigned>(LMUL));
372 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
373
374 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
375
378 CurDAG->RemoveDeadNode(Node);
379}
380
382 bool IsMasked) {
383 SDLoc DL(Node);
384 MVT VT = Node->getSimpleValueType(0);
385 MVT XLenVT = Subtarget->getXLenVT();
386 unsigned Log2SEW = Node->getConstantOperandVal(Node->getNumOperands() - 1);
388
389 unsigned CurOp = 2;
391
392 Operands.push_back(Node->getOperand(CurOp++));
393
394 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
395 /*IsStridedOrIndexed*/ false, Operands,
396 /*IsLoad=*/true);
397
398 const RISCV::VLSEGPseudo *P =
399 RISCV::getVLSEGPseudo(NF, IsMasked, /*Strided*/ false, /*FF*/ true,
400 Log2SEW, static_cast<unsigned>(LMUL));
401 MachineSDNode *Load = CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped,
402 XLenVT, MVT::Other, Operands);
403
404 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
405
406 ReplaceUses(SDValue(Node, 0), SDValue(Load, 0)); // Result
407 ReplaceUses(SDValue(Node, 1), SDValue(Load, 1)); // VL
408 ReplaceUses(SDValue(Node, 2), SDValue(Load, 2)); // Chain
409 CurDAG->RemoveDeadNode(Node);
410}
411
412void RISCVDAGToDAGISel::selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked,
413 bool IsOrdered) {
414 SDLoc DL(Node);
415 MVT VT = Node->getSimpleValueType(0);
416 unsigned Log2SEW = Node->getConstantOperandVal(Node->getNumOperands() - 1);
418
419 unsigned CurOp = 2;
421
422 Operands.push_back(Node->getOperand(CurOp++));
423
424 MVT IndexVT;
425 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
426 /*IsStridedOrIndexed*/ true, Operands,
427 /*IsLoad=*/true, &IndexVT);
428
429#ifndef NDEBUG
430 // Number of element = RVVBitsPerBlock * LMUL / SEW
431 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
432 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
433 if (DecodedLMUL.second)
434 ContainedTyNumElts /= DecodedLMUL.first;
435 else
436 ContainedTyNumElts *= DecodedLMUL.first;
437 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
438 "Element count mismatch");
439#endif
440
442 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
443 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
444 reportFatalUsageError("The V extension does not support EEW=64 for index "
445 "values when XLEN=32");
446 }
447 const RISCV::VLXSEGPseudo *P = RISCV::getVLXSEGPseudo(
448 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
449 static_cast<unsigned>(IndexLMUL));
451 CurDAG->getMachineNode(P->Pseudo, DL, MVT::Untyped, MVT::Other, Operands);
452
453 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
454
457 CurDAG->RemoveDeadNode(Node);
458}
459
460void RISCVDAGToDAGISel::selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked,
461 bool IsStrided) {
462 SDLoc DL(Node);
463 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
464 unsigned Log2SEW = Node->getConstantOperandVal(Node->getNumOperands() - 1);
466
467 unsigned CurOp = 2;
469
470 Operands.push_back(Node->getOperand(CurOp++));
471
472 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
473 Operands);
474
475 const RISCV::VSSEGPseudo *P = RISCV::getVSSEGPseudo(
476 NF, IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
478 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
479
480 CurDAG->setNodeMemRefs(Store, {cast<MemSDNode>(Node)->getMemOperand()});
481
483}
484
485void RISCVDAGToDAGISel::selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked,
486 bool IsOrdered) {
487 SDLoc DL(Node);
488 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
489 unsigned Log2SEW = Node->getConstantOperandVal(Node->getNumOperands() - 1);
491
492 unsigned CurOp = 2;
494
495 Operands.push_back(Node->getOperand(CurOp++));
496
497 MVT IndexVT;
498 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
499 /*IsStridedOrIndexed*/ true, Operands,
500 /*IsLoad=*/false, &IndexVT);
501
502#ifndef NDEBUG
503 // Number of element = RVVBitsPerBlock * LMUL / SEW
504 unsigned ContainedTyNumElts = RISCV::RVVBitsPerBlock >> Log2SEW;
505 auto DecodedLMUL = RISCVVType::decodeVLMUL(LMUL);
506 if (DecodedLMUL.second)
507 ContainedTyNumElts /= DecodedLMUL.first;
508 else
509 ContainedTyNumElts *= DecodedLMUL.first;
510 assert(ContainedTyNumElts == IndexVT.getVectorMinNumElements() &&
511 "Element count mismatch");
512#endif
513
515 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
516 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
517 reportFatalUsageError("The V extension does not support EEW=64 for index "
518 "values when XLEN=32");
519 }
520 const RISCV::VSXSEGPseudo *P = RISCV::getVSXSEGPseudo(
521 NF, IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
522 static_cast<unsigned>(IndexLMUL));
524 CurDAG->getMachineNode(P->Pseudo, DL, Node->getValueType(0), Operands);
525
526 CurDAG->setNodeMemRefs(Store, {cast<MemSDNode>(Node)->getMemOperand()});
527
529}
530
532 if (!Subtarget->hasVInstructions())
533 return;
534
535 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
536
537 SDLoc DL(Node);
538 MVT XLenVT = Subtarget->getXLenVT();
539
540 unsigned IntNo = Node->getConstantOperandVal(0);
541
542 assert((IntNo == Intrinsic::riscv_vsetvli ||
543 IntNo == Intrinsic::riscv_vsetvlimax) &&
544 "Unexpected vsetvli intrinsic");
545
546 bool VLMax = IntNo == Intrinsic::riscv_vsetvlimax;
547 unsigned Offset = (VLMax ? 1 : 2);
548
549 assert(Node->getNumOperands() == Offset + 2 &&
550 "Unexpected number of operands");
551
552 unsigned SEW =
553 RISCVVType::decodeVSEW(Node->getConstantOperandVal(Offset) & 0x7);
554 RISCVVType::VLMUL VLMul = static_cast<RISCVVType::VLMUL>(
555 Node->getConstantOperandVal(Offset + 1) & 0x7);
556
557 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
558 /*MaskAgnostic*/ true);
559 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
560
561 SDValue VLOperand;
562 unsigned Opcode = RISCV::PseudoVSETVLI;
563 if (auto *C = dyn_cast<ConstantSDNode>(Node->getOperand(1))) {
564 if (auto VLEN = Subtarget->getRealVLen())
565 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == C->getZExtValue())
566 VLMax = true;
567 }
568 if (VLMax || isAllOnesConstant(Node->getOperand(1))) {
569 VLOperand = CurDAG->getRegister(RISCV::X0, XLenVT);
570 Opcode = RISCV::PseudoVSETVLIX0;
571 } else {
572 VLOperand = Node->getOperand(1);
573
574 if (auto *C = dyn_cast<ConstantSDNode>(VLOperand)) {
575 uint64_t AVL = C->getZExtValue();
576 if (isUInt<5>(AVL)) {
577 SDValue VLImm = CurDAG->getTargetConstant(AVL, DL, XLenVT);
578 ReplaceNode(Node, CurDAG->getMachineNode(RISCV::PseudoVSETIVLI, DL,
579 XLenVT, VLImm, VTypeIOp));
580 return;
581 }
582 }
583 }
584
586 CurDAG->getMachineNode(Opcode, DL, XLenVT, VLOperand, VTypeIOp));
587}
588
590 if (!Subtarget->hasVendorXSfmmbase())
591 return;
592
593 assert(Node->getOpcode() == ISD::INTRINSIC_WO_CHAIN && "Unexpected opcode");
594
595 SDLoc DL(Node);
596 MVT XLenVT = Subtarget->getXLenVT();
597
598 unsigned IntNo = Node->getConstantOperandVal(0);
599
600 assert((IntNo == Intrinsic::riscv_sf_vsettnt ||
601 IntNo == Intrinsic::riscv_sf_vsettm ||
602 IntNo == Intrinsic::riscv_sf_vsettk) &&
603 "Unexpected XSfmm vset intrinsic");
604
605 unsigned SEW = RISCVVType::decodeVSEW(Node->getConstantOperandVal(2));
606 unsigned Widen = RISCVVType::decodeTWiden(Node->getConstantOperandVal(3));
607 unsigned PseudoOpCode =
608 IntNo == Intrinsic::riscv_sf_vsettnt ? RISCV::PseudoSF_VSETTNT
609 : IntNo == Intrinsic::riscv_sf_vsettm ? RISCV::PseudoSF_VSETTM
610 : RISCV::PseudoSF_VSETTK;
611
612 if (IntNo == Intrinsic::riscv_sf_vsettnt) {
613 unsigned VTypeI = RISCVVType::encodeXSfmmVType(SEW, Widen, 0);
614 SDValue VTypeIOp = CurDAG->getTargetConstant(VTypeI, DL, XLenVT);
615
616 ReplaceNode(Node, CurDAG->getMachineNode(PseudoOpCode, DL, XLenVT,
617 Node->getOperand(1), VTypeIOp));
618 } else {
619 SDValue Log2SEW = CurDAG->getTargetConstant(Log2_32(SEW), DL, XLenVT);
620 SDValue TWiden = CurDAG->getTargetConstant(Widen, DL, XLenVT);
622 CurDAG->getMachineNode(PseudoOpCode, DL, XLenVT,
623 Node->getOperand(1), Log2SEW, TWiden));
624 }
625}
626
628 MVT VT = Node->getSimpleValueType(0);
629 unsigned Opcode = Node->getOpcode();
630 assert((Opcode == ISD::AND || Opcode == ISD::OR || Opcode == ISD::XOR) &&
631 "Unexpected opcode");
632 SDLoc DL(Node);
633
634 // For operations of the form (x << C1) op C2, check if we can use
635 // ANDI/ORI/XORI by transforming it into (x op (C2>>C1)) << C1.
636 SDValue N0 = Node->getOperand(0);
637 SDValue N1 = Node->getOperand(1);
638
640 if (!Cst)
641 return false;
642
643 int64_t Val = Cst->getSExtValue();
644
645 // Check if immediate can already use ANDI/ORI/XORI.
646 if (isInt<12>(Val))
647 return false;
648
649 SDValue Shift = N0;
650
651 // If Val is simm32 and we have a sext_inreg from i32, then the binop
652 // produces at least 33 sign bits. We can peek through the sext_inreg and use
653 // a SLLIW at the end.
654 bool SignExt = false;
655 if (isInt<32>(Val) && N0.getOpcode() == ISD::SIGN_EXTEND_INREG &&
656 N0.hasOneUse() && cast<VTSDNode>(N0.getOperand(1))->getVT() == MVT::i32) {
657 SignExt = true;
658 Shift = N0.getOperand(0);
659 }
660
661 if (Shift.getOpcode() != ISD::SHL || !Shift.hasOneUse())
662 return false;
663
665 if (!ShlCst)
666 return false;
667
668 uint64_t ShAmt = ShlCst->getZExtValue();
669 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
670 "Unexpected shift amount!");
671
672 // Make sure that we don't change the operation by removing bits.
673 // This only matters for OR and XOR, AND is unaffected.
674 uint64_t RemovedBitsMask = maskTrailingOnes<uint64_t>(ShAmt);
675 if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
676 return false;
677
678 int64_t ShiftedVal = Val >> ShAmt;
679 if (!isInt<12>(ShiftedVal))
680 return false;
681
682 // If we peeked through a sext_inreg, make sure the shift is valid for SLLIW.
683 if (SignExt && ShAmt >= 32)
684 return false;
685
686 // Ok, we can reorder to get a smaller immediate.
687 unsigned BinOpc;
688 switch (Opcode) {
689 default: llvm_unreachable("Unexpected opcode");
690 case ISD::AND: BinOpc = RISCV::ANDI; break;
691 case ISD::OR: BinOpc = RISCV::ORI; break;
692 case ISD::XOR: BinOpc = RISCV::XORI; break;
693 }
694
695 unsigned ShOpc = SignExt ? RISCV::SLLIW : RISCV::SLLI;
696
697 SDNode *BinOp = CurDAG->getMachineNode(
698 BinOpc, DL, VT, Shift.getOperand(0),
699 CurDAG->getSignedTargetConstant(ShiftedVal, DL, VT));
700 SDNode *SLLI =
701 CurDAG->getMachineNode(ShOpc, DL, VT, SDValue(BinOp, 0),
702 CurDAG->getTargetConstant(ShAmt, DL, VT));
703 ReplaceNode(Node, SLLI);
704 return true;
705}
706
708 unsigned Opc;
709
710 if (Subtarget->hasVendorXTHeadBb())
711 Opc = RISCV::TH_EXT;
712 else if (Subtarget->hasVendorXAndesPerf())
713 Opc = RISCV::NDS_BFOS;
714 else if (Subtarget->hasVendorXqcibm())
715 Opc = RISCV::QC_EXT;
716 else
717 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
718 return false;
719
720 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
721 if (!N1C)
722 return false;
723
724 SDValue N0 = Node->getOperand(0);
725 if (!N0.hasOneUse())
726 return false;
727
728 auto BitfieldExtract = [&](SDValue N0, unsigned Msb, unsigned Lsb,
729 const SDLoc &DL, MVT VT) {
730 if (Opc == RISCV::QC_EXT) {
731 // QC.EXT X, width, shamt
732 // shamt is the same as Lsb
733 // width is the number of bits to extract from the Lsb
734 Msb = Msb - Lsb + 1;
735 }
736 return CurDAG->getMachineNode(Opc, DL, VT, N0.getOperand(0),
737 CurDAG->getTargetConstant(Msb, DL, VT),
738 CurDAG->getTargetConstant(Lsb, DL, VT));
739 };
740
741 SDLoc DL(Node);
742 MVT VT = Node->getSimpleValueType(0);
743 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
744 const unsigned RightShAmt = N1C->getZExtValue();
745 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
746 "Unexpected shift amount!");
747
748 // Transform (sra (shl X, C1) C2) with C1 < C2
749 // -> (SignedBitfieldExtract X, msb, lsb)
750 if (N0.getOpcode() == ISD::SHL) {
751 auto *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
752 if (!N01C)
753 return false;
754
755 const unsigned LeftShAmt = N01C->getZExtValue();
756 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
757 "Unexpected shift amount!");
758 // Make sure that this is a bitfield extraction (i.e., the shift-right
759 // amount can not be less than the left-shift).
760 if (LeftShAmt > RightShAmt)
761 return false;
762
763 const unsigned MsbPlusOne = VT.getSizeInBits() - LeftShAmt;
764 const unsigned Msb = MsbPlusOne - 1;
765 const unsigned Lsb = RightShAmt - LeftShAmt;
766
767 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
768 ReplaceNode(Node, Sbe);
769 return true;
770 }
771
772 // Transform (sra (sext_inreg X, _), C) ->
773 // (SignedBitfieldExtract X, msb, lsb)
774 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG) {
775 unsigned ExtSize =
776 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
777
778 // ExtSize of 32 should use sraiw via tablegen pattern.
779 if (ExtSize == 32)
780 return false;
781
782 const unsigned Msb = ExtSize - 1;
783 // If the shift-right amount is greater than Msb, it means that extracts
784 // the X[Msb] bit and sign-extend it.
785 const unsigned Lsb = RightShAmt > Msb ? Msb : RightShAmt;
786
787 SDNode *Sbe = BitfieldExtract(N0, Msb, Lsb, DL, VT);
788 ReplaceNode(Node, Sbe);
789 return true;
790 }
791
792 return false;
793}
794
796 // Only supported with XAndesPerf at the moment.
797 if (!Subtarget->hasVendorXAndesPerf())
798 return false;
799
800 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
801 if (!N1C)
802 return false;
803
804 SDValue N0 = Node->getOperand(0);
805 if (!N0.hasOneUse())
806 return false;
807
808 auto BitfieldInsert = [&](SDValue N0, unsigned Msb, unsigned Lsb,
809 const SDLoc &DL, MVT VT) {
810 unsigned Opc = RISCV::NDS_BFOS;
811 // If the Lsb is equal to the Msb, then the Lsb should be 0.
812 if (Lsb == Msb)
813 Lsb = 0;
814 return CurDAG->getMachineNode(Opc, DL, VT, N0.getOperand(0),
815 CurDAG->getTargetConstant(Lsb, DL, VT),
816 CurDAG->getTargetConstant(Msb, DL, VT));
817 };
818
819 SDLoc DL(Node);
820 MVT VT = Node->getSimpleValueType(0);
821 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
822 const unsigned RightShAmt = N1C->getZExtValue();
823 assert(RightShAmt > 0 && RightShAmt < VT.getSizeInBits() &&
824 "Unexpected shift amount!");
825
826 // Transform (sra (shl X, C1) C2) with C1 > C2
827 // -> (NDS.BFOS X, lsb, msb)
828 if (N0.getOpcode() == ISD::SHL) {
829 auto *N01C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
830 if (!N01C)
831 return false;
832
833 const unsigned LeftShAmt = N01C->getZExtValue();
834 assert(LeftShAmt > 0 && LeftShAmt < VT.getSizeInBits() &&
835 "Unexpected shift amount!");
836 // Make sure that this is a bitfield insertion (i.e., the shift-right
837 // amount should be less than the left-shift).
838 if (LeftShAmt <= RightShAmt)
839 return false;
840
841 const unsigned MsbPlusOne = VT.getSizeInBits() - RightShAmt;
842 const unsigned Msb = MsbPlusOne - 1;
843 const unsigned Lsb = LeftShAmt - RightShAmt;
844
845 SDNode *Sbi = BitfieldInsert(N0, Msb, Lsb, DL, VT);
846 ReplaceNode(Node, Sbi);
847 return true;
848 }
849
850 return false;
851}
852
854 const SDLoc &DL, MVT VT,
855 SDValue X, unsigned Msb,
856 unsigned Lsb) {
857 unsigned Opc;
858
859 if (Subtarget->hasVendorXTHeadBb()) {
860 Opc = RISCV::TH_EXTU;
861 } else if (Subtarget->hasVendorXAndesPerf()) {
862 Opc = RISCV::NDS_BFOZ;
863 } else if (Subtarget->hasVendorXqcibm()) {
864 Opc = RISCV::QC_EXTU;
865 // QC.EXTU X, width, shamt
866 // shamt is the same as Lsb
867 // width is the number of bits to extract from the Lsb
868 Msb = Msb - Lsb + 1;
869 } else {
870 // Only supported with XTHeadBb/XAndesPerf/Xqcibm at the moment.
871 return false;
872 }
873
874 SDNode *Ube = CurDAG->getMachineNode(Opc, DL, VT, X,
875 CurDAG->getTargetConstant(Msb, DL, VT),
876 CurDAG->getTargetConstant(Lsb, DL, VT));
877 ReplaceNode(Node, Ube);
878 return true;
879}
880
882 const SDLoc &DL, MVT VT,
883 SDValue X, unsigned Msb,
884 unsigned Lsb) {
885 // Only supported with XAndesPerf at the moment.
886 if (!Subtarget->hasVendorXAndesPerf())
887 return false;
888
889 unsigned Opc = RISCV::NDS_BFOZ;
890
891 // If the Lsb is equal to the Msb, then the Lsb should be 0.
892 if (Lsb == Msb)
893 Lsb = 0;
894 SDNode *Ubi = CurDAG->getMachineNode(Opc, DL, VT, X,
895 CurDAG->getTargetConstant(Lsb, DL, VT),
896 CurDAG->getTargetConstant(Msb, DL, VT));
897 ReplaceNode(Node, Ubi);
898 return true;
899}
900
902 // Target does not support indexed loads.
903 if (!Subtarget->hasVendorXTHeadMemIdx())
904 return false;
905
907 ISD::MemIndexedMode AM = Ld->getAddressingMode();
908 if (AM == ISD::UNINDEXED)
909 return false;
910
911 const ConstantSDNode *C = dyn_cast<ConstantSDNode>(Ld->getOffset());
912 if (!C)
913 return false;
914
915 EVT LoadVT = Ld->getMemoryVT();
916 assert((AM == ISD::PRE_INC || AM == ISD::POST_INC) &&
917 "Unexpected addressing mode");
918 bool IsPre = AM == ISD::PRE_INC;
919 bool IsPost = AM == ISD::POST_INC;
920 int64_t Offset = C->getSExtValue();
921
922 // The constants that can be encoded in the THeadMemIdx instructions
923 // are of the form (sign_extend(imm5) << imm2).
924 unsigned Shift;
925 for (Shift = 0; Shift < 4; Shift++)
926 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0))
927 break;
928
929 // Constant cannot be encoded.
930 if (Shift == 4)
931 return false;
932
933 bool IsZExt = (Ld->getExtensionType() == ISD::ZEXTLOAD);
934 unsigned Opcode;
935 if (LoadVT == MVT::i8 && IsPre)
936 Opcode = IsZExt ? RISCV::TH_LBUIB : RISCV::TH_LBIB;
937 else if (LoadVT == MVT::i8 && IsPost)
938 Opcode = IsZExt ? RISCV::TH_LBUIA : RISCV::TH_LBIA;
939 else if (LoadVT == MVT::i16 && IsPre)
940 Opcode = IsZExt ? RISCV::TH_LHUIB : RISCV::TH_LHIB;
941 else if (LoadVT == MVT::i16 && IsPost)
942 Opcode = IsZExt ? RISCV::TH_LHUIA : RISCV::TH_LHIA;
943 else if (LoadVT == MVT::i32 && IsPre)
944 Opcode = IsZExt ? RISCV::TH_LWUIB : RISCV::TH_LWIB;
945 else if (LoadVT == MVT::i32 && IsPost)
946 Opcode = IsZExt ? RISCV::TH_LWUIA : RISCV::TH_LWIA;
947 else if (LoadVT == MVT::i64 && IsPre)
948 Opcode = RISCV::TH_LDIB;
949 else if (LoadVT == MVT::i64 && IsPost)
950 Opcode = RISCV::TH_LDIA;
951 else
952 return false;
953
954 EVT Ty = Ld->getOffset().getValueType();
955 SDValue Ops[] = {
956 Ld->getBasePtr(),
957 CurDAG->getSignedTargetConstant(Offset >> Shift, SDLoc(Node), Ty),
958 CurDAG->getTargetConstant(Shift, SDLoc(Node), Ty), Ld->getChain()};
959 SDNode *New = CurDAG->getMachineNode(Opcode, SDLoc(Node), Ld->getValueType(0),
960 Ld->getValueType(1), MVT::Other, Ops);
961
962 MachineMemOperand *MemOp = cast<MemSDNode>(Node)->getMemOperand();
963 CurDAG->setNodeMemRefs(cast<MachineSDNode>(New), {MemOp});
964
965 ReplaceNode(Node, New);
966
967 return true;
968}
969
970static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT,
971 SDValue Lo, SDValue Hi) {
972 SDValue Ops[] = {
973 CurDAG->getTargetConstant(RISCV::GPRPairNoX0RegClassID, DL, MVT::i32), Lo,
974 CurDAG->getTargetConstant(RISCV::sub_gpr_even, DL, MVT::i32), Hi,
975 CurDAG->getTargetConstant(RISCV::sub_gpr_odd, DL, MVT::i32)};
976
977 return SDValue(
978 CurDAG->getMachineNode(TargetOpcode::REG_SEQUENCE, DL, VT, Ops), 0);
979}
980
981// Helper to extract Lo and Hi values from a GPR pair.
982static std::pair<SDValue, SDValue>
984 SDValue Lo =
985 CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_even, DL, MVT::i32, Pair);
986 SDValue Hi =
987 CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_odd, DL, MVT::i32, Pair);
988 return {Lo, Hi};
989}
990
991// Try to match WMACC pattern: ADDD where one operand pair comes from a
992// widening multiply (both results of UMUL_LOHI, SMUL_LOHI, or WMULSU).
994 assert(Node->getOpcode() == RISCVISD::ADDD && "Expected ADDD");
995
996 SDValue Op0Lo = Node->getOperand(0);
997 SDValue Op0Hi = Node->getOperand(1);
998 SDValue Op1Lo = Node->getOperand(2);
999 SDValue Op1Hi = Node->getOperand(3);
1000
1001 auto IsSupportedMulWithOneUse = [](SDValue Lo, SDValue Hi) {
1002 unsigned Opc = Lo.getOpcode();
1003 if (Opc != ISD::UMUL_LOHI && Opc != ISD::SMUL_LOHI &&
1004 Opc != RISCVISD::WMULSU)
1005 return false;
1006 return Lo.getNode() == Hi.getNode() && Lo.getResNo() == 0 &&
1007 Hi.getResNo() == 1 && Lo.hasOneUse() && Hi.hasOneUse();
1008 };
1009
1010 SDNode *MulNode = nullptr;
1011 SDValue AddLo, AddHi;
1012
1013 // Check if first operand pair is a supported multiply with single use.
1014 if (IsSupportedMulWithOneUse(Op0Lo, Op0Hi)) {
1015 MulNode = Op0Lo.getNode();
1016 AddLo = Op1Lo;
1017 AddHi = Op1Hi;
1018 }
1019 // ADDD is commutative. Check if second operand pair is a supported multiply
1020 // with single use.
1021 else if (IsSupportedMulWithOneUse(Op1Lo, Op1Hi)) {
1022 MulNode = Op1Lo.getNode();
1023 AddLo = Op0Lo;
1024 AddHi = Op0Hi;
1025 } else {
1026 return false;
1027 }
1028
1029 unsigned Opc;
1030 switch (MulNode->getOpcode()) {
1031 default:
1032 llvm_unreachable("Unexpected multiply opcode");
1033 case ISD::UMUL_LOHI:
1034 Opc = RISCV::WMACCU;
1035 break;
1036 case ISD::SMUL_LOHI:
1037 Opc = RISCV::WMACC;
1038 break;
1039 case RISCVISD::WMULSU:
1040 Opc = RISCV::WMACCSU;
1041 break;
1042 }
1043
1044 SDValue Acc = buildGPRPair(CurDAG, DL, MVT::Untyped, AddLo, AddHi);
1045
1046 // WMACC instruction format: rd, rs1, rs2 (rd is accumulator).
1047 SDValue M0 = MulNode->getOperand(0);
1048 SDValue M1 = MulNode->getOperand(1);
1049 MachineSDNode *New =
1050 CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Acc, M0, M1);
1051
1052 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
1053 ReplaceUses(SDValue(Node, 0), Lo);
1054 ReplaceUses(SDValue(Node, 1), Hi);
1055 CurDAG->RemoveDeadNode(Node);
1056 return true;
1057}
1058
1060 assert(TileNum <= 15 && "Invalid tile number");
1061 return RISCV::T0 + TileNum;
1062}
1063
1065 if (!Subtarget->hasVInstructions())
1066 return;
1067
1068 assert(Node->getOpcode() == ISD::INTRINSIC_VOID && "Unexpected opcode");
1069
1070 SDLoc DL(Node);
1071 unsigned IntNo = Node->getConstantOperandVal(1);
1072
1073 assert((IntNo == Intrinsic::riscv_sf_vc_x_se ||
1074 IntNo == Intrinsic::riscv_sf_vc_i_se) &&
1075 "Unexpected vsetvli intrinsic");
1076
1077 // imm, imm, imm, simm5/scalar, sew, log2lmul, vl
1078 unsigned Log2SEW = Log2_32(Node->getConstantOperandVal(6));
1079 SDValue SEWOp =
1080 CurDAG->getTargetConstant(Log2SEW, DL, Subtarget->getXLenVT());
1081 SmallVector<SDValue, 8> Operands = {Node->getOperand(2), Node->getOperand(3),
1082 Node->getOperand(4), Node->getOperand(5),
1083 Node->getOperand(8), SEWOp,
1084 Node->getOperand(0)};
1085
1086 unsigned Opcode;
1087 auto *LMulSDNode = cast<ConstantSDNode>(Node->getOperand(7));
1088 switch (LMulSDNode->getSExtValue()) {
1089 case 5:
1090 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF8
1091 : RISCV::PseudoSF_VC_I_SE_MF8;
1092 break;
1093 case 6:
1094 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF4
1095 : RISCV::PseudoSF_VC_I_SE_MF4;
1096 break;
1097 case 7:
1098 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_MF2
1099 : RISCV::PseudoSF_VC_I_SE_MF2;
1100 break;
1101 case 0:
1102 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M1
1103 : RISCV::PseudoSF_VC_I_SE_M1;
1104 break;
1105 case 1:
1106 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M2
1107 : RISCV::PseudoSF_VC_I_SE_M2;
1108 break;
1109 case 2:
1110 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M4
1111 : RISCV::PseudoSF_VC_I_SE_M4;
1112 break;
1113 case 3:
1114 Opcode = IntNo == Intrinsic::riscv_sf_vc_x_se ? RISCV::PseudoSF_VC_X_SE_M8
1115 : RISCV::PseudoSF_VC_I_SE_M8;
1116 break;
1117 }
1118
1119 ReplaceNode(Node, CurDAG->getMachineNode(
1120 Opcode, DL, Node->getSimpleValueType(0), Operands));
1121}
1122
1123static unsigned getSegInstNF(unsigned Intrinsic) {
1124#define INST_NF_CASE(NAME, NF) \
1125 case Intrinsic::riscv_##NAME##NF: \
1126 return NF;
1127#define INST_NF_CASE_MASK(NAME, NF) \
1128 case Intrinsic::riscv_##NAME##NF##_mask: \
1129 return NF;
1130#define INST_NF_CASE_FF(NAME, NF) \
1131 case Intrinsic::riscv_##NAME##NF##ff: \
1132 return NF;
1133#define INST_NF_CASE_FF_MASK(NAME, NF) \
1134 case Intrinsic::riscv_##NAME##NF##ff_mask: \
1135 return NF;
1136#define INST_ALL_NF_CASE_BASE(MACRO_NAME, NAME) \
1137 MACRO_NAME(NAME, 2) \
1138 MACRO_NAME(NAME, 3) \
1139 MACRO_NAME(NAME, 4) \
1140 MACRO_NAME(NAME, 5) \
1141 MACRO_NAME(NAME, 6) \
1142 MACRO_NAME(NAME, 7) \
1143 MACRO_NAME(NAME, 8)
1144#define INST_ALL_NF_CASE(NAME) \
1145 INST_ALL_NF_CASE_BASE(INST_NF_CASE, NAME) \
1146 INST_ALL_NF_CASE_BASE(INST_NF_CASE_MASK, NAME)
1147#define INST_ALL_NF_CASE_WITH_FF(NAME) \
1148 INST_ALL_NF_CASE(NAME) \
1149 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF, NAME) \
1150 INST_ALL_NF_CASE_BASE(INST_NF_CASE_FF_MASK, NAME)
1151 switch (Intrinsic) {
1152 default:
1153 llvm_unreachable("Unexpected segment load/store intrinsic");
1155 INST_ALL_NF_CASE(vlsseg)
1156 INST_ALL_NF_CASE(vloxseg)
1157 INST_ALL_NF_CASE(vluxseg)
1158 INST_ALL_NF_CASE(vsseg)
1159 INST_ALL_NF_CASE(vssseg)
1160 INST_ALL_NF_CASE(vsoxseg)
1161 INST_ALL_NF_CASE(vsuxseg)
1162 }
1163}
1164
1165static bool isApplicableToPLIOrPLUI(int Val) {
1166 // Check if the immediate is packed i8 or i10
1167 int16_t Bit31To16 = Val >> 16;
1168 int16_t Bit15To0 = Val;
1169 int8_t Bit15To8 = Bit15To0 >> 8;
1170 int8_t Bit7To0 = Val;
1171 if (Bit31To16 != Bit15To0)
1172 return false;
1173
1174 return isInt<10>(Bit15To0) || isShiftedInt<10, 6>(Bit15To0) ||
1175 Bit15To8 == Bit7To0;
1176}
1177
1179 // If we have a custom node, we have already selected.
1180 if (Node->isMachineOpcode()) {
1181 LLVM_DEBUG(dbgs() << "== "; Node->dump(CurDAG); dbgs() << "\n");
1182 Node->setNodeId(-1);
1183 return;
1184 }
1185
1186 // Instruction Selection not handled by the auto-generated tablegen selection
1187 // should be handled here.
1188 unsigned Opcode = Node->getOpcode();
1189 MVT XLenVT = Subtarget->getXLenVT();
1190 SDLoc DL(Node);
1191 MVT VT = Node->getSimpleValueType(0);
1192
1193 bool HasBitTest = Subtarget->hasBEXTILike();
1194
1195 switch (Opcode) {
1196 case ISD::Constant: {
1197 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1198 auto *ConstNode = cast<ConstantSDNode>(Node);
1199 if (ConstNode->isZero()) {
1200 SDValue New =
1201 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, RISCV::X0, VT);
1202 ReplaceNode(Node, New.getNode());
1203 return;
1204 }
1205 int64_t Imm = ConstNode->getSExtValue();
1206 // If only the lower 8 bits are used, try to convert this to a simm6 by
1207 // sign-extending bit 7. This is neutral without the C extension, and
1208 // allows C.LI to be used if C is present.
1212 // If the upper XLen-16 bits are not used, try to convert this to a simm12
1213 // by sign extending bit 15.
1214 else if (!isInt<16>(Imm) && isUInt<16>(Imm) &&
1217
1218 // If the upper XLen-16 bits are not used, the lower 2 bytes are the same,
1219 // and we can't use li, convert to an xlen splat so we can use pli.b.
1220 if (Subtarget->hasStdExtP() && !isInt<12>(Imm) &&
1221 (Imm & 0xff) == ((Imm >> 8) & 0xff) && hasAllHUsers(Node)) {
1222 // Splat the lower 16 bits to XLen. Sign extend for RV32.
1223 uint64_t Splat = Imm & 0xffff;
1224 Splat = (Splat << 16) | Splat;
1225 if (VT == MVT::i64)
1226 Imm = Splat << 32 | Splat;
1227 else
1229 } else {
1230 // If the upper 32-bits are not used try to convert this into a simm32 by
1231 // sign extending bit 32.
1234
1235 if (VT == MVT::i64 && !isInt<12>(Imm) && !isShiftedInt<20, 12>(Imm) &&
1236 Subtarget->hasStdExtP() && isApplicableToPLIOrPLUI(Imm) &&
1237 hasAllWUsers(Node)) {
1238 // If it's 4 packed 8-bit integers or 2 packed signed 16-bit integers,
1239 // we can simply copy lower 32 bits to higher 32 bits to make it able to
1240 // rematerialize to PLI_B or PLI_H
1241 Imm = ((uint64_t)Imm << 32) | (Imm & 0xFFFFFFFF);
1242 }
1243 }
1244
1245 ReplaceNode(Node, selectImm(CurDAG, DL, VT, Imm, *Subtarget).getNode());
1246 return;
1247 }
1248 case ISD::ConstantFP: {
1249 const APFloat &APF = cast<ConstantFPSDNode>(Node)->getValueAPF();
1250
1251 bool Is64Bit = Subtarget->is64Bit();
1252 bool HasZdinx = Subtarget->hasStdExtZdinx();
1253
1254 bool NegZeroF64 = APF.isNegZero() && VT == MVT::f64;
1255 SDValue Imm;
1256 // For +0.0 or f64 -0.0 we need to start from X0. For all others, we will
1257 // create an integer immediate.
1258 if (APF.isPosZero() || NegZeroF64) {
1259 if (VT == MVT::f64 && HasZdinx && !Is64Bit)
1260 Imm = CurDAG->getRegister(RISCV::X0_Pair, MVT::f64);
1261 else
1262 Imm = CurDAG->getRegister(RISCV::X0, XLenVT);
1263 } else {
1264 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
1265 *Subtarget);
1266 }
1267
1268 unsigned Opc;
1269 switch (VT.SimpleTy) {
1270 default:
1271 llvm_unreachable("Unexpected size");
1272 case MVT::bf16:
1273 assert(Subtarget->hasStdExtZfbfmin());
1274 Opc = RISCV::FMV_H_X;
1275 break;
1276 case MVT::f16:
1277 Opc = Subtarget->hasStdExtZhinxmin() ? RISCV::COPY : RISCV::FMV_H_X;
1278 break;
1279 case MVT::f32:
1280 Opc = Subtarget->hasStdExtZfinx() ? RISCV::COPY : RISCV::FMV_W_X;
1281 break;
1282 case MVT::f64:
1283 // For RV32, we can't move from a GPR, we need to convert instead. This
1284 // should only happen for +0.0 and -0.0.
1285 assert((Subtarget->is64Bit() || APF.isZero()) && "Unexpected constant");
1286 if (HasZdinx)
1287 Opc = RISCV::COPY;
1288 else
1289 Opc = Is64Bit ? RISCV::FMV_D_X : RISCV::FCVT_D_W;
1290 break;
1291 }
1292
1293 SDNode *Res;
1294 if (VT.SimpleTy == MVT::f16 && Opc == RISCV::COPY) {
1295 Res =
1296 CurDAG->getTargetExtractSubreg(RISCV::sub_16, DL, VT, Imm).getNode();
1297 } else if (VT.SimpleTy == MVT::f32 && Opc == RISCV::COPY) {
1298 Res =
1299 CurDAG->getTargetExtractSubreg(RISCV::sub_32, DL, VT, Imm).getNode();
1300 } else if (Opc == RISCV::FCVT_D_W_IN32X || Opc == RISCV::FCVT_D_W)
1301 Res = CurDAG->getMachineNode(
1302 Opc, DL, VT, Imm,
1303 CurDAG->getTargetConstant(RISCVFPRndMode::RNE, DL, XLenVT));
1304 else
1305 Res = CurDAG->getMachineNode(Opc, DL, VT, Imm);
1306
1307 // For f64 -0.0, we need to insert a fneg.d idiom.
1308 if (NegZeroF64) {
1309 Opc = RISCV::FSGNJN_D;
1310 if (HasZdinx)
1311 Opc = Is64Bit ? RISCV::FSGNJN_D_INX : RISCV::FSGNJN_D_IN32X;
1312 Res =
1313 CurDAG->getMachineNode(Opc, DL, VT, SDValue(Res, 0), SDValue(Res, 0));
1314 }
1315
1316 ReplaceNode(Node, Res);
1317 return;
1318 }
1319 case RISCVISD::BuildGPRPair:
1320 case RISCVISD::BuildPairF64:
1321 case RISCVISD::BuildPairGPRVec: {
1322 if (Opcode == RISCVISD::BuildPairF64 && !Subtarget->hasStdExtZdinx())
1323 break;
1324
1325 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::BuildPairF64) &&
1326 "BuildPairF64 only handled here on rv32i_zdinx");
1327
1328 SDValue N =
1329 buildGPRPair(CurDAG, DL, VT, Node->getOperand(0), Node->getOperand(1));
1330 ReplaceNode(Node, N.getNode());
1331 return;
1332 }
1333 case RISCVISD::SplitGPRPair:
1334 case RISCVISD::SplitF64:
1335 case RISCVISD::SplitGPRVec: {
1336 if (Subtarget->hasStdExtZdinx() || Opcode != RISCVISD::SplitF64) {
1337 assert((!Subtarget->is64Bit() || Opcode != RISCVISD::SplitF64) &&
1338 "SplitF64 only handled here on rv32i_zdinx");
1339
1340 if (!SDValue(Node, 0).use_empty()) {
1341 SDValue Lo = CurDAG->getTargetExtractSubreg(RISCV::sub_gpr_even, DL,
1342 Node->getValueType(0),
1343 Node->getOperand(0));
1344 ReplaceUses(SDValue(Node, 0), Lo);
1345 }
1346
1347 if (!SDValue(Node, 1).use_empty()) {
1348 SDValue Hi = CurDAG->getTargetExtractSubreg(
1349 RISCV::sub_gpr_odd, DL, Node->getValueType(1), Node->getOperand(0));
1350 ReplaceUses(SDValue(Node, 1), Hi);
1351 }
1352
1353 CurDAG->RemoveDeadNode(Node);
1354 return;
1355 }
1356
1357 if (!Subtarget->hasStdExtZfa())
1358 break;
1359 assert(Subtarget->hasStdExtD() && !Subtarget->is64Bit() &&
1360 "Unexpected subtarget");
1361
1362 // With Zfa, lower to fmv.x.w and fmvh.x.d.
1363 if (!SDValue(Node, 0).use_empty()) {
1364 SDNode *Lo = CurDAG->getMachineNode(RISCV::FMV_X_W_FPR64, DL, VT,
1365 Node->getOperand(0));
1366 ReplaceUses(SDValue(Node, 0), SDValue(Lo, 0));
1367 }
1368 if (!SDValue(Node, 1).use_empty()) {
1369 SDNode *Hi = CurDAG->getMachineNode(RISCV::FMVH_X_D, DL, VT,
1370 Node->getOperand(0));
1371 ReplaceUses(SDValue(Node, 1), SDValue(Hi, 0));
1372 }
1373
1374 CurDAG->RemoveDeadNode(Node);
1375 return;
1376 }
1377 case ISD::SHL: {
1378 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1379 if (!N1C)
1380 break;
1381 SDValue N0 = Node->getOperand(0);
1382 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
1384 break;
1385 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1386 unsigned ShAmt = N1C->getZExtValue();
1387 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1388 "Unexpected shift amount!");
1389 uint64_t Mask = N0.getConstantOperandVal(1);
1390
1391 if (isShiftedMask_64(Mask)) {
1392 unsigned XLen = Subtarget->getXLen();
1393 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
1394 unsigned TrailingZeros = llvm::countr_zero(Mask);
1395 if (ShAmt <= 32 && TrailingZeros > 0 && LeadingZeros == 32) {
1396 // Optimize (shl (and X, C2), C) -> (slli (srliw X, C3), C3+C)
1397 // where C2 has 32 leading zeros and C3 trailing zeros.
1398 SDNode *SRLIW = CurDAG->getMachineNode(
1399 RISCV::SRLIW, DL, VT, N0.getOperand(0),
1400 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
1401 SDNode *SLLI = CurDAG->getMachineNode(
1402 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1403 CurDAG->getTargetConstant(TrailingZeros + ShAmt, DL, VT));
1404 ReplaceNode(Node, SLLI);
1405 return;
1406 }
1407 if (TrailingZeros == 0 && LeadingZeros > ShAmt &&
1408 XLen - LeadingZeros > 11 && LeadingZeros != 32) {
1409 // Optimize (shl (and X, C2), C) -> (srli (slli X, C4), C4-C)
1410 // where C2 has C4 leading zeros and no trailing zeros.
1411 // This is profitable if the "and" was to be lowered to
1412 // (srli (slli X, C4), C4) and not (andi X, C2).
1413 // For "LeadingZeros == 32":
1414 // - with Zba it's just (slli.uw X, C)
1415 // - without Zba a tablegen pattern applies the very same
1416 // transform as we would have done here
1417 SDNode *SLLI = CurDAG->getMachineNode(
1418 RISCV::SLLI, DL, VT, N0.getOperand(0),
1419 CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1420 SDNode *SRLI = CurDAG->getMachineNode(
1421 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1422 CurDAG->getTargetConstant(LeadingZeros - ShAmt, DL, VT));
1423 ReplaceNode(Node, SRLI);
1424 return;
1425 }
1426 }
1427 break;
1428 }
1429 case ISD::SRL: {
1430 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1431 if (!N1C)
1432 break;
1433 SDValue N0 = Node->getOperand(0);
1434 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1435 break;
1436 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1437 unsigned ShAmt = N1C->getZExtValue();
1438 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1439 "Unexpected shift amount!");
1440 uint64_t Mask = N0.getConstantOperandVal(1);
1441
1442 // Optimize (srl (and X, C2), C) -> (slli (srliw X, C3), C3-C) where C2 has
1443 // 32 leading zeros and C3 trailing zeros.
1444 if (isShiftedMask_64(Mask) && N0.hasOneUse()) {
1445 unsigned XLen = Subtarget->getXLen();
1446 unsigned LeadingZeros = XLen - llvm::bit_width(Mask);
1447 unsigned TrailingZeros = llvm::countr_zero(Mask);
1448 if (LeadingZeros == 32 && TrailingZeros > ShAmt) {
1449 SDNode *SRLIW = CurDAG->getMachineNode(
1450 RISCV::SRLIW, DL, VT, N0.getOperand(0),
1451 CurDAG->getTargetConstant(TrailingZeros, DL, VT));
1452 SDNode *SLLI = CurDAG->getMachineNode(
1453 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1454 CurDAG->getTargetConstant(TrailingZeros - ShAmt, DL, VT));
1455 ReplaceNode(Node, SLLI);
1456 return;
1457 }
1458 }
1459
1460 // Optimize (srl (and X, C2), C) ->
1461 // (srli (slli X, (XLen-C3), (XLen-C3) + C)
1462 // Where C2 is a mask with C3 trailing ones.
1463 // Taking into account that the C2 may have had lower bits unset by
1464 // SimplifyDemandedBits. This avoids materializing the C2 immediate.
1465 // This pattern occurs when type legalizing right shifts for types with
1466 // less than XLen bits.
1467 Mask |= maskTrailingOnes<uint64_t>(ShAmt);
1468 if (!isMask_64(Mask))
1469 break;
1470 unsigned TrailingOnes = llvm::countr_one(Mask);
1471 if (ShAmt >= TrailingOnes)
1472 break;
1473 // If the mask has 32 trailing ones, use SRLI on RV32 or SRLIW on RV64.
1474 if (TrailingOnes == 32) {
1475 SDNode *SRLI = CurDAG->getMachineNode(
1476 Subtarget->is64Bit() ? RISCV::SRLIW : RISCV::SRLI, DL, VT,
1477 N0.getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
1478 ReplaceNode(Node, SRLI);
1479 return;
1480 }
1481
1482 // Only do the remaining transforms if the AND has one use.
1483 if (!N0.hasOneUse())
1484 break;
1485
1486 // If C2 is (1 << ShAmt) use bexti or th.tst if possible.
1487 if (HasBitTest && ShAmt + 1 == TrailingOnes) {
1488 SDNode *BEXTI = CurDAG->getMachineNode(
1489 Subtarget->hasStdExtZbs() ? RISCV::BEXTI : RISCV::TH_TST, DL, VT,
1490 N0.getOperand(0), CurDAG->getTargetConstant(ShAmt, DL, VT));
1491 ReplaceNode(Node, BEXTI);
1492 return;
1493 }
1494
1495 const unsigned Msb = TrailingOnes - 1;
1496 const unsigned Lsb = ShAmt;
1497 if (tryUnsignedBitfieldExtract(Node, DL, VT, N0.getOperand(0), Msb, Lsb))
1498 return;
1499
1500 unsigned LShAmt = Subtarget->getXLen() - TrailingOnes;
1501 SDNode *SLLI =
1502 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1503 CurDAG->getTargetConstant(LShAmt, DL, VT));
1504 SDNode *SRLI = CurDAG->getMachineNode(
1505 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1506 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1507 ReplaceNode(Node, SRLI);
1508 return;
1509 }
1510 case ISD::SRA: {
1512 return;
1513
1515 return;
1516
1517 // Optimize (sra (sext_inreg X, i16), C) ->
1518 // (srai (slli X, (XLen-16), (XLen-16) + C)
1519 // And (sra (sext_inreg X, i8), C) ->
1520 // (srai (slli X, (XLen-8), (XLen-8) + C)
1521 // This can occur when Zbb is enabled, which makes sext_inreg i16/i8 legal.
1522 // This transform matches the code we get without Zbb. The shifts are more
1523 // compressible, and this can help expose CSE opportunities in the sdiv by
1524 // constant optimization.
1525 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1526 if (!N1C)
1527 break;
1528 SDValue N0 = Node->getOperand(0);
1529 if (N0.getOpcode() != ISD::SIGN_EXTEND_INREG || !N0.hasOneUse())
1530 break;
1531 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1532 unsigned ShAmt = N1C->getZExtValue();
1533 assert(ShAmt > 0 && ShAmt < Subtarget->getXLen() &&
1534 "Unexpected shift amount!");
1535 unsigned ExtSize =
1536 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
1537 // ExtSize of 32 should use sraiw via tablegen pattern.
1538 if (ExtSize >= 32 || ShAmt >= ExtSize)
1539 break;
1540 unsigned LShAmt = Subtarget->getXLen() - ExtSize;
1541 SDNode *SLLI =
1542 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1543 CurDAG->getTargetConstant(LShAmt, DL, VT));
1544 SDNode *SRAI = CurDAG->getMachineNode(
1545 RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
1546 CurDAG->getTargetConstant(LShAmt + ShAmt, DL, VT));
1547 ReplaceNode(Node, SRAI);
1548 return;
1549 }
1551 // Optimize (sext_inreg (srl X, C), i8/i16) ->
1552 // (srai (slli X, XLen-ExtSize-C), XLen-ExtSize)
1553 // This is a bitfield extract pattern where we're extracting a signed
1554 // 8-bit or 16-bit field from position C.
1555 SDValue N0 = Node->getOperand(0);
1556 if (N0.getOpcode() != ISD::SRL || !N0.hasOneUse())
1557 break;
1558
1559 auto *ShAmtC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
1560 if (!ShAmtC)
1561 break;
1562
1563 unsigned ExtSize =
1564 cast<VTSDNode>(Node->getOperand(1))->getVT().getSizeInBits();
1565 unsigned ShAmt = ShAmtC->getZExtValue();
1566 unsigned XLen = Subtarget->getXLen();
1567
1568 // Only handle types less than 32, and make sure the shift amount is valid.
1569 if (ExtSize >= 32 || ShAmt >= XLen - ExtSize)
1570 break;
1571
1572 unsigned LShAmt = XLen - ExtSize - ShAmt;
1573 SDNode *SLLI =
1574 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1575 CurDAG->getTargetConstant(LShAmt, DL, VT));
1576 SDNode *SRAI = CurDAG->getMachineNode(
1577 RISCV::SRAI, DL, VT, SDValue(SLLI, 0),
1578 CurDAG->getTargetConstant(XLen - ExtSize, DL, VT));
1579 ReplaceNode(Node, SRAI);
1580 return;
1581 }
1582 case ISD::OR: {
1584 return;
1585
1586 break;
1587 }
1588 case ISD::XOR:
1590 return;
1591
1592 break;
1593 case ISD::AND: {
1594 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1595 if (!N1C)
1596 break;
1597 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
1598
1599 SDValue N0 = Node->getOperand(0);
1600
1601 bool LeftShift = N0.getOpcode() == ISD::SHL;
1602 if (LeftShift || N0.getOpcode() == ISD::SRL) {
1603 auto *C = dyn_cast<ConstantSDNode>(N0.getOperand(1));
1604 if (!C)
1605 break;
1606 unsigned C2 = C->getZExtValue();
1607 unsigned XLen = Subtarget->getXLen();
1608 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1609
1610 // Keep track of whether this is a c.andi. If we can't use c.andi, the
1611 // shift pair might offer more compression opportunities.
1612 // TODO: We could check for C extension here, but we don't have many lit
1613 // tests with the C extension enabled so not checking gets better
1614 // coverage.
1615 // TODO: What if ANDI faster than shift?
1616 bool IsCANDI = isInt<6>(N1C->getSExtValue());
1617
1618 uint64_t C1 = N1C->getZExtValue();
1619
1620 // Clear irrelevant bits in the mask.
1621 if (LeftShift)
1623 else
1624 C1 &= maskTrailingOnes<uint64_t>(XLen - C2);
1625
1626 // Some transforms should only be done if the shift has a single use or
1627 // the AND would become (srli (slli X, 32), 32)
1628 bool OneUseOrZExtW = N0.hasOneUse() || C1 == UINT64_C(0xFFFFFFFF);
1629
1630 SDValue X = N0.getOperand(0);
1631
1632 // Turn (and (srl x, c2) c1) -> (srli (slli x, c3-c2), c3) if c1 is a mask
1633 // with c3 leading zeros.
1634 if (!LeftShift && isMask_64(C1)) {
1635 unsigned Leading = XLen - llvm::bit_width(C1);
1636 if (C2 < Leading) {
1637 // If the number of leading zeros is C2+32 this can be SRLIW.
1638 if (C2 + 32 == Leading) {
1639 SDNode *SRLIW = CurDAG->getMachineNode(
1640 RISCV::SRLIW, DL, VT, X, CurDAG->getTargetConstant(C2, DL, VT));
1641 ReplaceNode(Node, SRLIW);
1642 return;
1643 }
1644
1645 // (and (srl (sexti32 Y), c2), c1) -> (srliw (sraiw Y, 31), c3 - 32)
1646 // if c1 is a mask with c3 leading zeros and c2 >= 32 and c3-c2==1.
1647 //
1648 // This pattern occurs when (i32 (srl (sra 31), c3 - 32)) is type
1649 // legalized and goes through DAG combine.
1650 if (C2 >= 32 && (Leading - C2) == 1 && N0.hasOneUse() &&
1651 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1652 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32) {
1653 SDNode *SRAIW =
1654 CurDAG->getMachineNode(RISCV::SRAIW, DL, VT, X.getOperand(0),
1655 CurDAG->getTargetConstant(31, DL, VT));
1656 SDNode *SRLIW = CurDAG->getMachineNode(
1657 RISCV::SRLIW, DL, VT, SDValue(SRAIW, 0),
1658 CurDAG->getTargetConstant(Leading - 32, DL, VT));
1659 ReplaceNode(Node, SRLIW);
1660 return;
1661 }
1662
1663 // Try to use an unsigned bitfield extract (e.g., th.extu) if
1664 // available.
1665 // Transform (and (srl x, C2), C1)
1666 // -> (<bfextract> x, msb, lsb)
1667 //
1668 // Make sure to keep this below the SRLIW cases, as we always want to
1669 // prefer the more common instruction.
1670 const unsigned Msb = llvm::bit_width(C1) + C2 - 1;
1671 const unsigned Lsb = C2;
1672 if (tryUnsignedBitfieldExtract(Node, DL, VT, X, Msb, Lsb))
1673 return;
1674
1675 // (srli (slli x, c3-c2), c3).
1676 // Skip if we could use (zext.w (sraiw X, C2)).
1677 bool Skip = Subtarget->hasStdExtZba() && Leading == 32 &&
1678 X.getOpcode() == ISD::SIGN_EXTEND_INREG &&
1679 cast<VTSDNode>(X.getOperand(1))->getVT() == MVT::i32;
1680 // Also Skip if we can use bexti or th.tst.
1681 Skip |= HasBitTest && Leading == XLen - 1;
1682 if (OneUseOrZExtW && !Skip) {
1683 SDNode *SLLI = CurDAG->getMachineNode(
1684 RISCV::SLLI, DL, VT, X,
1685 CurDAG->getTargetConstant(Leading - C2, DL, VT));
1686 SDNode *SRLI = CurDAG->getMachineNode(
1687 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1688 CurDAG->getTargetConstant(Leading, DL, VT));
1689 ReplaceNode(Node, SRLI);
1690 return;
1691 }
1692 }
1693 }
1694
1695 // Turn (and (shl x, c2), c1) -> (srli (slli c2+c3), c3) if c1 is a mask
1696 // shifted by c2 bits with c3 leading zeros.
1697 if (LeftShift && isShiftedMask_64(C1)) {
1698 unsigned Leading = XLen - llvm::bit_width(C1);
1699
1700 if (C2 + Leading < XLen &&
1701 C1 == (maskTrailingOnes<uint64_t>(XLen - (C2 + Leading)) << C2)) {
1702 // Use slli.uw when possible.
1703 if ((XLen - (C2 + Leading)) == 32 && Subtarget->hasStdExtZba()) {
1704 SDNode *SLLI_UW =
1705 CurDAG->getMachineNode(RISCV::SLLI_UW, DL, VT, X,
1706 CurDAG->getTargetConstant(C2, DL, VT));
1707 ReplaceNode(Node, SLLI_UW);
1708 return;
1709 }
1710
1711 // Try to use an unsigned bitfield insert (e.g., nds.bfoz) if
1712 // available.
1713 // Transform (and (shl x, c2), c1)
1714 // -> (<bfinsert> x, msb, lsb)
1715 // e.g.
1716 // (and (shl x, 12), 0x00fff000)
1717 // If XLen = 32 and C2 = 12, then
1718 // Msb = 32 - 8 - 1 = 23 and Lsb = 12
1719 const unsigned Msb = XLen - Leading - 1;
1720 const unsigned Lsb = C2;
1721 if (tryUnsignedBitfieldInsertInZero(Node, DL, VT, X, Msb, Lsb))
1722 return;
1723
1724 if (OneUseOrZExtW && !IsCANDI) {
1725 // (packh x0, X)
1726 if (Subtarget->hasStdExtZbkb() && C1 == 0xff00 && C2 == 8) {
1727 SDNode *PACKH = CurDAG->getMachineNode(
1728 RISCV::PACKH, DL, VT,
1729 CurDAG->getRegister(RISCV::X0, Subtarget->getXLenVT()), X);
1730 ReplaceNode(Node, PACKH);
1731 return;
1732 }
1733 // (srli (slli c2+c3), c3)
1734 SDNode *SLLI = CurDAG->getMachineNode(
1735 RISCV::SLLI, DL, VT, X,
1736 CurDAG->getTargetConstant(C2 + Leading, DL, VT));
1737 SDNode *SRLI = CurDAG->getMachineNode(
1738 RISCV::SRLI, DL, VT, SDValue(SLLI, 0),
1739 CurDAG->getTargetConstant(Leading, DL, VT));
1740 ReplaceNode(Node, SRLI);
1741 return;
1742 }
1743 }
1744 }
1745
1746 // Turn (and (shr x, c2), c1) -> (slli (srli x, c2+c3), c3) if c1 is a
1747 // shifted mask with c2 leading zeros and c3 trailing zeros.
1748 if (!LeftShift && isShiftedMask_64(C1)) {
1749 unsigned Leading = XLen - llvm::bit_width(C1);
1750 unsigned Trailing = llvm::countr_zero(C1);
1751 if (Leading == C2 && C2 + Trailing < XLen && OneUseOrZExtW &&
1752 !IsCANDI) {
1753 unsigned SrliOpc = RISCV::SRLI;
1754 // If the input is zexti32 we should use SRLIW.
1755 if (X.getOpcode() == ISD::AND &&
1756 isa<ConstantSDNode>(X.getOperand(1)) &&
1757 X.getConstantOperandVal(1) == UINT64_C(0xFFFFFFFF)) {
1758 SrliOpc = RISCV::SRLIW;
1759 X = X.getOperand(0);
1760 }
1761 SDNode *SRLI = CurDAG->getMachineNode(
1762 SrliOpc, DL, VT, X,
1763 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1764 SDNode *SLLI = CurDAG->getMachineNode(
1765 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1766 CurDAG->getTargetConstant(Trailing, DL, VT));
1767 ReplaceNode(Node, SLLI);
1768 return;
1769 }
1770 // If the leading zero count is C2+32, we can use SRLIW instead of SRLI.
1771 if (Leading > 32 && (Leading - 32) == C2 && C2 + Trailing < 32 &&
1772 OneUseOrZExtW && !IsCANDI) {
1773 SDNode *SRLIW = CurDAG->getMachineNode(
1774 RISCV::SRLIW, DL, VT, X,
1775 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1776 SDNode *SLLI = CurDAG->getMachineNode(
1777 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1778 CurDAG->getTargetConstant(Trailing, DL, VT));
1779 ReplaceNode(Node, SLLI);
1780 return;
1781 }
1782 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1783 if (Trailing > 0 && Leading + Trailing == 32 && C2 + Trailing < XLen &&
1784 OneUseOrZExtW && Subtarget->hasStdExtZba()) {
1785 SDNode *SRLI = CurDAG->getMachineNode(
1786 RISCV::SRLI, DL, VT, X,
1787 CurDAG->getTargetConstant(C2 + Trailing, DL, VT));
1788 SDNode *SLLI_UW = CurDAG->getMachineNode(
1789 RISCV::SLLI_UW, DL, VT, SDValue(SRLI, 0),
1790 CurDAG->getTargetConstant(Trailing, DL, VT));
1791 ReplaceNode(Node, SLLI_UW);
1792 return;
1793 }
1794 }
1795
1796 // Turn (and (shl x, c2), c1) -> (slli (srli x, c3-c2), c3) if c1 is a
1797 // shifted mask with no leading zeros and c3 trailing zeros.
1798 if (LeftShift && isShiftedMask_64(C1)) {
1799 unsigned Leading = XLen - llvm::bit_width(C1);
1800 unsigned Trailing = llvm::countr_zero(C1);
1801 if (Leading == 0 && C2 < Trailing && OneUseOrZExtW && !IsCANDI) {
1802 SDNode *SRLI = CurDAG->getMachineNode(
1803 RISCV::SRLI, DL, VT, X,
1804 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1805 SDNode *SLLI = CurDAG->getMachineNode(
1806 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1807 CurDAG->getTargetConstant(Trailing, DL, VT));
1808 ReplaceNode(Node, SLLI);
1809 return;
1810 }
1811 // If we have (32-C2) leading zeros, we can use SRLIW instead of SRLI.
1812 if (C2 < Trailing && Leading + C2 == 32 && OneUseOrZExtW && !IsCANDI) {
1813 SDNode *SRLIW = CurDAG->getMachineNode(
1814 RISCV::SRLIW, DL, VT, X,
1815 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1816 SDNode *SLLI = CurDAG->getMachineNode(
1817 RISCV::SLLI, DL, VT, SDValue(SRLIW, 0),
1818 CurDAG->getTargetConstant(Trailing, DL, VT));
1819 ReplaceNode(Node, SLLI);
1820 return;
1821 }
1822
1823 // If we have 32 bits in the mask, we can use SLLI_UW instead of SLLI.
1824 if (C2 < Trailing && Leading + Trailing == 32 && OneUseOrZExtW &&
1825 Subtarget->hasStdExtZba()) {
1826 SDNode *SRLI = CurDAG->getMachineNode(
1827 RISCV::SRLI, DL, VT, X,
1828 CurDAG->getTargetConstant(Trailing - C2, DL, VT));
1829 SDNode *SLLI_UW = CurDAG->getMachineNode(
1830 RISCV::SLLI_UW, DL, VT, SDValue(SRLI, 0),
1831 CurDAG->getTargetConstant(Trailing, DL, VT));
1832 ReplaceNode(Node, SLLI_UW);
1833 return;
1834 }
1835 }
1836 }
1837
1838 const uint64_t C1 = N1C->getZExtValue();
1839
1840 if (N0.getOpcode() == ISD::SRA && isa<ConstantSDNode>(N0.getOperand(1)) &&
1841 N0.hasOneUse()) {
1842 unsigned C2 = N0.getConstantOperandVal(1);
1843 unsigned XLen = Subtarget->getXLen();
1844 assert((C2 > 0 && C2 < XLen) && "Unexpected shift amount!");
1845
1846 SDValue X = N0.getOperand(0);
1847
1848 // Prefer SRAIW + ANDI when possible.
1849 bool Skip = C2 > 32 && isInt<12>(N1C->getSExtValue()) &&
1850 X.getOpcode() == ISD::SHL &&
1851 isa<ConstantSDNode>(X.getOperand(1)) &&
1852 X.getConstantOperandVal(1) == 32;
1853 // Turn (and (sra x, c2), c1) -> (srli (srai x, c2-c3), c3) if c1 is a
1854 // mask with c3 leading zeros and c2 is larger than c3.
1855 if (isMask_64(C1) && !Skip) {
1856 unsigned Leading = XLen - llvm::bit_width(C1);
1857 if (C2 > Leading) {
1858 SDNode *SRAI = CurDAG->getMachineNode(
1859 RISCV::SRAI, DL, VT, X,
1860 CurDAG->getTargetConstant(C2 - Leading, DL, VT));
1861 SDNode *SRLI = CurDAG->getMachineNode(
1862 RISCV::SRLI, DL, VT, SDValue(SRAI, 0),
1863 CurDAG->getTargetConstant(Leading, DL, VT));
1864 ReplaceNode(Node, SRLI);
1865 return;
1866 }
1867 }
1868
1869 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
1870 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
1871 // use (slli (srli (srai y, c2 - c3), c3 + c4), c4).
1872 if (isShiftedMask_64(C1) && !Skip) {
1873 unsigned Leading = XLen - llvm::bit_width(C1);
1874 unsigned Trailing = llvm::countr_zero(C1);
1875 if (C2 > Leading && Leading > 0 && Trailing > 0) {
1876 SDNode *SRAI = CurDAG->getMachineNode(
1877 RISCV::SRAI, DL, VT, N0.getOperand(0),
1878 CurDAG->getTargetConstant(C2 - Leading, DL, VT));
1879 SDNode *SRLI = CurDAG->getMachineNode(
1880 RISCV::SRLI, DL, VT, SDValue(SRAI, 0),
1881 CurDAG->getTargetConstant(Leading + Trailing, DL, VT));
1882 SDNode *SLLI = CurDAG->getMachineNode(
1883 RISCV::SLLI, DL, VT, SDValue(SRLI, 0),
1884 CurDAG->getTargetConstant(Trailing, DL, VT));
1885 ReplaceNode(Node, SLLI);
1886 return;
1887 }
1888 }
1889 }
1890
1891 // If C1 masks off the upper bits only (but can't be formed as an
1892 // ANDI), use an unsigned bitfield extract (e.g., th.extu), if
1893 // available.
1894 // Transform (and x, C1)
1895 // -> (<bfextract> x, msb, lsb)
1896 if (isMask_64(C1) && !isInt<12>(N1C->getSExtValue()) &&
1897 !(C1 == 0xffff && Subtarget->hasStdExtZbb()) &&
1898 !(C1 == 0xffffffff && Subtarget->hasStdExtZba())) {
1899 const unsigned Msb = llvm::bit_width(C1) - 1;
1900 if (tryUnsignedBitfieldExtract(Node, DL, VT, N0, Msb, 0))
1901 return;
1902 }
1903
1905 return;
1906
1907 break;
1908 }
1909 case ISD::MUL: {
1910 // Special case for calculating (mul (and X, C2), C1) where the full product
1911 // fits in XLen bits. We can shift X left by the number of leading zeros in
1912 // C2 and shift C1 left by XLen-lzcnt(C2). This will ensure the final
1913 // product has XLen trailing zeros, putting it in the output of MULHU. This
1914 // can avoid materializing a constant in a register for C2.
1915
1916 // RHS should be a constant.
1917 auto *N1C = dyn_cast<ConstantSDNode>(Node->getOperand(1));
1918 if (!N1C || !N1C->hasOneUse())
1919 break;
1920
1921 // LHS should be an AND with constant.
1922 SDValue N0 = Node->getOperand(0);
1923 if (N0.getOpcode() != ISD::AND || !isa<ConstantSDNode>(N0.getOperand(1)))
1924 break;
1925
1926 uint64_t C2 = N0.getConstantOperandVal(1);
1927
1928 // Constant should be a mask.
1929 if (!isMask_64(C2))
1930 break;
1931
1932 // If this can be an ANDI or ZEXT.H, don't do this if the ANDI/ZEXT has
1933 // multiple users or the constant is a simm12. This prevents inserting a
1934 // shift and still have uses of the AND/ZEXT. Shifting a simm12 will likely
1935 // make it more costly to materialize. Otherwise, using a SLLI might allow
1936 // it to be compressed.
1937 bool IsANDIOrZExt =
1938 isInt<12>(C2) ||
1939 (C2 == UINT64_C(0xFFFF) && Subtarget->hasStdExtZbb());
1940 // With XTHeadBb, we can use TH.EXTU.
1941 IsANDIOrZExt |= C2 == UINT64_C(0xFFFF) && Subtarget->hasVendorXTHeadBb();
1942 if (IsANDIOrZExt && (isInt<12>(N1C->getSExtValue()) || !N0.hasOneUse()))
1943 break;
1944 // If this can be a ZEXT.w, don't do this if the ZEXT has multiple users or
1945 // the constant is a simm32.
1946 bool IsZExtW = C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasStdExtZba();
1947 // With XTHeadBb, we can use TH.EXTU.
1948 IsZExtW |= C2 == UINT64_C(0xFFFFFFFF) && Subtarget->hasVendorXTHeadBb();
1949 if (IsZExtW && (isInt<32>(N1C->getSExtValue()) || !N0.hasOneUse()))
1950 break;
1951
1952 // We need to shift left the AND input and C1 by a total of XLen bits.
1953
1954 // How far left do we need to shift the AND input?
1955 unsigned XLen = Subtarget->getXLen();
1956 unsigned LeadingZeros = XLen - llvm::bit_width(C2);
1957
1958 // The constant gets shifted by the remaining amount unless that would
1959 // shift bits out.
1960 uint64_t C1 = N1C->getZExtValue();
1961 unsigned ConstantShift = XLen - LeadingZeros;
1962 if (ConstantShift > (XLen - llvm::bit_width(C1)))
1963 break;
1964
1965 uint64_t ShiftedC1 = C1 << ConstantShift;
1966 // If this RV32, we need to sign extend the constant.
1967 if (XLen == 32)
1968 ShiftedC1 = SignExtend64<32>(ShiftedC1);
1969
1970 // Create (mulhu (slli X, lzcnt(C2)), C1 << (XLen - lzcnt(C2))).
1971 SDNode *Imm = selectImm(CurDAG, DL, VT, ShiftedC1, *Subtarget).getNode();
1972 SDNode *SLLI =
1973 CurDAG->getMachineNode(RISCV::SLLI, DL, VT, N0.getOperand(0),
1974 CurDAG->getTargetConstant(LeadingZeros, DL, VT));
1975 SDNode *MULHU = CurDAG->getMachineNode(RISCV::MULHU, DL, VT,
1976 SDValue(SLLI, 0), SDValue(Imm, 0));
1977 ReplaceNode(Node, MULHU);
1978 return;
1979 }
1980 case ISD::SMUL_LOHI:
1981 case ISD::UMUL_LOHI:
1982 case RISCVISD::WMULSU:
1983 case RISCVISD::WADD:
1984 case RISCVISD::WSUB:
1985 case RISCVISD::WADDU:
1986 case RISCVISD::WSUBU: {
1987 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
1988 "Unexpected opcode");
1989
1990 unsigned Opc;
1991 switch (Node->getOpcode()) {
1992 default:
1993 llvm_unreachable("Unexpected opcode");
1994 case ISD::SMUL_LOHI:
1995 Opc = RISCV::WMUL;
1996 break;
1997 case ISD::UMUL_LOHI:
1998 Opc = RISCV::WMULU;
1999 break;
2000 case RISCVISD::WMULSU:
2001 Opc = RISCV::WMULSU;
2002 break;
2003 case RISCVISD::WADD:
2004 Opc = RISCV::WADD;
2005 break;
2006 case RISCVISD::WSUB:
2007 Opc = RISCV::WSUB;
2008 break;
2009 case RISCVISD::WADDU:
2010 Opc = RISCV::WADDU;
2011 break;
2012 case RISCVISD::WSUBU:
2013 Opc = RISCV::WSUBU;
2014 break;
2015 }
2016
2017 SDNode *Result = CurDAG->getMachineNode(
2018 Opc, DL, MVT::Untyped, Node->getOperand(0), Node->getOperand(1));
2019
2020 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(Result, 0));
2021 ReplaceUses(SDValue(Node, 0), Lo);
2022 ReplaceUses(SDValue(Node, 1), Hi);
2023 CurDAG->RemoveDeadNode(Node);
2024 return;
2025 }
2026 case RISCVISD::WSLL:
2027 case RISCVISD::WSLA: {
2028 // Custom select WSLL/WSLA for RV32P.
2029 assert(Subtarget->hasStdExtP() && !Subtarget->is64Bit() && VT == MVT::i32 &&
2030 "Unexpected opcode");
2031
2032 bool IsSigned = Node->getOpcode() == RISCVISD::WSLA;
2033
2034 SDValue ShAmt = Node->getOperand(1);
2035
2036 unsigned Opc;
2037
2038 auto *ShAmtC = dyn_cast<ConstantSDNode>(ShAmt);
2039 if (ShAmtC && ShAmtC->getZExtValue() < 64) {
2040 Opc = IsSigned ? RISCV::WSLAI : RISCV::WSLLI;
2041 ShAmt = CurDAG->getTargetConstant(ShAmtC->getZExtValue(), DL, XLenVT);
2042 } else {
2043 Opc = IsSigned ? RISCV::WSLA : RISCV::WSLL;
2044 }
2045
2046 SDNode *WShift = CurDAG->getMachineNode(Opc, DL, MVT::Untyped,
2047 Node->getOperand(0), ShAmt);
2048
2049 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(WShift, 0));
2050 ReplaceUses(SDValue(Node, 0), Lo);
2051 ReplaceUses(SDValue(Node, 1), Hi);
2052 CurDAG->RemoveDeadNode(Node);
2053 return;
2054 }
2055 case ISD::LOAD: {
2056 if (tryIndexedLoad(Node))
2057 return;
2058
2059 if (Subtarget->hasVendorXCVmem() && !Subtarget->is64Bit()) {
2060 // We match post-incrementing load here
2062 if (Load->getAddressingMode() != ISD::POST_INC)
2063 break;
2064
2065 SDValue Chain = Node->getOperand(0);
2066 SDValue Base = Node->getOperand(1);
2067 SDValue Offset = Node->getOperand(2);
2068
2069 bool Simm12 = false;
2070 bool SignExtend = Load->getExtensionType() == ISD::SEXTLOAD;
2071
2072 if (auto ConstantOffset = dyn_cast<ConstantSDNode>(Offset)) {
2073 int ConstantVal = ConstantOffset->getSExtValue();
2074 Simm12 = isInt<12>(ConstantVal);
2075 if (Simm12)
2076 Offset = CurDAG->getSignedTargetConstant(ConstantVal, SDLoc(Offset),
2077 Offset.getValueType());
2078 }
2079
2080 unsigned Opcode = 0;
2081 switch (Load->getMemoryVT().getSimpleVT().SimpleTy) {
2082 case MVT::i8:
2083 if (Simm12 && SignExtend)
2084 Opcode = RISCV::CV_LB_ri_inc;
2085 else if (Simm12 && !SignExtend)
2086 Opcode = RISCV::CV_LBU_ri_inc;
2087 else if (!Simm12 && SignExtend)
2088 Opcode = RISCV::CV_LB_rr_inc;
2089 else
2090 Opcode = RISCV::CV_LBU_rr_inc;
2091 break;
2092 case MVT::i16:
2093 if (Simm12 && SignExtend)
2094 Opcode = RISCV::CV_LH_ri_inc;
2095 else if (Simm12 && !SignExtend)
2096 Opcode = RISCV::CV_LHU_ri_inc;
2097 else if (!Simm12 && SignExtend)
2098 Opcode = RISCV::CV_LH_rr_inc;
2099 else
2100 Opcode = RISCV::CV_LHU_rr_inc;
2101 break;
2102 case MVT::i32:
2103 if (Simm12)
2104 Opcode = RISCV::CV_LW_ri_inc;
2105 else
2106 Opcode = RISCV::CV_LW_rr_inc;
2107 break;
2108 default:
2109 break;
2110 }
2111 if (!Opcode)
2112 break;
2113
2114 ReplaceNode(Node, CurDAG->getMachineNode(Opcode, DL, XLenVT, XLenVT,
2115 Chain.getSimpleValueType(), Base,
2116 Offset, Chain));
2117 return;
2118 }
2119 break;
2120 }
2121 case RISCVISD::LD_RV32: {
2122 assert(Subtarget->hasStdExtZilsd() && "LD_RV32 is only used with Zilsd");
2123
2125 SDValue Chain = Node->getOperand(0);
2126 SDValue Addr = Node->getOperand(1);
2128
2129 SDValue Ops[] = {Base, Offset, Chain};
2130 MachineSDNode *New = CurDAG->getMachineNode(
2131 RISCV::LD_RV32, DL, {MVT::Untyped, MVT::Other}, Ops);
2132 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
2133 CurDAG->setNodeMemRefs(New, {cast<MemSDNode>(Node)->getMemOperand()});
2134 ReplaceUses(SDValue(Node, 0), Lo);
2135 ReplaceUses(SDValue(Node, 1), Hi);
2136 ReplaceUses(SDValue(Node, 2), SDValue(New, 1));
2137 CurDAG->RemoveDeadNode(Node);
2138 return;
2139 }
2140 case RISCVISD::SD_RV32: {
2142 SDValue Chain = Node->getOperand(0);
2143 SDValue Addr = Node->getOperand(3);
2145
2146 SDValue Lo = Node->getOperand(1);
2147 SDValue Hi = Node->getOperand(2);
2148
2149 SDValue RegPair;
2150 // Peephole to use X0_Pair for storing zero.
2152 RegPair = CurDAG->getRegister(RISCV::X0_Pair, MVT::Untyped);
2153 } else {
2154 RegPair = buildGPRPair(CurDAG, DL, MVT::Untyped, Lo, Hi);
2155 }
2156
2157 MachineSDNode *New = CurDAG->getMachineNode(RISCV::SD_RV32, DL, MVT::Other,
2158 {RegPair, Base, Offset, Chain});
2159 CurDAG->setNodeMemRefs(New, {cast<MemSDNode>(Node)->getMemOperand()});
2160 ReplaceUses(SDValue(Node, 0), SDValue(New, 0));
2161 CurDAG->RemoveDeadNode(Node);
2162 return;
2163 }
2164 case RISCVISD::MQWACC:
2165 case RISCVISD::MQRWACC:
2166 case RISCVISD::WMACC:
2167 case RISCVISD::WMACCU:
2168 case RISCVISD::WMACCSU:
2169 case RISCVISD::PM2WADDA:
2170 case RISCVISD::PM2WADDA_X:
2171 case RISCVISD::PM2WADDAU:
2172 case RISCVISD::PM2WSUBA:
2173 case RISCVISD::PM2WSUBA_X:
2174 case RISCVISD::PM2WADDASU:
2175 case RISCVISD::WADDAU:
2176 case RISCVISD::WSUBAU:
2177 case RISCVISD::WADDA:
2178 case RISCVISD::WSUBA: {
2179 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2180 "Unexpected opcode");
2181
2182 SDValue Op0 = buildGPRPair(CurDAG, DL, MVT::Untyped, Node->getOperand(0),
2183 Node->getOperand(1));
2184 unsigned Opc;
2185 switch (Opcode) {
2186 default:
2187 llvm_unreachable("Unexpected opcode");
2188 case RISCVISD::MQWACC:
2189 Opc = RISCV::MQWACC;
2190 break;
2191 case RISCVISD::MQRWACC:
2192 Opc = RISCV::MQRWACC;
2193 break;
2194 case RISCVISD::WMACC:
2195 Opc = RISCV::WMACC;
2196 break;
2197 case RISCVISD::WMACCU:
2198 Opc = RISCV::WMACCU;
2199 break;
2200 case RISCVISD::WMACCSU:
2201 Opc = RISCV::WMACCSU;
2202 break;
2203 case RISCVISD::PM2WADDA:
2204 Opc = RISCV::PM2WADDA_H;
2205 break;
2206 case RISCVISD::PM2WADDA_X:
2207 Opc = RISCV::PM2WADDA_HX;
2208 break;
2209 case RISCVISD::PM2WADDAU:
2210 Opc = RISCV::PM2WADDAU_H;
2211 break;
2212 case RISCVISD::PM2WSUBA:
2213 Opc = RISCV::PM2WSUBA_H;
2214 break;
2215 case RISCVISD::PM2WSUBA_X:
2216 Opc = RISCV::PM2WSUBA_HX;
2217 break;
2218 case RISCVISD::PM2WADDASU:
2219 Opc = RISCV::PM2WADDASU_H;
2220 break;
2221 case RISCVISD::WADDAU:
2222 Opc = RISCV::WADDAU;
2223 break;
2224 case RISCVISD::WSUBAU:
2225 Opc = RISCV::WSUBAU;
2226 break;
2227 case RISCVISD::WADDA:
2228 Opc = RISCV::WADDA;
2229 break;
2230 case RISCVISD::WSUBA:
2231 Opc = RISCV::WSUBA;
2232 break;
2233 }
2234 MachineSDNode *New = CurDAG->getMachineNode(
2235 Opc, DL, MVT::Untyped, Op0, Node->getOperand(2), Node->getOperand(3));
2236 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
2237 ReplaceUses(SDValue(Node, 0), Lo);
2238 ReplaceUses(SDValue(Node, 1), Hi);
2239 CurDAG->RemoveDeadNode(Node);
2240 return;
2241 }
2242 case RISCVISD::PM2WADD:
2243 case RISCVISD::PM2WADDU:
2244 case RISCVISD::PM2WADDSU:
2245 case RISCVISD::PM2WADD_X:
2246 case RISCVISD::PM2WSUB:
2247 case RISCVISD::PM2WSUB_X: {
2248 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2249 "Unexpected opcode");
2250 unsigned Opc;
2251 switch (Opcode) {
2252 default:
2253 llvm_unreachable("Unexpected opcode");
2254 case RISCVISD::PM2WADD:
2255 Opc = RISCV::PM2WADD_H;
2256 break;
2257 case RISCVISD::PM2WADDU:
2258 Opc = RISCV::PM2WADDU_H;
2259 break;
2260 case RISCVISD::PM2WADDSU:
2261 Opc = RISCV::PM2WADDSU_H;
2262 break;
2263 case RISCVISD::PM2WADD_X:
2264 Opc = RISCV::PM2WADD_HX;
2265 break;
2266 case RISCVISD::PM2WSUB:
2267 Opc = RISCV::PM2WSUB_H;
2268 break;
2269 case RISCVISD::PM2WSUB_X:
2270 Opc = RISCV::PM2WSUB_HX;
2271 break;
2272 }
2273 MachineSDNode *New = CurDAG->getMachineNode(
2274 Opc, DL, MVT::Untyped, Node->getOperand(0), Node->getOperand(1));
2275 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
2276 ReplaceUses(SDValue(Node, 0), Lo);
2277 ReplaceUses(SDValue(Node, 1), Hi);
2278 CurDAG->RemoveDeadNode(Node);
2279 return;
2280 }
2281 case RISCVISD::ADDD:
2282 // Try to match WMACC pattern: ADDD where one operand pair comes from a
2283 // widening multiply.
2285 return;
2286
2287 // Fall through to regular ADDD selection.
2288 [[fallthrough]];
2289 case RISCVISD::SUBD: {
2290 assert(!Subtarget->is64Bit() && Subtarget->hasStdExtP() &&
2291 "Unexpected opcode");
2292
2293 SDValue Op0Lo = Node->getOperand(0);
2294 SDValue Op0Hi = Node->getOperand(1);
2295
2296 SDValue Op0;
2297 if (isNullConstant(Op0Lo) && isNullConstant(Op0Hi)) {
2298 Op0 = CurDAG->getRegister(RISCV::X0_Pair, MVT::Untyped);
2299 } else {
2300 Op0 = buildGPRPair(CurDAG, DL, MVT::Untyped, Op0Lo, Op0Hi);
2301 }
2302
2303 SDValue Op1Lo = Node->getOperand(2);
2304 SDValue Op1Hi = Node->getOperand(3);
2305 SDValue Op1 = buildGPRPair(CurDAG, DL, MVT::Untyped, Op1Lo, Op1Hi);
2306
2307 unsigned Opc = Opcode == RISCVISD::ADDD ? RISCV::ADDD : RISCV::SUBD;
2308 MachineSDNode *New =
2309 CurDAG->getMachineNode(Opc, DL, MVT::Untyped, Op0, Op1);
2310
2311 auto [Lo, Hi] = extractGPRPair(CurDAG, DL, SDValue(New, 0));
2312 ReplaceUses(SDValue(Node, 0), Lo);
2313 ReplaceUses(SDValue(Node, 1), Hi);
2314 CurDAG->RemoveDeadNode(Node);
2315 return;
2316 }
2318 unsigned IntNo = Node->getConstantOperandVal(0);
2319 switch (IntNo) {
2320 // By default we do not custom select any intrinsic.
2321 default:
2322 break;
2323 case Intrinsic::riscv_vmsgeu:
2324 case Intrinsic::riscv_vmsge: {
2325 SDValue Src1 = Node->getOperand(1);
2326 SDValue Src2 = Node->getOperand(2);
2327 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu;
2328 bool IsCmpConstant = false;
2329 bool IsCmpMinimum = false;
2330 // Only custom select scalar second operand.
2331 if (Src2.getValueType() != XLenVT)
2332 break;
2333 // Small constants are handled with patterns.
2334 int64_t CVal = 0;
2335 MVT Src1VT = Src1.getSimpleValueType();
2336 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
2337 IsCmpConstant = true;
2338 CVal = C->getSExtValue();
2339 if (CVal >= -15 && CVal <= 16) {
2340 if (!IsUnsigned || CVal != 0)
2341 break;
2342 IsCmpMinimum = true;
2343 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2344 Src1VT.getScalarSizeInBits())
2345 .getSExtValue()) {
2346 IsCmpMinimum = true;
2347 }
2348 }
2349 unsigned VMSLTOpcode, VMNANDOpcode, VMSetOpcode, VMSGTOpcode;
2350 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
2351 default:
2352 llvm_unreachable("Unexpected LMUL!");
2353#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2354 case RISCVVType::lmulenum: \
2355 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2356 : RISCV::PseudoVMSLT_VX_##suffix; \
2357 VMSGTOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix \
2358 : RISCV::PseudoVMSGT_VX_##suffix; \
2359 break;
2360 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2361 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2362 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2363 CASE_VMSLT_OPCODES(LMUL_1, M1)
2364 CASE_VMSLT_OPCODES(LMUL_2, M2)
2365 CASE_VMSLT_OPCODES(LMUL_4, M4)
2366 CASE_VMSLT_OPCODES(LMUL_8, M8)
2367#undef CASE_VMSLT_OPCODES
2368 }
2369 // Mask operations use the LMUL from the mask type.
2370 switch (RISCVTargetLowering::getLMUL(VT)) {
2371 default:
2372 llvm_unreachable("Unexpected LMUL!");
2373#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix) \
2374 case RISCVVType::lmulenum: \
2375 VMNANDOpcode = RISCV::PseudoVMNAND_MM_##suffix; \
2376 VMSetOpcode = RISCV::PseudoVMSET_M_##suffix; \
2377 break;
2378 CASE_VMNAND_VMSET_OPCODES(LMUL_F8, B64)
2379 CASE_VMNAND_VMSET_OPCODES(LMUL_F4, B32)
2380 CASE_VMNAND_VMSET_OPCODES(LMUL_F2, B16)
2381 CASE_VMNAND_VMSET_OPCODES(LMUL_1, B8)
2382 CASE_VMNAND_VMSET_OPCODES(LMUL_2, B4)
2383 CASE_VMNAND_VMSET_OPCODES(LMUL_4, B2)
2384 CASE_VMNAND_VMSET_OPCODES(LMUL_8, B1)
2385#undef CASE_VMNAND_VMSET_OPCODES
2386 }
2387 SDValue SEW = CurDAG->getTargetConstant(
2388 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
2389 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
2390 SDValue VL;
2391 selectVLOp(Node->getOperand(3), VL);
2392
2393 // If vmsge(u) with minimum value, expand it to vmset.
2394 if (IsCmpMinimum) {
2396 CurDAG->getMachineNode(VMSetOpcode, DL, VT, VL, MaskSEW));
2397 return;
2398 }
2399
2400 if (IsCmpConstant) {
2401 SDValue Imm =
2402 selectImm(CurDAG, SDLoc(Src2), XLenVT, CVal - 1, *Subtarget);
2403
2404 ReplaceNode(Node, CurDAG->getMachineNode(VMSGTOpcode, DL, VT,
2405 {Src1, Imm, VL, SEW}));
2406 return;
2407 }
2408
2409 // Expand to
2410 // vmslt{u}.vx vd, va, x; vmnand.mm vd, vd, vd
2411 SDValue Cmp = SDValue(
2412 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
2413 0);
2414 ReplaceNode(Node, CurDAG->getMachineNode(VMNANDOpcode, DL, VT,
2415 {Cmp, Cmp, VL, MaskSEW}));
2416 return;
2417 }
2418 case Intrinsic::riscv_vmsgeu_mask:
2419 case Intrinsic::riscv_vmsge_mask: {
2420 SDValue Src1 = Node->getOperand(2);
2421 SDValue Src2 = Node->getOperand(3);
2422 bool IsUnsigned = IntNo == Intrinsic::riscv_vmsgeu_mask;
2423 bool IsCmpConstant = false;
2424 bool IsCmpMinimum = false;
2425 // Only custom select scalar second operand.
2426 if (Src2.getValueType() != XLenVT)
2427 break;
2428 // Small constants are handled with patterns.
2429 MVT Src1VT = Src1.getSimpleValueType();
2430 int64_t CVal = 0;
2431 if (auto *C = dyn_cast<ConstantSDNode>(Src2)) {
2432 IsCmpConstant = true;
2433 CVal = C->getSExtValue();
2434 if (CVal >= -15 && CVal <= 16) {
2435 if (!IsUnsigned || CVal != 0)
2436 break;
2437 IsCmpMinimum = true;
2438 } else if (!IsUnsigned && CVal == APInt::getSignedMinValue(
2439 Src1VT.getScalarSizeInBits())
2440 .getSExtValue()) {
2441 IsCmpMinimum = true;
2442 }
2443 }
2444 unsigned VMSLTOpcode, VMSLTMaskOpcode, VMXOROpcode, VMANDNOpcode,
2445 VMOROpcode, VMSGTMaskOpcode;
2446 switch (RISCVTargetLowering::getLMUL(Src1VT)) {
2447 default:
2448 llvm_unreachable("Unexpected LMUL!");
2449#define CASE_VMSLT_OPCODES(lmulenum, suffix) \
2450 case RISCVVType::lmulenum: \
2451 VMSLTOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix \
2452 : RISCV::PseudoVMSLT_VX_##suffix; \
2453 VMSLTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSLTU_VX_##suffix##_MASK \
2454 : RISCV::PseudoVMSLT_VX_##suffix##_MASK; \
2455 VMSGTMaskOpcode = IsUnsigned ? RISCV::PseudoVMSGTU_VX_##suffix##_MASK \
2456 : RISCV::PseudoVMSGT_VX_##suffix##_MASK; \
2457 break;
2458 CASE_VMSLT_OPCODES(LMUL_F8, MF8)
2459 CASE_VMSLT_OPCODES(LMUL_F4, MF4)
2460 CASE_VMSLT_OPCODES(LMUL_F2, MF2)
2461 CASE_VMSLT_OPCODES(LMUL_1, M1)
2462 CASE_VMSLT_OPCODES(LMUL_2, M2)
2463 CASE_VMSLT_OPCODES(LMUL_4, M4)
2464 CASE_VMSLT_OPCODES(LMUL_8, M8)
2465#undef CASE_VMSLT_OPCODES
2466 }
2467 // Mask operations use the LMUL from the mask type.
2468 switch (RISCVTargetLowering::getLMUL(VT)) {
2469 default:
2470 llvm_unreachable("Unexpected LMUL!");
2471#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix) \
2472 case RISCVVType::lmulenum: \
2473 VMXOROpcode = RISCV::PseudoVMXOR_MM_##suffix; \
2474 VMANDNOpcode = RISCV::PseudoVMANDN_MM_##suffix; \
2475 VMOROpcode = RISCV::PseudoVMOR_MM_##suffix; \
2476 break;
2477 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F8, B64)
2478 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F4, B32)
2479 CASE_VMXOR_VMANDN_VMOR_OPCODES(LMUL_F2, B16)
2484#undef CASE_VMXOR_VMANDN_VMOR_OPCODES
2485 }
2486 SDValue SEW = CurDAG->getTargetConstant(
2487 Log2_32(Src1VT.getScalarSizeInBits()), DL, XLenVT);
2488 SDValue MaskSEW = CurDAG->getTargetConstant(0, DL, XLenVT);
2489 SDValue VL;
2490 selectVLOp(Node->getOperand(5), VL);
2491 SDValue MaskedOff = Node->getOperand(1);
2492 SDValue Mask = Node->getOperand(4);
2493
2494 // If vmsge(u) with minimum value, expand it to vmor mask, maskedoff.
2495 if (IsCmpMinimum) {
2496 // We don't need vmor if the MaskedOff and the Mask are the same
2497 // value.
2498 if (Mask == MaskedOff) {
2499 ReplaceUses(Node, Mask.getNode());
2500 return;
2501 }
2503 CurDAG->getMachineNode(VMOROpcode, DL, VT,
2504 {Mask, MaskedOff, VL, MaskSEW}));
2505 return;
2506 }
2507
2508 // If the MaskedOff value and the Mask are the same value use
2509 // vmslt{u}.vx vt, va, x; vmandn.mm vd, vd, vt
2510 // This avoids needing to copy v0 to vd before starting the next sequence.
2511 if (Mask == MaskedOff) {
2512 SDValue Cmp = SDValue(
2513 CurDAG->getMachineNode(VMSLTOpcode, DL, VT, {Src1, Src2, VL, SEW}),
2514 0);
2515 ReplaceNode(Node, CurDAG->getMachineNode(VMANDNOpcode, DL, VT,
2516 {Mask, Cmp, VL, MaskSEW}));
2517 return;
2518 }
2519
2520 SDValue PolicyOp =
2521 CurDAG->getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT);
2522
2523 if (IsCmpConstant) {
2524 SDValue Imm =
2525 selectImm(CurDAG, SDLoc(Src2), XLenVT, CVal - 1, *Subtarget);
2526
2527 ReplaceNode(Node, CurDAG->getMachineNode(
2528 VMSGTMaskOpcode, DL, VT,
2529 {MaskedOff, Src1, Imm, Mask, VL, SEW, PolicyOp}));
2530 return;
2531 }
2532
2533 // Otherwise use
2534 // vmslt{u}.vx vd, va, x, v0.t; vmxor.mm vd, vd, v0
2535 // The result is mask undisturbed.
2536 // We use the same instructions to emulate mask agnostic behavior, because
2537 // the agnostic result can be either undisturbed or all 1.
2538 SDValue Cmp = SDValue(CurDAG->getMachineNode(VMSLTMaskOpcode, DL, VT,
2539 {MaskedOff, Src1, Src2, Mask,
2540 VL, SEW, PolicyOp}),
2541 0);
2542 // vmxor.mm vd, vd, v0 is used to update active value.
2543 ReplaceNode(Node, CurDAG->getMachineNode(VMXOROpcode, DL, VT,
2544 {Cmp, Mask, VL, MaskSEW}));
2545 return;
2546 }
2547 case Intrinsic::riscv_vsetvli:
2548 case Intrinsic::riscv_vsetvlimax:
2549 return selectVSETVLI(Node);
2550 case Intrinsic::riscv_sf_vsettnt:
2551 case Intrinsic::riscv_sf_vsettm:
2552 case Intrinsic::riscv_sf_vsettk:
2553 return selectXSfmmVSET(Node);
2554 }
2555 break;
2556 }
2558 unsigned IntNo = Node->getConstantOperandVal(1);
2559 switch (IntNo) {
2560 // By default we do not custom select any intrinsic.
2561 default:
2562 break;
2563 case Intrinsic::riscv_vlseg2:
2564 case Intrinsic::riscv_vlseg3:
2565 case Intrinsic::riscv_vlseg4:
2566 case Intrinsic::riscv_vlseg5:
2567 case Intrinsic::riscv_vlseg6:
2568 case Intrinsic::riscv_vlseg7:
2569 case Intrinsic::riscv_vlseg8: {
2570 selectVLSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2571 /*IsStrided*/ false);
2572 return;
2573 }
2574 case Intrinsic::riscv_vlseg2_mask:
2575 case Intrinsic::riscv_vlseg3_mask:
2576 case Intrinsic::riscv_vlseg4_mask:
2577 case Intrinsic::riscv_vlseg5_mask:
2578 case Intrinsic::riscv_vlseg6_mask:
2579 case Intrinsic::riscv_vlseg7_mask:
2580 case Intrinsic::riscv_vlseg8_mask: {
2581 selectVLSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2582 /*IsStrided*/ false);
2583 return;
2584 }
2585 case Intrinsic::riscv_vlsseg2:
2586 case Intrinsic::riscv_vlsseg3:
2587 case Intrinsic::riscv_vlsseg4:
2588 case Intrinsic::riscv_vlsseg5:
2589 case Intrinsic::riscv_vlsseg6:
2590 case Intrinsic::riscv_vlsseg7:
2591 case Intrinsic::riscv_vlsseg8: {
2592 selectVLSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2593 /*IsStrided*/ true);
2594 return;
2595 }
2596 case Intrinsic::riscv_vlsseg2_mask:
2597 case Intrinsic::riscv_vlsseg3_mask:
2598 case Intrinsic::riscv_vlsseg4_mask:
2599 case Intrinsic::riscv_vlsseg5_mask:
2600 case Intrinsic::riscv_vlsseg6_mask:
2601 case Intrinsic::riscv_vlsseg7_mask:
2602 case Intrinsic::riscv_vlsseg8_mask: {
2603 selectVLSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2604 /*IsStrided*/ true);
2605 return;
2606 }
2607 case Intrinsic::riscv_vloxseg2:
2608 case Intrinsic::riscv_vloxseg3:
2609 case Intrinsic::riscv_vloxseg4:
2610 case Intrinsic::riscv_vloxseg5:
2611 case Intrinsic::riscv_vloxseg6:
2612 case Intrinsic::riscv_vloxseg7:
2613 case Intrinsic::riscv_vloxseg8:
2614 selectVLXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2615 /*IsOrdered*/ true);
2616 return;
2617 case Intrinsic::riscv_vluxseg2:
2618 case Intrinsic::riscv_vluxseg3:
2619 case Intrinsic::riscv_vluxseg4:
2620 case Intrinsic::riscv_vluxseg5:
2621 case Intrinsic::riscv_vluxseg6:
2622 case Intrinsic::riscv_vluxseg7:
2623 case Intrinsic::riscv_vluxseg8:
2624 selectVLXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2625 /*IsOrdered*/ false);
2626 return;
2627 case Intrinsic::riscv_vloxseg2_mask:
2628 case Intrinsic::riscv_vloxseg3_mask:
2629 case Intrinsic::riscv_vloxseg4_mask:
2630 case Intrinsic::riscv_vloxseg5_mask:
2631 case Intrinsic::riscv_vloxseg6_mask:
2632 case Intrinsic::riscv_vloxseg7_mask:
2633 case Intrinsic::riscv_vloxseg8_mask:
2634 selectVLXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2635 /*IsOrdered*/ true);
2636 return;
2637 case Intrinsic::riscv_vluxseg2_mask:
2638 case Intrinsic::riscv_vluxseg3_mask:
2639 case Intrinsic::riscv_vluxseg4_mask:
2640 case Intrinsic::riscv_vluxseg5_mask:
2641 case Intrinsic::riscv_vluxseg6_mask:
2642 case Intrinsic::riscv_vluxseg7_mask:
2643 case Intrinsic::riscv_vluxseg8_mask:
2644 selectVLXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2645 /*IsOrdered*/ false);
2646 return;
2647 case Intrinsic::riscv_vlseg8ff:
2648 case Intrinsic::riscv_vlseg7ff:
2649 case Intrinsic::riscv_vlseg6ff:
2650 case Intrinsic::riscv_vlseg5ff:
2651 case Intrinsic::riscv_vlseg4ff:
2652 case Intrinsic::riscv_vlseg3ff:
2653 case Intrinsic::riscv_vlseg2ff: {
2654 selectVLSEGFF(Node, getSegInstNF(IntNo), /*IsMasked*/ false);
2655 return;
2656 }
2657 case Intrinsic::riscv_vlseg8ff_mask:
2658 case Intrinsic::riscv_vlseg7ff_mask:
2659 case Intrinsic::riscv_vlseg6ff_mask:
2660 case Intrinsic::riscv_vlseg5ff_mask:
2661 case Intrinsic::riscv_vlseg4ff_mask:
2662 case Intrinsic::riscv_vlseg3ff_mask:
2663 case Intrinsic::riscv_vlseg2ff_mask: {
2664 selectVLSEGFF(Node, getSegInstNF(IntNo), /*IsMasked*/ true);
2665 return;
2666 }
2667 case Intrinsic::riscv_vloxei:
2668 case Intrinsic::riscv_vloxei_mask:
2669 case Intrinsic::riscv_vluxei:
2670 case Intrinsic::riscv_vluxei_mask: {
2671 bool IsMasked = IntNo == Intrinsic::riscv_vloxei_mask ||
2672 IntNo == Intrinsic::riscv_vluxei_mask;
2673 bool IsOrdered = IntNo == Intrinsic::riscv_vloxei ||
2674 IntNo == Intrinsic::riscv_vloxei_mask;
2675
2676 MVT VT = Node->getSimpleValueType(0);
2677 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2678
2679 unsigned CurOp = 2;
2681 Operands.push_back(Node->getOperand(CurOp++));
2682
2683 MVT IndexVT;
2684 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2685 /*IsStridedOrIndexed*/ true, Operands,
2686 /*IsLoad=*/true, &IndexVT);
2687
2689 "Element count mismatch");
2690
2693 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
2694 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2695 reportFatalUsageError("The V extension does not support EEW=64 for "
2696 "index values when XLEN=32");
2697 }
2698 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
2699 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
2700 static_cast<unsigned>(IndexLMUL));
2702 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
2703
2704 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
2705
2707 return;
2708 }
2709 case Intrinsic::riscv_vlm:
2710 case Intrinsic::riscv_vle:
2711 case Intrinsic::riscv_vle_mask:
2712 case Intrinsic::riscv_vlse:
2713 case Intrinsic::riscv_vlse_mask: {
2714 bool IsMasked = IntNo == Intrinsic::riscv_vle_mask ||
2715 IntNo == Intrinsic::riscv_vlse_mask;
2716 bool IsStrided =
2717 IntNo == Intrinsic::riscv_vlse || IntNo == Intrinsic::riscv_vlse_mask;
2718
2719 MVT VT = Node->getSimpleValueType(0);
2720 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2721
2722 // The riscv_vlm intrinsic are always tail agnostic and no passthru
2723 // operand at the IR level. In pseudos, they have both policy and
2724 // passthru operand. The passthru operand is needed to track the
2725 // "tail undefined" state, and the policy is there just for
2726 // for consistency - it will always be "don't care" for the
2727 // unmasked form.
2728 bool HasPassthruOperand = IntNo != Intrinsic::riscv_vlm;
2729 unsigned CurOp = 2;
2731 if (HasPassthruOperand)
2732 Operands.push_back(Node->getOperand(CurOp++));
2733 else {
2734 // We eagerly lower to implicit_def (instead of undef), as we
2735 // otherwise fail to select nodes such as: nxv1i1 = undef
2736 SDNode *Passthru =
2737 CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT);
2738 Operands.push_back(SDValue(Passthru, 0));
2739 }
2740 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
2741 Operands, /*IsLoad=*/true);
2742
2744 const RISCV::VLEPseudo *P =
2745 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
2746 static_cast<unsigned>(LMUL));
2748 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
2749
2750 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
2751
2753 return;
2754 }
2755 case Intrinsic::riscv_vleff:
2756 case Intrinsic::riscv_vleff_mask: {
2757 bool IsMasked = IntNo == Intrinsic::riscv_vleff_mask;
2758
2759 MVT VT = Node->getSimpleValueType(0);
2760 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2761
2762 unsigned CurOp = 2;
2764 Operands.push_back(Node->getOperand(CurOp++));
2765 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2766 /*IsStridedOrIndexed*/ false, Operands,
2767 /*IsLoad=*/true);
2768
2770 const RISCV::VLEPseudo *P =
2771 RISCV::getVLEPseudo(IsMasked, /*Strided*/ false, /*FF*/ true,
2772 Log2SEW, static_cast<unsigned>(LMUL));
2773 MachineSDNode *Load = CurDAG->getMachineNode(
2774 P->Pseudo, DL, Node->getVTList(), Operands);
2775 CurDAG->setNodeMemRefs(Load, {cast<MemSDNode>(Node)->getMemOperand()});
2776
2778 return;
2779 }
2780 case Intrinsic::riscv_nds_vln:
2781 case Intrinsic::riscv_nds_vln_mask:
2782 case Intrinsic::riscv_nds_vlnu:
2783 case Intrinsic::riscv_nds_vlnu_mask: {
2784 bool IsMasked = IntNo == Intrinsic::riscv_nds_vln_mask ||
2785 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2786 bool IsUnsigned = IntNo == Intrinsic::riscv_nds_vlnu ||
2787 IntNo == Intrinsic::riscv_nds_vlnu_mask;
2788
2789 MVT VT = Node->getSimpleValueType(0);
2790 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2791 unsigned CurOp = 2;
2793
2794 Operands.push_back(Node->getOperand(CurOp++));
2795 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2796 /*IsStridedOrIndexed=*/false, Operands,
2797 /*IsLoad=*/true);
2798
2800 const RISCV::NDSVLNPseudo *P = RISCV::getNDSVLNPseudo(
2801 IsMasked, IsUnsigned, Log2SEW, static_cast<unsigned>(LMUL));
2803 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
2804
2805 if (auto *MemOp = dyn_cast<MemSDNode>(Node))
2806 CurDAG->setNodeMemRefs(Load, {MemOp->getMemOperand()});
2807
2809 return;
2810 }
2811 }
2812 break;
2813 }
2814 case ISD::INTRINSIC_VOID: {
2815 unsigned IntNo = Node->getConstantOperandVal(1);
2816 switch (IntNo) {
2817 case Intrinsic::riscv_vsseg2:
2818 case Intrinsic::riscv_vsseg3:
2819 case Intrinsic::riscv_vsseg4:
2820 case Intrinsic::riscv_vsseg5:
2821 case Intrinsic::riscv_vsseg6:
2822 case Intrinsic::riscv_vsseg7:
2823 case Intrinsic::riscv_vsseg8: {
2824 selectVSSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2825 /*IsStrided*/ false);
2826 return;
2827 }
2828 case Intrinsic::riscv_vsseg2_mask:
2829 case Intrinsic::riscv_vsseg3_mask:
2830 case Intrinsic::riscv_vsseg4_mask:
2831 case Intrinsic::riscv_vsseg5_mask:
2832 case Intrinsic::riscv_vsseg6_mask:
2833 case Intrinsic::riscv_vsseg7_mask:
2834 case Intrinsic::riscv_vsseg8_mask: {
2835 selectVSSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2836 /*IsStrided*/ false);
2837 return;
2838 }
2839 case Intrinsic::riscv_vssseg2:
2840 case Intrinsic::riscv_vssseg3:
2841 case Intrinsic::riscv_vssseg4:
2842 case Intrinsic::riscv_vssseg5:
2843 case Intrinsic::riscv_vssseg6:
2844 case Intrinsic::riscv_vssseg7:
2845 case Intrinsic::riscv_vssseg8: {
2846 selectVSSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2847 /*IsStrided*/ true);
2848 return;
2849 }
2850 case Intrinsic::riscv_vssseg2_mask:
2851 case Intrinsic::riscv_vssseg3_mask:
2852 case Intrinsic::riscv_vssseg4_mask:
2853 case Intrinsic::riscv_vssseg5_mask:
2854 case Intrinsic::riscv_vssseg6_mask:
2855 case Intrinsic::riscv_vssseg7_mask:
2856 case Intrinsic::riscv_vssseg8_mask: {
2857 selectVSSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2858 /*IsStrided*/ true);
2859 return;
2860 }
2861 case Intrinsic::riscv_vsoxseg2:
2862 case Intrinsic::riscv_vsoxseg3:
2863 case Intrinsic::riscv_vsoxseg4:
2864 case Intrinsic::riscv_vsoxseg5:
2865 case Intrinsic::riscv_vsoxseg6:
2866 case Intrinsic::riscv_vsoxseg7:
2867 case Intrinsic::riscv_vsoxseg8:
2868 selectVSXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2869 /*IsOrdered*/ true);
2870 return;
2871 case Intrinsic::riscv_vsuxseg2:
2872 case Intrinsic::riscv_vsuxseg3:
2873 case Intrinsic::riscv_vsuxseg4:
2874 case Intrinsic::riscv_vsuxseg5:
2875 case Intrinsic::riscv_vsuxseg6:
2876 case Intrinsic::riscv_vsuxseg7:
2877 case Intrinsic::riscv_vsuxseg8:
2878 selectVSXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ false,
2879 /*IsOrdered*/ false);
2880 return;
2881 case Intrinsic::riscv_vsoxseg2_mask:
2882 case Intrinsic::riscv_vsoxseg3_mask:
2883 case Intrinsic::riscv_vsoxseg4_mask:
2884 case Intrinsic::riscv_vsoxseg5_mask:
2885 case Intrinsic::riscv_vsoxseg6_mask:
2886 case Intrinsic::riscv_vsoxseg7_mask:
2887 case Intrinsic::riscv_vsoxseg8_mask:
2888 selectVSXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2889 /*IsOrdered*/ true);
2890 return;
2891 case Intrinsic::riscv_vsuxseg2_mask:
2892 case Intrinsic::riscv_vsuxseg3_mask:
2893 case Intrinsic::riscv_vsuxseg4_mask:
2894 case Intrinsic::riscv_vsuxseg5_mask:
2895 case Intrinsic::riscv_vsuxseg6_mask:
2896 case Intrinsic::riscv_vsuxseg7_mask:
2897 case Intrinsic::riscv_vsuxseg8_mask:
2898 selectVSXSEG(Node, getSegInstNF(IntNo), /*IsMasked*/ true,
2899 /*IsOrdered*/ false);
2900 return;
2901 case Intrinsic::riscv_vsoxei:
2902 case Intrinsic::riscv_vsoxei_mask:
2903 case Intrinsic::riscv_vsuxei:
2904 case Intrinsic::riscv_vsuxei_mask: {
2905 bool IsMasked = IntNo == Intrinsic::riscv_vsoxei_mask ||
2906 IntNo == Intrinsic::riscv_vsuxei_mask;
2907 bool IsOrdered = IntNo == Intrinsic::riscv_vsoxei ||
2908 IntNo == Intrinsic::riscv_vsoxei_mask;
2909
2910 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
2911 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2912
2913 unsigned CurOp = 2;
2915 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
2916
2917 MVT IndexVT;
2918 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked,
2919 /*IsStridedOrIndexed*/ true, Operands,
2920 /*IsLoad=*/false, &IndexVT);
2921
2923 "Element count mismatch");
2924
2927 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
2928 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
2929 reportFatalUsageError("The V extension does not support EEW=64 for "
2930 "index values when XLEN=32");
2931 }
2932 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
2933 IsMasked, IsOrdered, IndexLog2EEW,
2934 static_cast<unsigned>(LMUL), static_cast<unsigned>(IndexLMUL));
2936 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
2937
2938 CurDAG->setNodeMemRefs(Store, {cast<MemSDNode>(Node)->getMemOperand()});
2939
2941 return;
2942 }
2943 case Intrinsic::riscv_vsm:
2944 case Intrinsic::riscv_vse:
2945 case Intrinsic::riscv_vse_mask:
2946 case Intrinsic::riscv_vsse:
2947 case Intrinsic::riscv_vsse_mask: {
2948 bool IsMasked = IntNo == Intrinsic::riscv_vse_mask ||
2949 IntNo == Intrinsic::riscv_vsse_mask;
2950 bool IsStrided =
2951 IntNo == Intrinsic::riscv_vsse || IntNo == Intrinsic::riscv_vsse_mask;
2952
2953 MVT VT = Node->getOperand(2)->getSimpleValueType(0);
2954 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
2955
2956 unsigned CurOp = 2;
2958 Operands.push_back(Node->getOperand(CurOp++)); // Store value.
2959
2960 addVectorLoadStoreOperands(Node, Log2SEW, DL, CurOp, IsMasked, IsStrided,
2961 Operands);
2962
2964 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
2965 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
2967 CurDAG->getMachineNode(P->Pseudo, DL, Node->getVTList(), Operands);
2968 CurDAG->setNodeMemRefs(Store, {cast<MemSDNode>(Node)->getMemOperand()});
2969
2971 return;
2972 }
2973 case Intrinsic::riscv_sf_vc_x_se:
2974 case Intrinsic::riscv_sf_vc_i_se:
2976 return;
2977 case Intrinsic::riscv_sf_vlte8:
2978 case Intrinsic::riscv_sf_vlte16:
2979 case Intrinsic::riscv_sf_vlte32:
2980 case Intrinsic::riscv_sf_vlte64: {
2981 unsigned Log2SEW;
2982 unsigned PseudoInst;
2983 switch (IntNo) {
2984 case Intrinsic::riscv_sf_vlte8:
2985 PseudoInst = RISCV::PseudoSF_VLTE8;
2986 Log2SEW = 3;
2987 break;
2988 case Intrinsic::riscv_sf_vlte16:
2989 PseudoInst = RISCV::PseudoSF_VLTE16;
2990 Log2SEW = 4;
2991 break;
2992 case Intrinsic::riscv_sf_vlte32:
2993 PseudoInst = RISCV::PseudoSF_VLTE32;
2994 Log2SEW = 5;
2995 break;
2996 case Intrinsic::riscv_sf_vlte64:
2997 PseudoInst = RISCV::PseudoSF_VLTE64;
2998 Log2SEW = 6;
2999 break;
3000 }
3001
3002 SDValue SEWOp = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
3003 SDValue TWidenOp = CurDAG->getTargetConstant(1, DL, XLenVT);
3004 SDValue Operands[] = {Node->getOperand(2),
3005 Node->getOperand(3),
3006 Node->getOperand(4),
3007 SEWOp,
3008 TWidenOp,
3009 Node->getOperand(0)};
3010
3011 MachineSDNode *TileLoad =
3012 CurDAG->getMachineNode(PseudoInst, DL, Node->getVTList(), Operands);
3013 CurDAG->setNodeMemRefs(TileLoad,
3014 {cast<MemSDNode>(Node)->getMemOperand()});
3015
3016 ReplaceNode(Node, TileLoad);
3017 return;
3018 }
3019 case Intrinsic::riscv_sf_mm_s_s:
3020 case Intrinsic::riscv_sf_mm_s_u:
3021 case Intrinsic::riscv_sf_mm_u_s:
3022 case Intrinsic::riscv_sf_mm_u_u:
3023 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
3024 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
3025 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
3026 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3027 case Intrinsic::riscv_sf_mm_f_f: {
3028 bool HasFRM = false;
3029 unsigned PseudoInst;
3030 switch (IntNo) {
3031 case Intrinsic::riscv_sf_mm_s_s:
3032 PseudoInst = RISCV::PseudoSF_MM_S_S;
3033 break;
3034 case Intrinsic::riscv_sf_mm_s_u:
3035 PseudoInst = RISCV::PseudoSF_MM_S_U;
3036 break;
3037 case Intrinsic::riscv_sf_mm_u_s:
3038 PseudoInst = RISCV::PseudoSF_MM_U_S;
3039 break;
3040 case Intrinsic::riscv_sf_mm_u_u:
3041 PseudoInst = RISCV::PseudoSF_MM_U_U;
3042 break;
3043 case Intrinsic::riscv_sf_mm_e5m2_e5m2:
3044 PseudoInst = RISCV::PseudoSF_MM_E5M2_E5M2;
3045 HasFRM = true;
3046 break;
3047 case Intrinsic::riscv_sf_mm_e5m2_e4m3:
3048 PseudoInst = RISCV::PseudoSF_MM_E5M2_E4M3;
3049 HasFRM = true;
3050 break;
3051 case Intrinsic::riscv_sf_mm_e4m3_e5m2:
3052 PseudoInst = RISCV::PseudoSF_MM_E4M3_E5M2;
3053 HasFRM = true;
3054 break;
3055 case Intrinsic::riscv_sf_mm_e4m3_e4m3:
3056 PseudoInst = RISCV::PseudoSF_MM_E4M3_E4M3;
3057 HasFRM = true;
3058 break;
3059 case Intrinsic::riscv_sf_mm_f_f:
3060 if (Node->getOperand(3).getValueType().getScalarType() == MVT::bf16)
3061 PseudoInst = RISCV::PseudoSF_MM_F_F_ALT;
3062 else
3063 PseudoInst = RISCV::PseudoSF_MM_F_F;
3064 HasFRM = true;
3065 break;
3066 }
3067 uint64_t TileNum = Node->getConstantOperandVal(2);
3068 SDValue Op1 = Node->getOperand(3);
3069 SDValue Op2 = Node->getOperand(4);
3070 MVT VT = Op1->getSimpleValueType(0);
3071 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
3072 SDValue TmOp = Node->getOperand(5);
3073 SDValue TnOp = Node->getOperand(6);
3074 SDValue TkOp = Node->getOperand(7);
3075 SDValue TWidenOp = Node->getOperand(8);
3076 SDValue Chain = Node->getOperand(0);
3077
3078 // sf.mm.f.f with sew=32, twiden=2 is invalid
3079 if (IntNo == Intrinsic::riscv_sf_mm_f_f && Log2SEW == 5 &&
3080 TWidenOp->getAsZExtVal() == 2)
3081 reportFatalUsageError("sf.mm.f.f doesn't support (sew=32, twiden=2)");
3082
3084 {CurDAG->getRegister(getTileReg(TileNum), XLenVT), Op1, Op2});
3085 if (HasFRM)
3086 Operands.push_back(
3087 CurDAG->getTargetConstant(RISCVFPRndMode::DYN, DL, XLenVT));
3088 Operands.append({TmOp, TnOp, TkOp,
3089 CurDAG->getTargetConstant(Log2SEW, DL, XLenVT), TWidenOp,
3090 Chain});
3091
3092 auto *NewNode =
3093 CurDAG->getMachineNode(PseudoInst, DL, Node->getVTList(), Operands);
3094
3095 ReplaceNode(Node, NewNode);
3096 return;
3097 }
3098 case Intrinsic::riscv_sf_vtzero_t: {
3099 uint64_t TileNum = Node->getConstantOperandVal(2);
3100 SDValue Tm = Node->getOperand(3);
3101 SDValue Tn = Node->getOperand(4);
3102 SDValue Log2SEW = Node->getOperand(5);
3103 SDValue TWiden = Node->getOperand(6);
3104 SDValue Chain = Node->getOperand(0);
3105 auto *NewNode = CurDAG->getMachineNode(
3106 RISCV::PseudoSF_VTZERO_T, DL, Node->getVTList(),
3107 {CurDAG->getRegister(getTileReg(TileNum), XLenVT), Tm, Tn, Log2SEW,
3108 TWiden, Chain});
3109
3110 ReplaceNode(Node, NewNode);
3111 return;
3112 }
3113 }
3114 break;
3115 }
3116 case ISD::BITCAST: {
3117 MVT SrcVT = Node->getOperand(0).getSimpleValueType();
3118 // Just drop bitcasts between vectors if both are fixed or both are
3119 // scalable.
3120 if ((VT.isScalableVector() && SrcVT.isScalableVector()) ||
3121 (VT.isFixedLengthVector() && SrcVT.isFixedLengthVector())) {
3122 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
3123 CurDAG->RemoveDeadNode(Node);
3124 return;
3125 }
3126 if (Subtarget->hasStdExtP()) {
3127 bool Is32BitCast =
3128 (VT == MVT::i32 && (SrcVT == MVT::v4i8 || SrcVT == MVT::v2i16)) ||
3129 (SrcVT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
3130 bool Is64BitCast =
3131 (VT == MVT::i64 && (SrcVT == MVT::v8i8 || SrcVT == MVT::v4i16 ||
3132 SrcVT == MVT::v2i32)) ||
3133 (SrcVT == MVT::i64 &&
3134 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
3135 if (Is32BitCast || Is64BitCast) {
3136 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
3137 CurDAG->RemoveDeadNode(Node);
3138 return;
3139 }
3140 }
3141 break;
3142 }
3143 case ISD::SPLAT_VECTOR: {
3144 if (!Subtarget->hasStdExtP())
3145 break;
3146 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Node->getOperand(0))) {
3147 bool IsDoubleWide = Subtarget->isPExtPackedDoubleType(VT);
3148
3149 if (ConstNode->isZero()) {
3150 MCPhysReg X0Reg = IsDoubleWide ? RISCV::X0_Pair : RISCV::X0;
3151 SDValue New =
3152 CurDAG->getCopyFromReg(CurDAG->getEntryNode(), DL, X0Reg, VT);
3153 ReplaceNode(Node, New.getNode());
3154 return;
3155 }
3156
3157 unsigned EltSize = VT.getVectorElementType().getSizeInBits();
3158 APInt Val = ConstNode->getAPIntValue().trunc(EltSize);
3159
3160 // Use LI for all ones since it can be compressed to c.li.
3161 if (Val.isAllOnes() && !IsDoubleWide) {
3162 SDNode *NewNode = CurDAG->getMachineNode(
3163 RISCV::ADDI, DL, VT, CurDAG->getRegister(RISCV::X0, VT),
3164 CurDAG->getAllOnesConstant(DL, XLenVT, /*IsTarget=*/true));
3165 ReplaceNode(Node, NewNode);
3166 return;
3167 }
3168
3169 // Find the smallest splat.
3170 if (Val.getBitWidth() > 16 && Val.isSplat(16))
3171 Val = Val.trunc(16);
3172 if (Val.getBitWidth() > 8 && Val.isSplat(8))
3173 Val = Val.trunc(8);
3174
3175 EltSize = Val.getBitWidth();
3176 int64_t Imm = Val.getSExtValue();
3177
3178 unsigned Opc = 0;
3179 if (EltSize == 8) {
3180 Opc = IsDoubleWide ? RISCV::PLI_DB : RISCV::PLI_B;
3181 } else if (EltSize == 16 && isInt<10>(Imm)) {
3182 Opc = IsDoubleWide ? RISCV::PLI_DH : RISCV::PLI_H;
3183 } else if (!IsDoubleWide && EltSize == 32 && isInt<10>(Imm)) {
3184 Opc = RISCV::PLI_W;
3185 } else if (EltSize == 16 && isShiftedInt<10, 6>(Imm)) {
3186 Opc = IsDoubleWide ? RISCV::PLUI_DH : RISCV::PLUI_H;
3187 Imm = Imm >> 6;
3188 } else if (!IsDoubleWide && EltSize == 32 && isShiftedInt<10, 22>(Imm)) {
3189 Opc = RISCV::PLUI_W;
3190 Imm = Imm >> 22;
3191 }
3192
3193 if (Opc) {
3194 SDNode *NewNode = CurDAG->getMachineNode(
3195 Opc, DL, VT, CurDAG->getSignedTargetConstant(Imm, DL, XLenVT));
3196 ReplaceNode(Node, NewNode);
3197 return;
3198 }
3199 }
3200
3201 break;
3202 }
3204 if (Subtarget->hasStdExtP()) {
3205 MVT SrcVT = Node->getOperand(0).getSimpleValueType();
3206 if (SrcVT == Subtarget->getXLenVT() &&
3207 VT.getSizeInBits() == SrcVT.getSizeInBits()) {
3208 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
3209 CurDAG->RemoveDeadNode(Node);
3210 return;
3211 }
3212 }
3213 break;
3215 case RISCVISD::TUPLE_INSERT: {
3216 SDValue V = Node->getOperand(0);
3217 SDValue SubV = Node->getOperand(1);
3218 SDLoc DL(SubV);
3219 auto Idx = Node->getConstantOperandVal(2);
3220 MVT SubVecVT = SubV.getSimpleValueType();
3221
3222 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3223 MVT SubVecContainerVT = SubVecVT;
3224 // Establish the correct scalable-vector types for any fixed-length type.
3225 if (SubVecVT.isFixedLengthVector()) {
3226 SubVecContainerVT = TLI.getContainerForFixedLengthVector(SubVecVT);
3228 [[maybe_unused]] bool ExactlyVecRegSized =
3229 Subtarget->expandVScale(SubVecVT.getSizeInBits())
3230 .isKnownMultipleOf(Subtarget->expandVScale(VecRegSize));
3231 assert(isPowerOf2_64(Subtarget->expandVScale(SubVecVT.getSizeInBits())
3232 .getKnownMinValue()));
3233 assert(Idx == 0 && (ExactlyVecRegSized || V.isUndef()));
3234 }
3235 MVT ContainerVT = VT;
3236 if (VT.isFixedLengthVector())
3237 ContainerVT = TLI.getContainerForFixedLengthVector(VT);
3238
3239 const auto *TRI = Subtarget->getRegisterInfo();
3240 unsigned SubRegIdx;
3241 std::tie(SubRegIdx, Idx) =
3243 ContainerVT, SubVecContainerVT, Idx, TRI);
3244
3245 // If the Idx hasn't been completely eliminated then this is a subvector
3246 // insert which doesn't naturally align to a vector register. These must
3247 // be handled using instructions to manipulate the vector registers.
3248 if (Idx != 0)
3249 break;
3250
3251 RISCVVType::VLMUL SubVecLMUL =
3252 RISCVTargetLowering::getLMUL(SubVecContainerVT);
3253 [[maybe_unused]] bool IsSubVecPartReg =
3254 SubVecLMUL == RISCVVType::VLMUL::LMUL_F2 ||
3255 SubVecLMUL == RISCVVType::VLMUL::LMUL_F4 ||
3256 SubVecLMUL == RISCVVType::VLMUL::LMUL_F8;
3257 assert((V.getValueType().isRISCVVectorTuple() || !IsSubVecPartReg ||
3258 V.isUndef()) &&
3259 "Expecting lowering to have created legal INSERT_SUBVECTORs when "
3260 "the subvector is smaller than a full-sized register");
3261
3262 // If we haven't set a SubRegIdx, then we must be going between
3263 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
3264 if (SubRegIdx == RISCV::NoSubRegister) {
3265 unsigned InRegClassID =
3268 InRegClassID &&
3269 "Unexpected subvector extraction");
3270 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
3271 SDNode *NewNode = CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS,
3272 DL, VT, SubV, RC);
3273 ReplaceNode(Node, NewNode);
3274 return;
3275 }
3276
3277 SDValue Insert = CurDAG->getTargetInsertSubreg(SubRegIdx, DL, VT, V, SubV);
3278 ReplaceNode(Node, Insert.getNode());
3279 return;
3280 }
3281 case RISCVISD::TUPLE_CAST: {
3282 // TUPLE_CAST reinterprets a vector tuple as a different tuple type with
3283 // the same register class (same NF and LMUL), just a different minimum
3284 // number of elements per field. The two types occupy identical
3285 // registers, so just forward the operand like we do for same-register-
3286 // class ISD::BITCAST, without emitting a copy.
3289 Node->getOperand(0).getSimpleValueType()) &&
3290 "Expected input and output of TUPLE_CAST to use the same "
3291 "register class");
3292 ReplaceUses(SDValue(Node, 0), Node->getOperand(0));
3293 CurDAG->RemoveDeadNode(Node);
3294 return;
3295 }
3297 case RISCVISD::TUPLE_EXTRACT: {
3298 if (Subtarget->hasStdExtP())
3299 break;
3300
3301 SDValue V = Node->getOperand(0);
3302 auto Idx = Node->getConstantOperandVal(1);
3303 MVT InVT = V.getSimpleValueType();
3304
3305 SDLoc DL(V);
3306
3307 const RISCVTargetLowering &TLI = *Subtarget->getTargetLowering();
3308 MVT SubVecContainerVT = VT;
3309 // Establish the correct scalable-vector types for any fixed-length type.
3310 if (VT.isFixedLengthVector()) {
3311 assert(Idx == 0);
3312 SubVecContainerVT = TLI.getContainerForFixedLengthVector(VT);
3313 }
3314 if (InVT.isFixedLengthVector())
3315 InVT = TLI.getContainerForFixedLengthVector(InVT);
3316
3317 const auto *TRI = Subtarget->getRegisterInfo();
3318 unsigned SubRegIdx;
3319 std::tie(SubRegIdx, Idx) =
3321 InVT, SubVecContainerVT, Idx, TRI);
3322
3323 // If the Idx hasn't been completely eliminated then this is a subvector
3324 // extract which doesn't naturally align to a vector register. These must
3325 // be handled using instructions to manipulate the vector registers.
3326 if (Idx != 0)
3327 break;
3328
3329 // If we haven't set a SubRegIdx, then we must be going between
3330 // equally-sized LMUL types (e.g. VR -> VR). This can be done as a copy.
3331 if (SubRegIdx == RISCV::NoSubRegister) {
3332 unsigned InRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(InVT);
3334 InRegClassID &&
3335 "Unexpected subvector extraction");
3336 SDValue RC = CurDAG->getTargetConstant(InRegClassID, DL, XLenVT);
3337 SDNode *NewNode =
3338 CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, DL, VT, V, RC);
3339 ReplaceNode(Node, NewNode);
3340 return;
3341 }
3342
3343 SDValue Extract = CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, V);
3344 ReplaceNode(Node, Extract.getNode());
3345 return;
3346 }
3347 case RISCVISD::VMV_S_X_VL:
3348 case RISCVISD::VFMV_S_F_VL:
3349 case RISCVISD::VMV_V_X_VL:
3350 case RISCVISD::VFMV_V_F_VL: {
3351 // Try to match splat of a scalar load to a strided load with stride of x0.
3352 bool IsScalarMove = Node->getOpcode() == RISCVISD::VMV_S_X_VL ||
3353 Node->getOpcode() == RISCVISD::VFMV_S_F_VL;
3354 if (!Node->getOperand(0).isUndef())
3355 break;
3356 SDValue Src = Node->getOperand(1);
3357 auto *Ld = dyn_cast<LoadSDNode>(Src);
3358 // Can't fold load update node because the second
3359 // output is used so that load update node can't be removed.
3360 if (!Ld || Ld->isIndexed())
3361 break;
3362 EVT MemVT = Ld->getMemoryVT();
3363 // The memory VT should be the same size as the element type.
3364 if (MemVT.getStoreSize() != VT.getVectorElementType().getStoreSize())
3365 break;
3366 if (!IsProfitableToFold(Src, Node, Node) ||
3367 !IsLegalToFold(Src, Node, Node, TM.getOptLevel()))
3368 break;
3369
3370 SDValue VL;
3371 if (IsScalarMove) {
3372 // We could deal with more VL if we update the VSETVLI insert pass to
3373 // avoid introducing more VSETVLI.
3374 if (!isOneConstant(Node->getOperand(2)))
3375 break;
3376 selectVLOp(Node->getOperand(2), VL);
3377 } else
3378 selectVLOp(Node->getOperand(2), VL);
3379
3380 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
3381 SDValue SEW = CurDAG->getTargetConstant(Log2SEW, DL, XLenVT);
3382
3383 // If VL=1, then we don't need to do a strided load and can just do a
3384 // regular load.
3385 bool IsStrided = !isOneConstant(VL);
3386
3387 // Only do a strided load if we have optimized zero-stride vector load.
3388 if (IsStrided && !Subtarget->hasOptimizedZeroStrideLoad())
3389 break;
3390
3392 SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT), 0),
3393 Ld->getBasePtr()};
3394 if (IsStrided)
3395 Operands.push_back(CurDAG->getRegister(RISCV::X0, XLenVT));
3397 SDValue PolicyOp = CurDAG->getTargetConstant(Policy, DL, XLenVT);
3398 Operands.append({VL, SEW, PolicyOp, Ld->getChain()});
3399
3401 const RISCV::VLEPseudo *P = RISCV::getVLEPseudo(
3402 /*IsMasked*/ false, IsStrided, /*FF*/ false,
3403 Log2SEW, static_cast<unsigned>(LMUL));
3405 CurDAG->getMachineNode(P->Pseudo, DL, {VT, MVT::Other}, Operands);
3406 // Update the chain.
3407 ReplaceUses(Src.getValue(1), SDValue(Load, 1));
3408 // Record the mem-refs
3409 CurDAG->setNodeMemRefs(Load, {Ld->getMemOperand()});
3410 // Replace the splat with the vlse.
3412 return;
3413 }
3414 case RISCVISD::LPAD_CALL:
3415 case RISCVISD::LPAD_CALL_INDIRECT: {
3416 bool IsIndirect = Opcode == RISCVISD::LPAD_CALL_INDIRECT;
3417 unsigned PseudoOpc = IsIndirect ? RISCV::PseudoCALLIndirectLpadAlign
3418 : RISCV::PseudoCALLLpadAlign;
3419
3420 uint32_t LpadLabel = 0;
3421 if (std::optional<uint32_t> Label =
3422 Subtarget->getCLOpts().landing_pad_label) {
3423 if (!isUInt<20>(*Label))
3424 report_fatal_error("riscv-landing-pad-label=<val>, <val> needs to fit "
3425 "in unsigned 20-bits");
3426 LpadLabel = *Label;
3427 }
3428
3429 // Preserve the argument-register and register-mask operands, between
3430 // Callee and the optional glue, so the pseudo call still reports its
3431 // call-preserved mask to the register allocator.
3433 Ops.push_back(Node->getOperand(1));
3434 Ops.push_back(CurDAG->getTargetConstant(LpadLabel, DL, XLenVT));
3435
3436 unsigned NumOps = Node->getNumOperands();
3437 bool HasGlue = Node->getGluedNode() != nullptr;
3438 unsigned RegOperandsEnd = HasGlue ? NumOps - 1 : NumOps;
3439 for (unsigned I = 2; I != RegOperandsEnd; ++I)
3440 Ops.push_back(Node->getOperand(I));
3441
3442 Ops.push_back(Node->getOperand(0));
3443 if (HasGlue)
3444 Ops.push_back(Node->getOperand(NumOps - 1));
3445
3447 CurDAG->getMachineNode(PseudoOpc, DL, Node->getVTList(), Ops));
3448 return;
3449 }
3450 case ISD::PREFETCH:
3451 // MIPS's prefetch instruction already encodes the hint within the
3452 // instruction itself, so no extra NTL hint is needed.
3453 if (Subtarget->hasVendorXMIPSCBOP())
3454 break;
3455
3456 unsigned Locality = Node->getConstantOperandVal(3);
3457 if (Locality > 2)
3458 break;
3459
3460 auto *LoadStoreMem = cast<MemSDNode>(Node);
3461 MachineMemOperand *MMO = LoadStoreMem->getMemOperand();
3463
3464 int NontemporalLevel = 0;
3465 switch (Locality) {
3466 case 0:
3467 NontemporalLevel = 3; // NTL.ALL
3468 break;
3469 case 1:
3470 NontemporalLevel = 1; // NTL.PALL
3471 break;
3472 case 2:
3473 NontemporalLevel = 0; // NTL.P1
3474 break;
3475 default:
3476 llvm_unreachable("unexpected locality value.");
3477 }
3478
3479 if (NontemporalLevel & 0b1)
3481 if (NontemporalLevel & 0b10)
3483 break;
3484 }
3485
3486 // Select the default instruction.
3487 SelectCode(Node);
3488}
3489
3491 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID,
3492 std::vector<SDValue> &OutOps) {
3493 // Always produce a register and immediate operand, as expected by
3494 // RISCVAsmPrinter::PrintAsmMemoryOperand.
3495 switch (ConstraintID) {
3498 SDValue Op0, Op1;
3499 [[maybe_unused]] bool Found = SelectAddrRegImm(Op, Op0, Op1);
3500 assert(Found && "SelectAddrRegImm should always succeed");
3501 OutOps.push_back(Op0);
3502 OutOps.push_back(Op1);
3503 return false;
3504 }
3506 OutOps.push_back(Op);
3507 OutOps.push_back(
3508 CurDAG->getTargetConstant(0, SDLoc(Op), Subtarget->getXLenVT()));
3509 return false;
3510 default:
3511 report_fatal_error("Unexpected asm memory constraint " +
3512 InlineAsm::getMemConstraintName(ConstraintID));
3513 }
3514
3515 return true;
3516}
3517
3519 SDValue &Offset) {
3520 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
3521 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), Subtarget->getXLenVT());
3522 Offset = CurDAG->getTargetConstant(0, SDLoc(Addr), Subtarget->getXLenVT());
3523 return true;
3524 }
3525
3526 return false;
3527}
3528
3529// Fold constant addresses.
3530static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL,
3531 const MVT VT, const RISCVSubtarget *Subtarget,
3533 bool IsPrefetch = false) {
3534 if (!isa<ConstantSDNode>(Addr))
3535 return false;
3536
3537 int64_t CVal = cast<ConstantSDNode>(Addr)->getSExtValue();
3538
3539 // If the constant is a simm12, we can fold the whole constant and use X0 as
3540 // the base. If the constant can be materialized with LUI+simm12, use LUI as
3541 // the base. We can't use generateInstSeq because it favors LUI+ADDIW.
3542 int64_t Lo12 = SignExtend64<12>(CVal);
3543 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
3544 if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
3545 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3546 return false;
3547 if (Hi) {
3548 int64_t Hi20 = (Hi >> 12) & 0xfffff;
3549 Base = SDValue(
3550 CurDAG->getMachineNode(RISCV::LUI, DL, VT,
3551 CurDAG->getTargetConstant(Hi20, DL, VT)),
3552 0);
3553 } else {
3554 Base = CurDAG->getRegister(RISCV::X0, VT);
3555 }
3556 Offset = CurDAG->getSignedTargetConstant(Lo12, DL, VT);
3557 return true;
3558 }
3559
3560 // Ask how constant materialization would handle this constant.
3562
3563 // If the last instruction would be an ADDI, we can fold its immediate and
3564 // emit the rest of the sequence as the base.
3565 if (Seq.back().getOpcode() != RISCV::ADDI)
3566 return false;
3567 Lo12 = Seq.back().getImm();
3568 if (IsPrefetch && (Lo12 & 0b11111) != 0)
3569 return false;
3570
3571 // Drop the last instruction.
3572 Seq.pop_back();
3573 assert(!Seq.empty() && "Expected more instructions in sequence");
3574
3575 Base = selectImmSeq(CurDAG, DL, VT, Seq);
3576 Offset = CurDAG->getSignedTargetConstant(Lo12, DL, VT);
3577 return true;
3578}
3579
3580// Is this ADD instruction only used as the base pointer of scalar loads and
3581// stores?
3583 for (auto *User : Add->users()) {
3584 if (User->getOpcode() != ISD::LOAD && User->getOpcode() != ISD::STORE &&
3585 User->getOpcode() != RISCVISD::LD_RV32 &&
3586 User->getOpcode() != RISCVISD::SD_RV32 &&
3587 User->getOpcode() != ISD::ATOMIC_LOAD &&
3588 User->getOpcode() != ISD::ATOMIC_STORE)
3589 return false;
3590 EVT VT = cast<MemSDNode>(User)->getMemoryVT();
3591 if (!VT.isScalarInteger() && VT != MVT::f16 && VT != MVT::f32 &&
3592 VT != MVT::f64)
3593 return false;
3594 // Don't allow stores of the value. It must be used as the address.
3595 if (User->getOpcode() == ISD::STORE &&
3596 cast<StoreSDNode>(User)->getValue() == Add)
3597 return false;
3598 if (User->getOpcode() == ISD::ATOMIC_STORE &&
3599 cast<AtomicSDNode>(User)->getVal() == Add)
3600 return false;
3601 if (User->getOpcode() == RISCVISD::SD_RV32 &&
3602 (User->getOperand(0) == Add || User->getOperand(1) == Add))
3603 return false;
3604 if (isStrongerThanMonotonic(cast<MemSDNode>(User)->getSuccessOrdering()))
3605 return false;
3606 }
3607
3608 return true;
3609}
3610
3612 switch (User->getOpcode()) {
3613 default:
3614 return false;
3615 case ISD::LOAD:
3616 case RISCVISD::LD_RV32:
3617 case ISD::ATOMIC_LOAD:
3618 break;
3619 case ISD::STORE:
3620 // Don't allow stores of Add. It must only be used as the address.
3622 return false;
3623 break;
3624 case RISCVISD::SD_RV32:
3625 // Don't allow stores of Add. It must only be used as the address.
3626 if (User->getOperand(0) == Add || User->getOperand(1) == Add)
3627 return false;
3628 break;
3629 case ISD::ATOMIC_STORE:
3630 // Don't allow stores of Add. It must only be used as the address.
3631 if (cast<AtomicSDNode>(User)->getVal() == Add)
3632 return false;
3633 break;
3634 }
3635
3636 return true;
3637}
3638
3639// To prevent SelectAddrRegImm from folding offsets that conflict with the
3640// fusion of PseudoMovAddr, check if the offset of every use of a given address
3641// is within the alignment.
3643 Align Alignment) {
3644 assert(Addr->getOpcode() == RISCVISD::ADD_LO);
3645 for (auto *User : Addr->users()) {
3646 // If the user is a load or store, then the offset is 0 which is always
3647 // within alignment.
3648 if (isRegImmLoadOrStore(User, Addr))
3649 continue;
3650
3651 if (CurDAG->isBaseWithConstantOffset(SDValue(User, 0))) {
3652 int64_t CVal = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
3653 if (!isInt<12>(CVal) || Alignment <= CVal)
3654 return false;
3655
3656 // Make sure all uses are foldable load/stores.
3657 for (auto *AddUser : User->users())
3658 if (!isRegImmLoadOrStore(AddUser, SDValue(User, 0)))
3659 return false;
3660
3661 continue;
3662 }
3663
3664 return false;
3665 }
3666
3667 return true;
3668}
3669
3671 SDValue &Offset) {
3672 if (SelectAddrFrameIndex(Addr, Base, Offset))
3673 return true;
3674
3675 SDLoc DL(Addr);
3676 MVT VT = Addr.getSimpleValueType();
3677
3678 if (Addr.getOpcode() == RISCVISD::ADD_LO) {
3679 bool CanFold = true;
3680 // Unconditionally fold if operand 1 is not a global address (e.g.
3681 // externsymbol)
3682 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Addr.getOperand(1))) {
3683 const DataLayout &DL = CurDAG->getDataLayout();
3684 Align Alignment = commonAlignment(
3685 GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
3686 if (!areOffsetsWithinAlignment(Addr, Alignment))
3687 CanFold = false;
3688 }
3689 if (CanFold) {
3690 Base = Addr.getOperand(0);
3691 Offset = Addr.getOperand(1);
3692 return true;
3693 }
3694 }
3695
3696 if (CurDAG->isBaseWithConstantOffset(Addr)) {
3697 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3698 if (isInt<12>(CVal)) {
3699 Base = Addr.getOperand(0);
3700 if (Base.getOpcode() == RISCVISD::ADD_LO) {
3701 SDValue LoOperand = Base.getOperand(1);
3702 if (auto *GA = dyn_cast<GlobalAddressSDNode>(LoOperand)) {
3703 // If the Lo in (ADD_LO hi, lo) is a global variable's address
3704 // (its low part, really), then we can rely on the alignment of that
3705 // variable to provide a margin of safety before low part can overflow
3706 // the 12 bits of the load/store offset. Check if CVal falls within
3707 // that margin; if so (low part + CVal) can't overflow.
3708 const DataLayout &DL = CurDAG->getDataLayout();
3709 Align Alignment = commonAlignment(
3710 GA->getGlobal()->getPointerAlignment(DL), GA->getOffset());
3711 if ((CVal == 0 || Alignment > CVal) &&
3712 areOffsetsWithinAlignment(Base, Alignment)) {
3713 int64_t CombinedOffset = CVal + GA->getOffset();
3714 Base = Base.getOperand(0);
3715 Offset = CurDAG->getTargetGlobalAddress(
3716 GA->getGlobal(), SDLoc(LoOperand), LoOperand.getValueType(),
3717 CombinedOffset, GA->getTargetFlags());
3718 return true;
3719 }
3720 }
3721 }
3722
3723 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
3724 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
3725 Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
3726 return true;
3727 }
3728 }
3729
3730 // Handle ADD with large immediates.
3731 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
3732 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3733 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3734
3735 // Handle immediates in the range [-4096,-2049] or [2048, 4094]. We can use
3736 // an ADDI for part of the offset and fold the rest into the load/store.
3737 // This mirrors the AddiPair PatFrag in RISCVInstrInfo.td.
3738 if (CVal >= -4096 && CVal <= 4094) {
3739 int64_t Adj = CVal < 0 ? -2048 : 2047;
3740 Base = SDValue(
3741 CurDAG->getMachineNode(RISCV::ADDI, DL, VT, Addr.getOperand(0),
3742 CurDAG->getSignedTargetConstant(Adj, DL, VT)),
3743 0);
3744 Offset = CurDAG->getSignedTargetConstant(CVal - Adj, DL, VT);
3745 return true;
3746 }
3747
3748 // For larger immediates, we might be able to save one instruction from
3749 // constant materialization by folding the Lo12 bits of the immediate into
3750 // the address. We should only do this if the ADD is only used by loads and
3751 // stores that can fold the lo12 bits. Otherwise, the ADD will get iseled
3752 // separately with the full materialized immediate creating extra
3753 // instructions.
3754 if (isWorthFoldingAdd(Addr) &&
3755 selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
3756 Offset, /*IsPrefetch=*/false)) {
3757 // Insert an ADD instruction with the materialized Hi52 bits.
3758 Base = SDValue(
3759 CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
3760 0);
3761 return true;
3762 }
3763 }
3764
3765 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3766 /*IsPrefetch=*/false))
3767 return true;
3768
3769 Base = Addr;
3770 Offset = CurDAG->getTargetConstant(0, DL, VT);
3771 return true;
3772}
3773
3774/// Similar to SelectAddrRegImm, but only matches a register, or a register
3775/// plus a simm12 offset. Doesn't match a FrameIndex or global address, since
3776/// those aren't valid for the callers of this function (e.g. the target of
3777/// an indirect branch).
3779 SDValue &Offset) {
3780 SDLoc DL(Addr);
3781 MVT VT = Addr.getSimpleValueType();
3782
3783 if (CurDAG->isBaseWithConstantOffset(Addr)) {
3784 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3785 if (isInt<12>(CVal)) {
3786 Base = Addr.getOperand(0);
3787 Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
3788 return true;
3789 }
3790 }
3791
3792 Base = Addr;
3793 Offset = CurDAG->getTargetConstant(0, DL, VT);
3794 return true;
3795}
3796
3797/// Similar to SelectAddrRegImm, except that the offset is a 26-bit signed
3798/// immediate. This is used by the Qualcomm Xqcilo large offset load/store
3799/// instructions (qc.e.lw/qc.e.sw), whose offset field is 26 bits wide.
3800/// Only matches offsets that do not fit a 12-bit signed immediate, so that
3801/// offsets in the simm12 range keep using the shorter (and possibly
3802/// compressible) standard load/store instructions.
3804 SDValue &Offset) {
3805 SDLoc DL(Addr);
3806 MVT VT = Addr.getSimpleValueType();
3807
3808 if (CurDAG->isBaseWithConstantOffset(Addr)) {
3809 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3810 // Fold a 26-bit (but not 12-bit) signed offset directly into the
3811 // load/store.
3812 if (isInt<26>(CVal) && !isInt<12>(CVal)) {
3813 Base = Addr.getOperand(0);
3814 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
3815 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
3816 Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
3817 return true;
3818 }
3819 }
3820
3821 // The offset is just outside the 26-bit range. Split off a small (simm12)
3822 // adjustment with a plain ADDI and fold the remaining 26-bit offset into the
3823 // load/store. A plain ADDI is used (rather than the wide
3824 // qc.e.addi/qc.e.addai) because the adjustment fits simm12: this keeps it a
3825 // short, compressible (c.addi) instruction and is available without Xqcilia.
3826 //
3827 // Skip the split if the address is used other than as a foldable load/store
3828 // base. `isWorthFoldingAdd()` returns true when every user of the add node is
3829 // a scalar load/store using it as an address operand. If it return false, it
3830 // means that some use consumes the add result as a value (e.g. it feeds
3831 // another add, is a stored value, is used in arithmetic) and that use forces
3832 // the add to be materialized into a register.
3833 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1)) &&
3834 isWorthFoldingAdd(Addr)) {
3835 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3836 if (!isInt<26>(CVal)) {
3837 // check if lw in lui + add + lw combination can be compressed.
3838 // The check here purely based on the immediate value and hopes that
3839 // register allocator would assign a register from a GPRC set so that the
3840 // instruction can get compressed.
3841 bool IsLwCompressable = isShiftedUInt<5, 2>(CVal & ((1 << 12) - 1));
3842
3843 int64_t Imm26 = CVal < 0 ? minIntN(26) : maxIntN(26);
3844 int64_t Adj = CVal - Imm26;
3845 // If Adj fits within 6-bits, then both combinations will take 8 bytes
3846 // however c.addi + qc.e.lw/sw will take 1 less cycle. Also, if lw is not
3847 // compressable then both combination would take 10 bytes but again
3848 // addi + qc.e.lw/sw will take 1 less cycle.
3849 if (isInt<6>(Adj) || (isInt<12>(Adj) && !IsLwCompressable)) {
3850 Base = SDValue(CurDAG->getMachineNode(
3851 RISCV::ADDI, DL, VT, Addr.getOperand(0),
3852 CurDAG->getSignedTargetConstant(Adj, DL, VT)),
3853 0);
3854 Offset = CurDAG->getSignedTargetConstant(Imm26, DL, VT);
3855 return true;
3856 }
3857 }
3858 }
3859
3860 // Don't match: let the standard addressing modes handle it.
3861 return false;
3862}
3863
3864/// Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
3866 SDValue &Offset) {
3867 if (SelectAddrFrameIndex(Addr, Base, Offset))
3868 return true;
3869
3870 SDLoc DL(Addr);
3871 MVT VT = Addr.getSimpleValueType();
3872
3873 if (CurDAG->isBaseWithConstantOffset(Addr)) {
3874 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3875 if (isUInt<9>(CVal)) {
3876 Base = Addr.getOperand(0);
3877
3878 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
3879 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
3880 Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
3881 return true;
3882 }
3883 }
3884
3885 Base = Addr;
3886 Offset = CurDAG->getTargetConstant(0, DL, VT);
3887 return true;
3888}
3889
3890/// Similar to SelectAddrRegImm, except that the least significant 5 bits of
3891/// Offset should be all zeros.
3893 SDValue &Offset) {
3894 if (SelectAddrFrameIndex(Addr, Base, Offset))
3895 return true;
3896
3897 SDLoc DL(Addr);
3898 MVT VT = Addr.getSimpleValueType();
3899
3900 if (CurDAG->isBaseWithConstantOffset(Addr)) {
3901 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3902 if (isInt<12>(CVal)) {
3903 Base = Addr.getOperand(0);
3904
3905 // Early-out if not a valid offset.
3906 if ((CVal & 0b11111) != 0) {
3907 Base = Addr;
3908 Offset = CurDAG->getTargetConstant(0, DL, VT);
3909 return true;
3910 }
3911
3912 if (auto *FIN = dyn_cast<FrameIndexSDNode>(Base))
3913 Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), VT);
3914 Offset = CurDAG->getSignedTargetConstant(CVal, DL, VT);
3915 return true;
3916 }
3917 }
3918
3919 // Handle ADD with large immediates.
3920 if (Addr.getOpcode() == ISD::ADD && isa<ConstantSDNode>(Addr.getOperand(1))) {
3921 int64_t CVal = cast<ConstantSDNode>(Addr.getOperand(1))->getSExtValue();
3922 assert(!isInt<12>(CVal) && "simm12 not already handled?");
3923
3924 // Handle immediates in the range [-4096,-2049] or [2017, 4063]. We can save
3925 // one instruction by folding adjustment (-2048 or 2016) into the address.
3926 // The upper bound keeps CVal - 2016 within simm12 ([−2048, 2047]).
3927 if ((-2049 >= CVal && CVal >= -4096) || (4063 >= CVal && CVal >= 2017)) {
3928 int64_t Adj = CVal < 0 ? -2048 : 2016;
3929 int64_t AdjustedOffset = CVal - Adj;
3930 Base =
3931 SDValue(CurDAG->getMachineNode(
3932 RISCV::ADDI, DL, VT, Addr.getOperand(0),
3933 CurDAG->getSignedTargetConstant(AdjustedOffset, DL, VT)),
3934 0);
3935 Offset = CurDAG->getSignedTargetConstant(Adj, DL, VT);
3936 return true;
3937 }
3938
3939 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr.getOperand(1), Base,
3940 Offset, /*IsPrefetch=*/true)) {
3941 // Insert an ADD instruction with the materialized Hi52 bits.
3942 Base = SDValue(
3943 CurDAG->getMachineNode(RISCV::ADD, DL, VT, Addr.getOperand(0), Base),
3944 0);
3945 return true;
3946 }
3947 }
3948
3949 if (selectConstantAddr(CurDAG, DL, VT, Subtarget, Addr, Base, Offset,
3950 /*IsPrefetch=*/true))
3951 return true;
3952
3953 Base = Addr;
3954 Offset = CurDAG->getTargetConstant(0, DL, VT);
3955 return true;
3956}
3957
3958/// Return true if this a load/store that we have a RegRegScale instruction for.
3960 const RISCVSubtarget &Subtarget) {
3961 unsigned UserOpc = User->getOpcode();
3962 if (UserOpc != ISD::LOAD && UserOpc != ISD::STORE)
3963 return false;
3964 EVT VT = cast<MemSDNode>(User)->getMemoryVT();
3965 // Zilx only provides indexed loads, so it must not enable reg+reg-scale
3966 // address folding for stores. XTheadMemIdx and Xqcisls have scaled stores.
3967 bool HasScalarIntegerMemIdx =
3968 Subtarget.hasVendorXTHeadMemIdx() || Subtarget.hasVendorXqcisls() ||
3969 (Subtarget.hasStdExtZilx() && UserOpc == ISD::LOAD);
3970 if (!(VT.isScalarInteger() && HasScalarIntegerMemIdx) &&
3971 !((VT == MVT::f32 || VT == MVT::f64) &&
3972 Subtarget.hasVendorXTHeadFMemIdx()))
3973 return false;
3974 // Don't allow stores of the value. It must be used as the address.
3975 if (UserOpc == ISD::STORE && cast<StoreSDNode>(User)->getValue() == Add)
3976 return false;
3977
3978 return true;
3979}
3980
3981/// Is it profitable to fold this Add into RegRegScale load/store. If \p
3982/// Shift is non-null, then we have matched a shl+add. We allow reassociating
3983/// (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2)) if there is a
3984/// single addi and we don't have a SHXADD instruction we could use.
3985/// FIXME: May still need to check how many and what kind of users the SHL has.
3987 SDValue Add,
3988 SDValue Shift = SDValue()) {
3989 bool FoundADDI = false;
3990 for (auto *User : Add->users()) {
3992 continue;
3993
3994 // Allow a single ADDI that is used by loads/stores if we matched a shift.
3995 if (!Shift || FoundADDI || User->getOpcode() != ISD::ADD ||
3997 !isInt<12>(cast<ConstantSDNode>(User->getOperand(1))->getSExtValue()))
3998 return false;
3999
4000 FoundADDI = true;
4001
4002 // If we have a SHXADD instruction, prefer that over reassociating an ADDI.
4003 assert(Shift.getOpcode() == ISD::SHL);
4004 unsigned ShiftAmt = Shift.getConstantOperandVal(1);
4005 if (Subtarget.hasShlAdd(ShiftAmt))
4006 return false;
4007
4008 // All users of the ADDI should be load/store.
4009 for (auto *ADDIUser : User->users())
4010 if (!isRegRegScaleLoadOrStore(ADDIUser, SDValue(User, 0), Subtarget))
4011 return false;
4012 }
4013
4014 return true;
4015}
4016
4018 ArrayRef<unsigned> Amounts,
4019 SDValue &Base, SDValue &Index,
4020 SDValue &Scale) {
4021 if (Addr.getOpcode() != ISD::ADD)
4022 return false;
4023 SDValue LHS = Addr.getOperand(0);
4024 SDValue RHS = Addr.getOperand(1);
4025
4026 EVT VT = Addr.getSimpleValueType();
4027 auto SelectShl = [this, VT, Amounts](SDValue N, SDValue &Index,
4028 SDValue &Shift) {
4029 if (N.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(N.getOperand(1)))
4030 return false;
4031
4032 // Only match shifts by a value in range [0, MaxShiftAmount].
4033 unsigned ShiftAmt = N.getConstantOperandVal(1);
4034 if (!llvm::is_contained(Amounts, ShiftAmt))
4035 return false;
4036
4037 Index = N.getOperand(0);
4038 Shift = CurDAG->getTargetConstant(ShiftAmt, SDLoc(N), VT);
4039 return true;
4040 };
4041
4042 if (auto *C1 = dyn_cast<ConstantSDNode>(RHS)) {
4043 // (add (add (shl A C2) B) C1) -> (add (add B C1) (shl A C2))
4044 if (LHS.getOpcode() == ISD::ADD &&
4045 !isa<ConstantSDNode>(LHS.getOperand(1)) &&
4046 isInt<12>(C1->getSExtValue())) {
4047 if (SelectShl(LHS.getOperand(1), Index, Scale) &&
4048 isWorthFoldingIntoRegRegScale(*Subtarget, LHS, LHS.getOperand(1))) {
4049 SDValue C1Val = CurDAG->getTargetConstant(*C1->getConstantIntValue(),
4050 SDLoc(Addr), VT);
4051 Base = SDValue(CurDAG->getMachineNode(RISCV::ADDI, SDLoc(Addr), VT,
4052 LHS.getOperand(0), C1Val),
4053 0);
4054 return true;
4055 }
4056
4057 // Add is commutative so we need to check both operands.
4058 if (SelectShl(LHS.getOperand(0), Index, Scale) &&
4059 isWorthFoldingIntoRegRegScale(*Subtarget, LHS, LHS.getOperand(0))) {
4060 SDValue C1Val = CurDAG->getTargetConstant(*C1->getConstantIntValue(),
4061 SDLoc(Addr), VT);
4062 Base = SDValue(CurDAG->getMachineNode(RISCV::ADDI, SDLoc(Addr), VT,
4063 LHS.getOperand(1), C1Val),
4064 0);
4065 return true;
4066 }
4067 }
4068
4069 // Don't match add with constants.
4070 // FIXME: Is this profitable for large constants that have 0s in the lower
4071 // 12 bits that we can materialize with LUI?
4072 return false;
4073 }
4074
4075 // Try to match a shift on the RHS.
4076 if (SelectShl(RHS, Index, Scale)) {
4077 if (!isWorthFoldingIntoRegRegScale(*Subtarget, Addr, RHS))
4078 return false;
4079 Base = LHS;
4080 return true;
4081 }
4082
4083 // Try to match a shift on the LHS.
4084 if (SelectShl(LHS, Index, Scale)) {
4085 if (!isWorthFoldingIntoRegRegScale(*Subtarget, Addr, LHS))
4086 return false;
4087 Base = RHS;
4088 return true;
4089 }
4090
4091 if (!isWorthFoldingIntoRegRegScale(*Subtarget, Addr))
4092 return false;
4093
4094 // Bail out if 0 is not in candidate shift amounts.
4095 if (!llvm::is_contained(Amounts, 0))
4096 return false;
4097
4098 Base = LHS;
4099 Index = RHS;
4100 Scale = CurDAG->getTargetConstant(0, SDLoc(Addr), VT);
4101 return true;
4102}
4103
4105 ArrayRef<unsigned> Amounts,
4106 unsigned Bits, SDValue &Base,
4107 SDValue &Index,
4108 SDValue &Scale) {
4109 if (!SelectAddrRegRegScale(Addr, Amounts, Base, Index, Scale))
4110 return false;
4111
4112 if (Index.getOpcode() == ISD::AND) {
4113 auto *C = dyn_cast<ConstantSDNode>(Index.getOperand(1));
4114 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
4115 Index = Index.getOperand(0);
4116 return true;
4117 }
4118 }
4119
4120 return false;
4121}
4122
4124 SDValue &Offset) {
4125 if (Addr.getOpcode() != ISD::ADD)
4126 return false;
4127
4128 if (isa<ConstantSDNode>(Addr.getOperand(1)))
4129 return false;
4130
4131 Base = Addr.getOperand(0);
4132 Offset = Addr.getOperand(1);
4133 return true;
4134}
4135
4137 SDValue &ShAmt) {
4138 // Shift instructions on RISC-V only read the lower 5 or 6 bits of the shift
4139 // amount. Look through an AND that doesn't affect those bits.
4140 if (!selectShiftAmountMask(N, ShiftWidth, ShAmt))
4141 ShAmt = N;
4142
4143 if (ShAmt.getOpcode() == ISD::ADD &&
4144 isa<ConstantSDNode>(ShAmt.getOperand(1))) {
4145 uint64_t Imm = ShAmt.getConstantOperandVal(1);
4146 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
4147 // to avoid the ADD.
4148 if (Imm != 0 && Imm % ShiftWidth == 0) {
4149 ShAmt = ShAmt.getOperand(0);
4150 return true;
4151 }
4152 } else if (ShAmt.getOpcode() == ISD::SUB &&
4153 isa<ConstantSDNode>(ShAmt.getOperand(0))) {
4154 uint64_t Imm = ShAmt.getConstantOperandVal(0);
4155 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X to
4156 // generate a NEG instead of a SUB of a constant.
4157 if (Imm != 0 && Imm % ShiftWidth == 0) {
4158 SDLoc DL(ShAmt);
4159 EVT VT = ShAmt.getValueType();
4160 SDValue Zero = CurDAG->getRegister(RISCV::X0, VT);
4161 unsigned NegOpc = VT == MVT::i64 ? RISCV::SUBW : RISCV::SUB;
4162 MachineSDNode *Neg = CurDAG->getMachineNode(NegOpc, DL, VT, Zero,
4163 ShAmt.getOperand(1));
4164 ShAmt = SDValue(Neg, 0);
4165 return true;
4166 }
4167 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
4168 // to generate a NOT instead of a SUB of a constant.
4169 if (Imm % ShiftWidth == ShiftWidth - 1) {
4170 SDLoc DL(ShAmt);
4171 EVT VT = ShAmt.getValueType();
4172 MachineSDNode *Not = CurDAG->getMachineNode(
4173 RISCV::XORI, DL, VT, ShAmt.getOperand(1),
4174 CurDAG->getAllOnesConstant(DL, VT, /*isTarget=*/true));
4175 ShAmt = SDValue(Not, 0);
4176 return true;
4177 }
4178 }
4179
4180 return true;
4181}
4182
4183/// RISC-V doesn't have general instructions for integer setne/seteq, but we can
4184/// check for equality with 0. This function emits instructions that convert the
4185/// seteq/setne into something that can be compared with 0.
4186/// \p ExpectedCCVal indicates the condition code to attempt to match (e.g.
4187/// ISD::SETNE).
4189 SDValue &Val, bool OneUse) {
4190 assert(ISD::isIntEqualitySetCC(ExpectedCCVal) &&
4191 "Unexpected condition code!");
4192
4193 // We're looking for a setcc.
4194 if (N->getOpcode() != ISD::SETCC)
4195 return false;
4196
4197 if (OneUse && !N->hasOneUse())
4198 return false;
4199
4200 // Must be an equality comparison.
4201 ISD::CondCode CCVal = N->getOperandCondCode(2);
4202 if (CCVal != ExpectedCCVal)
4203 return false;
4204
4205 SDValue LHS = N->getOperand(0);
4206 SDValue RHS = N->getOperand(1);
4207
4208 if (!LHS.getValueType().isScalarInteger())
4209 return false;
4210
4211 // If the RHS side is 0, we don't need any extra instructions, return the LHS.
4212 if (isNullConstant(RHS)) {
4213 Val = LHS;
4214 return true;
4215 }
4216
4217 SDLoc DL(N);
4218
4219 if (auto *C = dyn_cast<ConstantSDNode>(RHS)) {
4220 int64_t CVal = C->getSExtValue();
4221 // If the RHS is -2048, we can use xori to produce 0 if the LHS is -2048 and
4222 // non-zero otherwise.
4223 if (CVal == -2048) {
4224 Val = SDValue(
4225 CurDAG->getMachineNode(
4226 RISCV::XORI, DL, N->getValueType(0), LHS,
4227 CurDAG->getSignedTargetConstant(CVal, DL, N->getValueType(0))),
4228 0);
4229 return true;
4230 }
4231 // If the RHS is [-2047,2048], we can use addi/addiw with -RHS to produce 0
4232 // if the LHS is equal to the RHS and non-zero otherwise.
4233 if (isInt<12>(CVal) || CVal == 2048) {
4234 unsigned Opc = RISCV::ADDI;
4235 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4236 cast<VTSDNode>(LHS.getOperand(1))->getVT() == MVT::i32) {
4237 Opc = RISCV::ADDIW;
4238 LHS = LHS.getOperand(0);
4239 }
4240
4241 Val = SDValue(CurDAG->getMachineNode(Opc, DL, N->getValueType(0), LHS,
4242 CurDAG->getSignedTargetConstant(
4243 -CVal, DL, N->getValueType(0))),
4244 0);
4245 return true;
4246 }
4247 if (isPowerOf2_64(CVal) && Subtarget->hasStdExtZbs()) {
4248 Val = SDValue(
4249 CurDAG->getMachineNode(
4250 RISCV::BINVI, DL, N->getValueType(0), LHS,
4251 CurDAG->getTargetConstant(Log2_64(CVal), DL, N->getValueType(0))),
4252 0);
4253 return true;
4254 }
4255 // Same as the addi case above but for larger immediates (signed 26-bit) use
4256 // the QC_E_ADDI instruction from the Xqcilia extension, if available. Avoid
4257 // anything which can be done with a single lui as it might be compressible.
4258 if (Subtarget->hasVendorXqcilia() && isInt<26>(CVal) &&
4259 (CVal & 0xFFF) != 0) {
4260 Val = SDValue(
4261 CurDAG->getMachineNode(
4262 RISCV::QC_E_ADDI, DL, N->getValueType(0), LHS,
4263 CurDAG->getSignedTargetConstant(-CVal, DL, N->getValueType(0))),
4264 0);
4265 return true;
4266 }
4267 }
4268
4269 // If nothing else we can XOR the LHS and RHS to produce zero if they are
4270 // equal and a non-zero value if they aren't.
4271 Val = SDValue(
4272 CurDAG->getMachineNode(RISCV::XOR, DL, N->getValueType(0), LHS, RHS), 0);
4273 return true;
4274}
4275
4277 if (N.getOpcode() == ISD::SIGN_EXTEND_INREG &&
4278 cast<VTSDNode>(N.getOperand(1))->getVT().getSizeInBits() == Bits) {
4279 Val = N.getOperand(0);
4280 return true;
4281 }
4282
4283 auto UnwrapShlSra = [](SDValue N, unsigned ShiftAmt) {
4284 if (N.getOpcode() != ISD::SRA || !isa<ConstantSDNode>(N.getOperand(1)))
4285 return N;
4286
4287 SDValue N0 = N.getOperand(0);
4288 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
4289 N.getConstantOperandVal(1) == ShiftAmt &&
4290 N0.getConstantOperandVal(1) == ShiftAmt)
4291 return N0.getOperand(0);
4292
4293 return N;
4294 };
4295
4296 MVT VT = N.getSimpleValueType();
4297 if (CurDAG->ComputeNumSignBits(N) > (VT.getSizeInBits() - Bits)) {
4298 Val = UnwrapShlSra(N, VT.getSizeInBits() - Bits);
4299 return true;
4300 }
4301
4302 return false;
4303}
4304
4306 if (N.getOpcode() == ISD::AND) {
4307 auto *C = dyn_cast<ConstantSDNode>(N.getOperand(1));
4308 if (C && C->getZExtValue() == maskTrailingOnes<uint64_t>(Bits)) {
4309 Val = N.getOperand(0);
4310 return true;
4311 }
4312 }
4313 MVT VT = N.getSimpleValueType();
4314 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), Bits);
4315 if (CurDAG->MaskedValueIsZero(N, Mask)) {
4316 Val = N;
4317 return true;
4318 }
4319
4320 return false;
4321}
4322
4323/// Look for various patterns that can be done with a SHL that can be folded
4324/// into a SHXADD. \p ShAmt contains 1, 2, or 3 and is set based on which
4325/// SHXADD we are trying to match.
4327 SDValue &Val) {
4328 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1))) {
4329 MVT VT = N.getSimpleValueType();
4330 assert(VT == Subtarget->getXLenVT());
4331 SDValue N0 = N.getOperand(0);
4332
4333 if (bool LeftShift = N0.getOpcode() == ISD::SHL;
4334 (LeftShift || N0.getOpcode() == ISD::SRL) &&
4336 uint64_t Mask = N.getConstantOperandVal(1);
4337 unsigned C2 = N0.getConstantOperandVal(1);
4338 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4339
4340 unsigned XLen = Subtarget->getXLen();
4341 if (LeftShift)
4342 Mask &= maskTrailingZeros<uint64_t>(C2);
4343 else
4344 Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
4345
4346 if (isShiftedMask_64(Mask)) {
4347 unsigned Leading = XLen - llvm::bit_width(Mask);
4348 unsigned Trailing = llvm::countr_zero(Mask);
4349 if (Trailing != ShAmt)
4350 return false;
4351
4352 unsigned Opcode;
4353 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with no
4354 // leading zeros and c3 trailing zeros. We can use an SRLI by c3-c2
4355 // followed by a SHXADD with c3 for the X amount.
4356 if (LeftShift && Leading == 0 && C2 < Trailing)
4357 Opcode = RISCV::SRLI;
4358 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with 32-c2
4359 // leading zeros and c3 trailing zeros. We can use an SRLIW by c3-c2
4360 // followed by a SHXADD with c3 for the X amount.
4361 else if (LeftShift && Leading == 32 - C2 && C2 < Trailing)
4362 Opcode = RISCV::SRLIW;
4363 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with c2
4364 // leading zeros and c3 trailing zeros. We can use an SRLI by c2+c3
4365 // followed by a SHXADD using c3 for the X amount.
4366 else if (!LeftShift && Leading == C2)
4367 Opcode = RISCV::SRLI;
4368 // Look for (and (shr y, c2), c1) where c1 is a shifted mask with 32+c2
4369 // leading zeros and c3 trailing zeros. We can use an SRLIW by c2+c3
4370 // followed by a SHXADD using c3 for the X amount.
4371 else if (!LeftShift && Leading == 32 + C2)
4372 Opcode = RISCV::SRLIW;
4373 else
4374 return false;
4375
4376 SDLoc DL(N);
4377 ShAmt = LeftShift ? Trailing - C2 : Trailing + C2;
4378 Val = SDValue(
4379 CurDAG->getMachineNode(Opcode, DL, VT, N0.getOperand(0),
4380 CurDAG->getTargetConstant(ShAmt, DL, VT)),
4381 0);
4382 return true;
4383 }
4384 } else if (N0.getOpcode() == ISD::SRA && N0.hasOneUse() &&
4386 uint64_t Mask = N.getConstantOperandVal(1);
4387 unsigned C2 = N0.getConstantOperandVal(1);
4388
4389 // Look for (and (sra y, c2), c1) where c1 is a shifted mask with c3
4390 // leading zeros and c4 trailing zeros. If c2 is greater than c3, we can
4391 // use (srli (srai y, c2 - c3), c3 + c4) followed by a SHXADD with c4 as
4392 // the X amount.
4393 if (isShiftedMask_64(Mask)) {
4394 unsigned XLen = Subtarget->getXLen();
4395 unsigned Leading = XLen - llvm::bit_width(Mask);
4396 unsigned Trailing = llvm::countr_zero(Mask);
4397 if (C2 > Leading && Leading > 0 && Trailing == ShAmt) {
4398 SDLoc DL(N);
4399 EVT VT = N.getValueType();
4400 Val = SDValue(CurDAG->getMachineNode(
4401 RISCV::SRAI, DL, VT, N0.getOperand(0),
4402 CurDAG->getTargetConstant(C2 - Leading, DL, VT)),
4403 0);
4404 Val = SDValue(CurDAG->getMachineNode(
4405 RISCV::SRLI, DL, VT, Val,
4406 CurDAG->getTargetConstant(Leading + ShAmt, DL, VT)),
4407 0);
4408 return true;
4409 }
4410 }
4411 }
4412 } else if (bool LeftShift = N.getOpcode() == ISD::SHL;
4413 (LeftShift || N.getOpcode() == ISD::SRL) &&
4414 isa<ConstantSDNode>(N.getOperand(1))) {
4415 MVT VT = N.getSimpleValueType();
4416 assert(VT == Subtarget->getXLenVT() && "Unexpected VT!");
4417 SDValue N0 = N.getOperand(0);
4418 if (N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
4420 uint64_t Mask = N0.getConstantOperandVal(1);
4421 if (isShiftedMask_64(Mask)) {
4422 unsigned C1 = N.getConstantOperandVal(1);
4423 assert(C1 > 0 && C1 < Subtarget->getXLen() &&
4424 "Unexpected shift amount!");
4425 unsigned XLen = Subtarget->getXLen();
4426 unsigned Leading = XLen - llvm::bit_width(Mask);
4427 unsigned Trailing = llvm::countr_zero(Mask);
4428 // Look for (shl (and X, Mask), C1) where Mask has 32 leading zeros and
4429 // C3 trailing zeros. If C1+C3==ShAmt we can use SRLIW+SHXADD.
4430 if (LeftShift && Leading == 32 && Trailing > 0 &&
4431 (Trailing + C1) == ShAmt) {
4432 SDLoc DL(N);
4433 EVT VT = N.getValueType();
4434 Val = SDValue(CurDAG->getMachineNode(
4435 RISCV::SRLIW, DL, VT, N0.getOperand(0),
4436 CurDAG->getTargetConstant(Trailing, DL, VT)),
4437 0);
4438 return true;
4439 }
4440 // Look for (srl (and X, Mask), C1) where Mask has 32 leading zeros and
4441 // C3 trailing zeros. If C3-C1==ShAmt we can use SRLIW+SHXADD.
4442 if (!LeftShift && Leading == 32 && Trailing > C1 &&
4443 (Trailing - C1) == ShAmt) {
4444 SDLoc DL(N);
4445 Val = SDValue(CurDAG->getMachineNode(
4446 RISCV::SRLIW, DL, VT, N0.getOperand(0),
4447 CurDAG->getTargetConstant(Trailing, DL, VT)),
4448 0);
4449 return true;
4450 }
4451 }
4452 }
4453 }
4454
4455 return false;
4456}
4457
4458/// Look for various patterns that can be done with a SHL that can be folded
4459/// into a SHXADD_UW. \p ShAmt contains 1, 2, or 3 and is set based on which
4460/// SHXADD_UW we are trying to match.
4462 SDValue &Val) {
4463 if (N.getOpcode() == ISD::AND && isa<ConstantSDNode>(N.getOperand(1)) &&
4464 N.hasOneUse()) {
4465 assert(N.getSimpleValueType() == Subtarget->getXLenVT() &&
4466 "Unexpected VT!");
4467 SDValue N0 = N.getOperand(0);
4468 if (N0.getOpcode() == ISD::SHL && isa<ConstantSDNode>(N0.getOperand(1)) &&
4469 N0.hasOneUse()) {
4470 uint64_t Mask = N.getConstantOperandVal(1);
4471 unsigned C2 = N0.getConstantOperandVal(1);
4472 assert(C2 > 0 && C2 < Subtarget->getXLen() && "Unexpected shift amount!");
4473
4474 Mask &= maskTrailingZeros<uint64_t>(C2);
4475
4476 // Look for (and (shl y, c2), c1) where c1 is a shifted mask with
4477 // 32-ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
4478 // c2-ShAmt followed by SHXADD_UW with ShAmt for the X amount.
4479 if (isShiftedMask_64(Mask)) {
4480 unsigned Leading = llvm::countl_zero(Mask);
4481 unsigned Trailing = llvm::countr_zero(Mask);
4482 if (Leading == 32 - ShAmt && Trailing == C2 && Trailing > ShAmt) {
4483 SDLoc DL(N);
4484 EVT VT = N.getValueType();
4485 Val = SDValue(CurDAG->getMachineNode(
4486 RISCV::SLLI, DL, VT, N0.getOperand(0),
4487 CurDAG->getTargetConstant(C2 - ShAmt, DL, VT)),
4488 0);
4489 return true;
4490 }
4491 }
4492 }
4493 }
4494
4495 return false;
4496}
4497
4499 assert(N->getOpcode() == ISD::OR || N->getOpcode() == RISCVISD::OR_VL);
4500 if (N->getFlags().hasDisjoint())
4501 return true;
4502 return CurDAG->haveNoCommonBitsSet(N->getOperand(0), N->getOperand(1));
4503}
4504
4505bool RISCVDAGToDAGISel::selectImm64IfCheaper(int64_t Imm, int64_t OrigImm,
4506 SDValue N, SDValue &Val) {
4507 int OrigCost = RISCVMatInt::getIntMatCost(APInt(64, OrigImm), 64, *Subtarget,
4508 /*CompressionCost=*/true);
4510 /*CompressionCost=*/true);
4511 if (OrigCost <= Cost)
4512 return false;
4513
4514 Val = selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), Imm, *Subtarget);
4515 return true;
4516}
4517
4519 if (!isa<ConstantSDNode>(N))
4520 return false;
4521 int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue();
4522 if ((Imm >> 31) != 1)
4523 return false;
4524
4525 for (const SDNode *U : N->users()) {
4526 switch (U->getOpcode()) {
4527 case ISD::ADD:
4528 break;
4529 case ISD::OR:
4530 if (orDisjoint(U))
4531 break;
4532 return false;
4533 default:
4534 return false;
4535 }
4536 }
4537
4538 return selectImm64IfCheaper(0xffffffff00000000 | Imm, Imm, N, Val);
4539}
4540
4542 if (!isa<ConstantSDNode>(N))
4543 return false;
4544 int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue();
4545 if (isInt<32>(Imm))
4546 return false;
4547 if (Imm == INT64_MIN)
4548 return false;
4549
4550 for (const SDNode *U : N->users()) {
4551 switch (U->getOpcode()) {
4552 case ISD::ADD:
4553 break;
4554 case RISCVISD::VMV_V_X_VL:
4555 if (!all_of(U->users(), [](const SDNode *V) {
4556 return V->getOpcode() == ISD::ADD ||
4557 V->getOpcode() == RISCVISD::ADD_VL;
4558 }))
4559 return false;
4560 break;
4561 default:
4562 return false;
4563 }
4564 }
4565
4566 return selectImm64IfCheaper(-Imm, Imm, N, Val);
4567}
4568
4570 if (!isa<ConstantSDNode>(N))
4571 return false;
4572 int64_t Imm = cast<ConstantSDNode>(N)->getSExtValue();
4573
4574 // For 32-bit signed constants, we can only substitute LUI+ADDI with LUI.
4575 if (isInt<32>(Imm) && ((Imm & 0xfff) != 0xfff || Imm == -1))
4576 return false;
4577
4578 // Abandon this transform if the constant is needed elsewhere.
4579 for (const SDNode *U : N->users()) {
4580 switch (U->getOpcode()) {
4581 case ISD::AND:
4582 case ISD::OR:
4583 case ISD::XOR:
4584 if (!(Subtarget->hasStdExtZbb() || Subtarget->hasStdExtZbkb()))
4585 return false;
4586 break;
4587 case RISCVISD::VMV_V_X_VL:
4588 if (!Subtarget->hasStdExtZvkb())
4589 return false;
4590 if (!all_of(U->users(), [](const SDNode *V) {
4591 return V->getOpcode() == ISD::AND ||
4592 V->getOpcode() == RISCVISD::AND_VL;
4593 }))
4594 return false;
4595 break;
4596 default:
4597 return false;
4598 }
4599 }
4600
4601 if (isInt<32>(Imm)) {
4602 Val =
4603 selectImm(CurDAG, SDLoc(N), N->getSimpleValueType(0), ~Imm, *Subtarget);
4604 return true;
4605 }
4606
4607 // For 64-bit constants, the instruction sequences get complex,
4608 // so we select inverted only if it's cheaper.
4609 return selectImm64IfCheaper(~Imm, Imm, N, Val);
4610}
4611
4612static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo,
4613 unsigned Bits,
4614 const TargetInstrInfo *TII) {
4615 unsigned MCOpcode = RISCV::getRVVMCOpcode(User->getMachineOpcode());
4616
4617 if (!MCOpcode)
4618 return false;
4619
4620 const MCInstrDesc &MCID = TII->get(User->getMachineOpcode());
4621 const uint64_t TSFlags = MCID.TSFlags;
4622 if (!RISCVII::hasSEWOp(TSFlags))
4623 return false;
4624 assert(RISCVII::hasVLOp(TSFlags));
4625
4626 unsigned ChainOpIdx = User->getNumOperands() - 1;
4627 bool HasChainOp = User->getOperand(ChainOpIdx).getValueType() == MVT::Other;
4628 bool HasVecPolicyOp = RISCVII::hasVecPolicyOp(TSFlags);
4629 unsigned VLIdx = User->getNumOperands() - HasVecPolicyOp - HasChainOp - 2;
4630 const unsigned Log2SEW = User->getConstantOperandVal(VLIdx + 1);
4631
4632 if (UserOpNo == VLIdx)
4633 return false;
4634
4635 auto NumDemandedBits =
4636 RISCV::getVectorLowDemandedScalarBits(MCOpcode, Log2SEW);
4637 return NumDemandedBits && Bits >= *NumDemandedBits;
4638}
4639
4640// Return true if all users of this SDNode* only consume the lower \p Bits.
4641// This can be used to form W instructions for add/sub/mul/shl even when the
4642// root isn't a sext_inreg. This can allow the ADDW/SUBW/MULW/SLLIW to CSE if
4643// SimplifyDemandedBits has made it so some users see a sext_inreg and some
4644// don't. The sext_inreg+add/sub/mul/shl will get selected, but still leave
4645// the add/sub/mul/shl to become non-W instructions. By checking the users we
4646// may be able to use a W instruction and CSE with the other instruction if
4647// this has happened. We could try to detect that the CSE opportunity exists
4648// before doing this, but that would be more complicated.
4650 const unsigned Depth) const {
4651 assert((Node->getOpcode() == ISD::ADD || Node->getOpcode() == ISD::SUB ||
4652 Node->getOpcode() == ISD::MUL || Node->getOpcode() == ISD::SHL ||
4653 Node->getOpcode() == ISD::SRL || Node->getOpcode() == ISD::AND ||
4654 Node->getOpcode() == ISD::OR || Node->getOpcode() == ISD::XOR ||
4655 Node->getOpcode() == ISD::SIGN_EXTEND_INREG ||
4656 isa<ConstantSDNode>(Node) || Depth != 0) &&
4657 "Unexpected opcode");
4658
4660 return false;
4661
4662 // The PatFrags that call this may run before RISCVGenDAGISel.inc has checked
4663 // the VT. Ensure the type is scalar to avoid wasting time on vectors.
4664 if (Depth == 0 && !Node->getValueType(0).isScalarInteger())
4665 return false;
4666
4667 for (SDUse &Use : Node->uses()) {
4668 SDNode *User = Use.getUser();
4669 // Users of this node should have already been instruction selected
4670 if (!User->isMachineOpcode())
4671 return false;
4672
4673 // TODO: Add more opcodes?
4674 switch (User->getMachineOpcode()) {
4675 default:
4677 break;
4678 return false;
4679 case RISCV::ADDW:
4680 case RISCV::ADDIW:
4681 case RISCV::SUBW:
4682 case RISCV::MULW:
4683 case RISCV::SLLW:
4684 case RISCV::SLLIW:
4685 case RISCV::SRAW:
4686 case RISCV::SRAIW:
4687 case RISCV::SRLW:
4688 case RISCV::SRLIW:
4689 case RISCV::DIVW:
4690 case RISCV::DIVUW:
4691 case RISCV::REMW:
4692 case RISCV::REMUW:
4693 case RISCV::ROLW:
4694 case RISCV::RORW:
4695 case RISCV::RORIW:
4696 case RISCV::CLSW:
4697 case RISCV::CLZW:
4698 case RISCV::CTZW:
4699 case RISCV::CPOPW:
4700 case RISCV::SLLI_UW:
4701 case RISCV::ABSW:
4702 case RISCV::FMV_W_X:
4703 case RISCV::FCVT_H_W:
4704 case RISCV::FCVT_H_W_INX:
4705 case RISCV::FCVT_H_WU:
4706 case RISCV::FCVT_H_WU_INX:
4707 case RISCV::FCVT_S_W:
4708 case RISCV::FCVT_S_W_INX:
4709 case RISCV::FCVT_S_WU:
4710 case RISCV::FCVT_S_WU_INX:
4711 case RISCV::FCVT_D_W:
4712 case RISCV::FCVT_D_W_INX:
4713 case RISCV::FCVT_D_WU:
4714 case RISCV::FCVT_D_WU_INX:
4715 case RISCV::TH_REVW:
4716 case RISCV::TH_SRRIW:
4717 if (Bits >= 32)
4718 break;
4719 return false;
4720 case RISCV::SLL:
4721 case RISCV::SRA:
4722 case RISCV::SRL:
4723 case RISCV::ROL:
4724 case RISCV::ROR:
4725 case RISCV::BSET:
4726 case RISCV::BCLR:
4727 case RISCV::BINV:
4728 case RISCV::BEXT:
4729 // Shift amount and bit index operands only use log2(Xlen) bits.
4730 if (Use.getOperandNo() == 1 && Bits >= Log2_32(Subtarget->getXLen()))
4731 break;
4732 return false;
4733 case RISCV::SLLI:
4734 // SLLI only uses the lower (XLen - ShAmt) bits.
4735 if (Bits >= Subtarget->getXLen() - User->getConstantOperandVal(1))
4736 break;
4737 return false;
4738 case RISCV::ANDI:
4739 if (Bits >= (unsigned)llvm::bit_width(User->getConstantOperandVal(1)))
4740 break;
4741 goto RecCheck;
4742 case RISCV::ORI: {
4743 uint64_t Imm = cast<ConstantSDNode>(User->getOperand(1))->getSExtValue();
4744 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(~Imm))
4745 break;
4746 [[fallthrough]];
4747 }
4748 case RISCV::AND:
4749 case RISCV::OR:
4750 case RISCV::XOR:
4751 case RISCV::XORI:
4752 case RISCV::ANDN:
4753 case RISCV::ORN:
4754 case RISCV::XNOR:
4755 case RISCV::SH1ADD:
4756 case RISCV::SH2ADD:
4757 case RISCV::SH3ADD:
4758 case RISCV::PseudoMERGE:
4759 RecCheck:
4760 if (hasAllNBitUsers(User, Bits, Depth + 1))
4761 break;
4762 return false;
4763 case RISCV::SRLI: {
4764 unsigned ShAmt = User->getConstantOperandVal(1);
4765 // If we are shifting right by less than Bits, and users don't demand any
4766 // bits that were shifted into [Bits-1:0], then we can consider this as an
4767 // N-Bit user.
4768 if (Bits > ShAmt && hasAllNBitUsers(User, Bits - ShAmt, Depth + 1))
4769 break;
4770 return false;
4771 }
4772 case RISCV::SEXT_B:
4773 case RISCV::PACKH:
4774 if (Bits >= 8)
4775 break;
4776 return false;
4777 case RISCV::SEXT_H:
4778 case RISCV::FMV_H_X:
4779 case RISCV::ZEXT_H_RV32:
4780 case RISCV::ZEXT_H_RV64:
4781 case RISCV::PACKW:
4782 if (Bits >= 16)
4783 break;
4784 return false;
4785 case RISCV::PACK:
4786 if (Bits >= (Subtarget->getXLen() / 2))
4787 break;
4788 return false;
4789 case RISCV::PPAIRE_H:
4790 // If only the lower 32-bits of the result are used, then only the
4791 // lower 16 bits of the inputs are used.
4792 if (Bits >= 16 && hasAllNBitUsers(User, 32, Depth + 1))
4793 break;
4794 return false;
4795 case RISCV::ADD_UW:
4796 case RISCV::SH1ADD_UW:
4797 case RISCV::SH2ADD_UW:
4798 case RISCV::SH3ADD_UW:
4799 // The first operand to add.uw/shXadd.uw is implicitly zero extended from
4800 // 32 bits.
4801 if (Use.getOperandNo() == 0 && Bits >= 32)
4802 break;
4803 return false;
4804 case RISCV::SB:
4805 if (Use.getOperandNo() == 0 && Bits >= 8)
4806 break;
4807 return false;
4808 case RISCV::SH:
4809 if (Use.getOperandNo() == 0 && Bits >= 16)
4810 break;
4811 return false;
4812 case RISCV::SW:
4813 if (Use.getOperandNo() == 0 && Bits >= 32)
4814 break;
4815 return false;
4816 case RISCV::TH_EXT:
4817 case RISCV::TH_EXTU: {
4818 unsigned Msb = User->getConstantOperandVal(1);
4819 unsigned Lsb = User->getConstantOperandVal(2);
4820 // Behavior of Msb < Lsb is not well documented.
4821 if (Msb >= Lsb && Bits > Msb)
4822 break;
4823 return false;
4824 }
4825 }
4826 }
4827
4828 return true;
4829}
4830
4831// Select a constant that can be represented as (sign_extend(imm5) << imm2).
4833 SDValue &Shl2) {
4834 auto *C = dyn_cast<ConstantSDNode>(N);
4835 if (!C)
4836 return false;
4837
4838 int64_t Offset = C->getSExtValue();
4839 for (unsigned Shift = 0; Shift < 4; Shift++) {
4840 if (isInt<5>(Offset >> Shift) && ((Offset % (1LL << Shift)) == 0)) {
4841 EVT VT = N->getValueType(0);
4842 Simm5 = CurDAG->getSignedTargetConstant(Offset >> Shift, SDLoc(N), VT);
4843 Shl2 = CurDAG->getTargetConstant(Shift, SDLoc(N), VT);
4844 return true;
4845 }
4846 }
4847
4848 return false;
4849}
4850
4851// Select VL as a 5 bit immediate or a value that will become a register. This
4852// allows us to choose between VSETIVLI or VSETVLI later.
4854 auto *C = dyn_cast<ConstantSDNode>(N);
4855 if (C && isUInt<5>(C->getZExtValue())) {
4856 VL = CurDAG->getTargetConstant(C->getZExtValue(), SDLoc(N),
4857 N->getValueType(0));
4858 } else if (C && C->isAllOnes()) {
4859 // Treat all ones as VLMax.
4860 VL = CurDAG->getSignedTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
4861 N->getValueType(0));
4862 } else if (isa<RegisterSDNode>(N) &&
4863 cast<RegisterSDNode>(N)->getReg() == RISCV::X0) {
4864 // All our VL operands use an operand that allows GPRNoX0 or an immediate
4865 // as the register class. Convert X0 to a special immediate to pass the
4866 // MachineVerifier. This is recognized specially by the vsetvli insertion
4867 // pass.
4868 VL = CurDAG->getSignedTargetConstant(RISCV::VLMaxSentinel, SDLoc(N),
4869 N->getValueType(0));
4870 } else {
4871 VL = N;
4872 }
4873
4874 return true;
4875}
4876
4878 if (N.getOpcode() == ISD::INSERT_SUBVECTOR) {
4879 if (!N.getOperand(0).isUndef())
4880 return SDValue();
4881 N = N.getOperand(1);
4882 }
4883 SDValue Splat = N;
4884 if ((Splat.getOpcode() != RISCVISD::VMV_V_X_VL &&
4885 Splat.getOpcode() != RISCVISD::VMV_S_X_VL) ||
4886 !Splat.getOperand(0).isUndef())
4887 return SDValue();
4888 assert(Splat.getNumOperands() == 3 && "Unexpected number of operands");
4889 return Splat;
4890}
4891
4894 if (!Splat)
4895 return false;
4896
4897 SplatVal = Splat.getOperand(1);
4898 return true;
4899}
4900
4902 SelectionDAG &DAG,
4904 std::function<bool(int64_t)> ValidateImm,
4905 bool Decrement = false) {
4907 if (!Splat || !isa<ConstantSDNode>(Splat.getOperand(1)))
4908 return false;
4909
4910 const unsigned SplatEltSize = Splat.getScalarValueSizeInBits();
4911 assert(Subtarget.getXLenVT() == Splat.getOperand(1).getSimpleValueType() &&
4912 "Unexpected splat operand type");
4913
4914 // The semantics of RISCVISD::VMV_V_X_VL is that when the operand
4915 // type is wider than the resulting vector element type: an implicit
4916 // truncation first takes place. Therefore, perform a manual
4917 // truncation/sign-extension in order to ignore any truncated bits and catch
4918 // any zero-extended immediate.
4919 // For example, we wish to match (i8 -1) -> (XLenVT 255) as a simm5 by first
4920 // sign-extending to (XLenVT -1).
4921 APInt SplatConst = Splat.getConstantOperandAPInt(1).sextOrTrunc(SplatEltSize);
4922
4923 int64_t SplatImm = SplatConst.getSExtValue();
4924
4925 if (!ValidateImm(SplatImm))
4926 return false;
4927
4928 if (Decrement)
4929 SplatImm -= 1;
4930
4931 SplatVal =
4932 DAG.getSignedTargetConstant(SplatImm, SDLoc(N), Subtarget.getXLenVT());
4933 return true;
4934}
4935
4937 return selectVSplatImmHelper(N, SplatVal, *CurDAG, *Subtarget,
4938 [](int64_t Imm) { return isInt<5>(Imm); });
4939}
4940
4942 return selectVSplatImmHelper(
4943 N, SplatVal, *CurDAG, *Subtarget,
4944 [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4945 /*Decrement=*/true);
4946}
4947
4949 return selectVSplatImmHelper(
4950 N, SplatVal, *CurDAG, *Subtarget,
4951 [](int64_t Imm) { return Imm >= -15 && Imm <= 16; },
4952 /*Decrement=*/false);
4953}
4954
4956 SDValue &SplatVal) {
4957 return selectVSplatImmHelper(
4958 N, SplatVal, *CurDAG, *Subtarget,
4959 [](int64_t Imm) { return Imm != 0 && Imm >= -15 && Imm <= 16; },
4960 /*Decrement=*/true);
4961}
4962
4964 SDValue &SplatVal) {
4965 return selectVSplatImmHelper(
4966 N, SplatVal, *CurDAG, *Subtarget,
4967 [Bits](int64_t Imm) { return isUIntN(Bits, Imm); });
4968}
4969
4972 return Splat && selectNegImm(Splat.getOperand(1), SplatVal);
4973}
4974
4976 auto IsExtOrTrunc = [](SDValue N) {
4977 switch (N->getOpcode()) {
4978 case ISD::SIGN_EXTEND:
4979 case ISD::ZERO_EXTEND:
4980 // There's no passthru on these _VL nodes so any VL/mask is ok, since any
4981 // inactive elements will be undef.
4982 case RISCVISD::TRUNCATE_VECTOR_VL:
4983 case RISCVISD::VSEXT_VL:
4984 case RISCVISD::VZEXT_VL:
4985 return true;
4986 default:
4987 return false;
4988 }
4989 };
4990
4991 // We can have multiple nested nodes, so unravel them all if needed.
4992 while (IsExtOrTrunc(N)) {
4993 if (!N.hasOneUse() || N.getScalarValueSizeInBits() < 8)
4994 return false;
4995 N = N->getOperand(0);
4996 }
4997
4998 return selectVSplat(N, SplatVal);
4999}
5000
5002 // Allow bitcasts from XLenVT -> FP.
5003 if (N.getOpcode() == ISD::BITCAST &&
5004 N.getOperand(0).getValueType() == Subtarget->getXLenVT()) {
5005 Imm = N.getOperand(0);
5006 return true;
5007 }
5008 // Allow moves from XLenVT to FP.
5009 if (N.getOpcode() == RISCVISD::FMV_H_X ||
5010 N.getOpcode() == RISCVISD::FMV_W_X_RV64) {
5011 Imm = N.getOperand(0);
5012 return true;
5013 }
5014
5015 // Otherwise, look for FP constants that can materialized with scalar int.
5017 if (!CFP)
5018 return false;
5019 const APFloat &APF = CFP->getValueAPF();
5020 // td can handle +0.0 already.
5021 if (APF.isPosZero())
5022 return false;
5023
5024 MVT VT = CFP->getSimpleValueType(0);
5025
5026 MVT XLenVT = Subtarget->getXLenVT();
5027 if (VT == MVT::f64 && !Subtarget->is64Bit()) {
5028 assert(APF.isNegZero() && "Unexpected constant.");
5029 return false;
5030 }
5031 SDLoc DL(N);
5032 Imm = selectImm(CurDAG, DL, XLenVT, APF.bitcastToAPInt().getSExtValue(),
5033 *Subtarget);
5034 return true;
5035}
5036
5038 SDValue &Imm) {
5039 if (auto *C = dyn_cast<ConstantSDNode>(N)) {
5040 int64_t ImmVal = SignExtend64(C->getSExtValue(), Width);
5041
5042 if (!isInt<5>(ImmVal))
5043 return false;
5044
5045 Imm = CurDAG->getSignedTargetConstant(ImmVal, SDLoc(N),
5046 Subtarget->getXLenVT());
5047 return true;
5048 }
5049
5050 return false;
5051}
5052
5053// Match XOR with a VMSET_VL operand. Return the other operand.
5055 if (N.getOpcode() != ISD::XOR)
5056 return false;
5057
5058 if (N.getOperand(0).getOpcode() == RISCVISD::VMSET_VL) {
5059 Res = N.getOperand(1);
5060 return true;
5061 }
5062
5063 if (N.getOperand(1).getOpcode() == RISCVISD::VMSET_VL) {
5064 Res = N.getOperand(0);
5065 return true;
5066 }
5067
5068 return false;
5069}
5070
5071// Match VMXOR_VL with a VMSET_VL operand. Making sure that that VL operand
5072// matches the parent's VL. Return the other operand of the VMXOR_VL.
5074 SDValue &Res) {
5075 if (N.getOpcode() != RISCVISD::VMXOR_VL)
5076 return false;
5077
5078 assert(Parent &&
5079 (Parent->getOpcode() == RISCVISD::VMAND_VL ||
5080 Parent->getOpcode() == RISCVISD::VMOR_VL ||
5081 Parent->getOpcode() == RISCVISD::VMXOR_VL) &&
5082 "Unexpected parent");
5083
5084 // The VL should match the parent.
5085 if (Parent->getOperand(2) != N->getOperand(2))
5086 return false;
5087
5088 if (N.getOperand(0).getOpcode() == RISCVISD::VMSET_VL) {
5089 Res = N.getOperand(1);
5090 return true;
5091 }
5092
5093 if (N.getOperand(1).getOpcode() == RISCVISD::VMSET_VL) {
5094 Res = N.getOperand(0);
5095 return true;
5096 }
5097
5098 return false;
5099}
5100
5101// Try to remove sext.w if the input is a W instruction or can be made into
5102// a W instruction cheaply.
5103bool RISCVDAGToDAGISel::doPeepholeSExtW(SDNode *N) {
5104 // Look for the sext.w pattern, addiw rd, rs1, 0.
5105 if (N->getMachineOpcode() != RISCV::ADDIW ||
5106 !isNullConstant(N->getOperand(1)))
5107 return false;
5108
5109 SDValue N0 = N->getOperand(0);
5110 if (!N0.isMachineOpcode())
5111 return false;
5112
5113 switch (N0.getMachineOpcode()) {
5114 default:
5115 break;
5116 case RISCV::ADD:
5117 case RISCV::ADDI:
5118 case RISCV::SUB:
5119 case RISCV::MUL:
5120 case RISCV::SLLI: {
5121 // Convert sext.w+add/sub/mul to their W instructions. This will create
5122 // a new independent instruction. This improves latency.
5123 unsigned Opc;
5124 switch (N0.getMachineOpcode()) {
5125 default:
5126 llvm_unreachable("Unexpected opcode!");
5127 case RISCV::ADD: Opc = RISCV::ADDW; break;
5128 case RISCV::ADDI: Opc = RISCV::ADDIW; break;
5129 case RISCV::SUB: Opc = RISCV::SUBW; break;
5130 case RISCV::MUL: Opc = RISCV::MULW; break;
5131 case RISCV::SLLI: Opc = RISCV::SLLIW; break;
5132 }
5133
5134 SDValue N00 = N0.getOperand(0);
5135 SDValue N01 = N0.getOperand(1);
5136
5137 // Shift amount needs to be uimm5.
5138 if (N0.getMachineOpcode() == RISCV::SLLI &&
5139 !isUInt<5>(cast<ConstantSDNode>(N01)->getSExtValue()))
5140 break;
5141
5142 SDNode *Result =
5143 CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
5144 N00, N01);
5145 ReplaceUses(N, Result);
5146 return true;
5147 }
5148 case RISCV::ADDW:
5149 case RISCV::ADDIW:
5150 case RISCV::SUBW:
5151 case RISCV::MULW:
5152 case RISCV::SLLIW:
5153 case RISCV::PACKW:
5154 case RISCV::TH_MULAW:
5155 case RISCV::TH_MULAH:
5156 case RISCV::TH_MULSW:
5157 case RISCV::TH_MULSH:
5158 if (N0.getValueType() == MVT::i32)
5159 break;
5160
5161 // Result is already sign extended just remove the sext.w.
5162 // NOTE: We only handle the nodes that are selected with hasAllWUsers.
5163 ReplaceUses(N, N0.getNode());
5164 return true;
5165 }
5166
5167 return false;
5168}
5169
5170static bool usesAllOnesMask(SDValue MaskOp) {
5171 const auto IsVMSet = [](unsigned Opc) {
5172 return Opc == RISCV::PseudoVMSET_M_B1 || Opc == RISCV::PseudoVMSET_M_B16 ||
5173 Opc == RISCV::PseudoVMSET_M_B2 || Opc == RISCV::PseudoVMSET_M_B32 ||
5174 Opc == RISCV::PseudoVMSET_M_B4 || Opc == RISCV::PseudoVMSET_M_B64 ||
5175 Opc == RISCV::PseudoVMSET_M_B8;
5176 };
5177
5178 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
5179 // undefined behaviour if it's the wrong bitwidth, so we could choose to
5180 // assume that it's all-ones? Same applies to its VL.
5181 return MaskOp->isMachineOpcode() && IsVMSet(MaskOp.getMachineOpcode());
5182}
5183
5185 if (!V.isMachineOpcode())
5186 return false;
5187 if (V.getMachineOpcode() == TargetOpcode::REG_SEQUENCE) {
5188 for (unsigned I = 1; I < V.getNumOperands(); I += 2)
5189 if (!isImplicitDef(V.getOperand(I)))
5190 return false;
5191 return true;
5192 }
5193 return V.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF;
5194}
5195
5196// Optimize masked RVV pseudo instructions with a known all-ones mask to their
5197// corresponding "unmasked" pseudo versions.
5198bool RISCVDAGToDAGISel::doPeepholeMaskedRVV(MachineSDNode *N) {
5199 const RISCV::RISCVMaskedPseudoInfo *I =
5200 RISCV::getMaskedPseudoInfo(N->getMachineOpcode());
5201 if (!I)
5202 return false;
5203
5204 unsigned MaskOpIdx = I->MaskOpIdx;
5205 if (!usesAllOnesMask(N->getOperand(MaskOpIdx)))
5206 return false;
5207
5208 // There are two classes of pseudos in the table - compares and
5209 // everything else. See the comment on RISCVMaskedPseudo for details.
5210 const unsigned Opc = I->UnmaskedPseudo;
5211 const MCInstrDesc &MCID = TII->get(Opc);
5212 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(MCID);
5213
5214 const MCInstrDesc &MaskedMCID = TII->get(N->getMachineOpcode());
5215 const bool MaskedHasPassthru = RISCVII::isFirstDefTiedToFirstUse(MaskedMCID);
5216
5217 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
5219 "Unmasked pseudo has policy but masked pseudo doesn't?");
5220 assert(RISCVII::hasVecPolicyOp(MCID.TSFlags) == HasPassthru &&
5221 "Unexpected pseudo structure");
5222 assert(!(HasPassthru && !MaskedHasPassthru) &&
5223 "Unmasked pseudo has passthru but masked pseudo doesn't?");
5224
5226 // Skip the passthru operand at index 0 if the unmasked don't have one.
5227 bool ShouldSkip = !HasPassthru && MaskedHasPassthru;
5228 bool DropPolicy = !RISCVII::hasVecPolicyOp(MCID.TSFlags) &&
5229 RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags);
5230 bool HasChainOp =
5231 N->getOperand(N->getNumOperands() - 1).getValueType() == MVT::Other;
5232 unsigned LastOpNum = N->getNumOperands() - 1 - HasChainOp;
5233 for (unsigned I = ShouldSkip, E = N->getNumOperands(); I != E; I++) {
5234 // Skip the mask
5235 SDValue Op = N->getOperand(I);
5236 if (I == MaskOpIdx)
5237 continue;
5238 if (DropPolicy && I == LastOpNum)
5239 continue;
5240 Ops.push_back(Op);
5241 }
5242
5243 MachineSDNode *Result =
5244 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
5245
5246 if (!N->memoperands_empty())
5247 CurDAG->setNodeMemRefs(Result, N->memoperands());
5248
5249 Result->setFlags(N->getFlags());
5250 ReplaceUses(N, Result);
5251
5252 return true;
5253}
5254
5255/// If our passthru is an implicit_def, use noreg instead. This side
5256/// steps issues with MachineCSE not being able to CSE expressions with
5257/// IMPLICIT_DEF operands while preserving the semantic intent. See
5258/// pr64282 for context. Note that this transform is the last one
5259/// performed at ISEL DAG to DAG.
5260bool RISCVDAGToDAGISel::doPeepholeNoRegPassThru() {
5261 bool MadeChange = false;
5262 SelectionDAG::allnodes_iterator Position = CurDAG->allnodes_end();
5263
5264 while (Position != CurDAG->allnodes_begin()) {
5265 SDNode *N = &*--Position;
5266 if (N->use_empty() || !N->isMachineOpcode())
5267 continue;
5268
5269 const unsigned Opc = N->getMachineOpcode();
5270 if (!RISCVVPseudosTable::getPseudoInfo(Opc) ||
5272 !isImplicitDef(N->getOperand(0)))
5273 continue;
5274
5276 Ops.push_back(CurDAG->getRegister(RISCV::NoRegister, N->getValueType(0)));
5277 for (unsigned I = 1, E = N->getNumOperands(); I != E; I++) {
5278 SDValue Op = N->getOperand(I);
5279 Ops.push_back(Op);
5280 }
5281
5282 MachineSDNode *Result =
5283 CurDAG->getMachineNode(Opc, SDLoc(N), N->getVTList(), Ops);
5284 Result->setFlags(N->getFlags());
5285 CurDAG->setNodeMemRefs(Result, cast<MachineSDNode>(N)->memoperands());
5286 ReplaceUses(N, Result);
5287 MadeChange = true;
5288 }
5289 return MadeChange;
5290}
5291
5292
5293// This pass converts a legalized DAG into a RISCV-specific DAG, ready
5294// for instruction scheduling.
5299
5303
5305
5310
static SDValue Widen(SelectionDAG *CurDAG, SDValue N)
return SDValue()
const AArch64Subtarget & Subtarget
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define DEBUG_TYPE
static constexpr Value * getValue(Ty &ValueOrUse)
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define I(x, y, z)
Definition MD5.cpp:57
Register const TargetRegisterInfo * TRI
static MCRegister getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
#define P(N)
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition PassSupport.h:56
static bool getVal(MDTuple *MD, const char *Key, uint64_t &Val)
static bool usesAllOnesMask(SDValue MaskOp)
static Register getTileReg(uint64_t TileNum)
static SDValue selectImm(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, int64_t Imm, const RISCVSubtarget &Subtarget)
static bool isRegRegScaleLoadOrStore(SDNode *User, SDValue Add, const RISCVSubtarget &Subtarget)
Return true if this a load/store that we have a RegRegScale instruction for.
static std::pair< SDValue, SDValue > extractGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, SDValue Pair)
#define CASE_VMNAND_VMSET_OPCODES(lmulenum, suffix)
static bool isWorthFoldingAdd(SDValue Add)
static SDValue selectImmSeq(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, RISCVMatInt::InstSeq &Seq)
static bool isImplicitDef(SDValue V)
#define CASE_VMXOR_VMANDN_VMOR_OPCODES(lmulenum, suffix)
static bool selectVSplatImmHelper(SDValue N, SDValue &SplatVal, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, std::function< bool(int64_t)> ValidateImm, bool Decrement=false)
static unsigned getSegInstNF(unsigned Intrinsic)
static bool isWorthFoldingIntoRegRegScale(const RISCVSubtarget &Subtarget, SDValue Add, SDValue Shift=SDValue())
Is it profitable to fold this Add into RegRegScale load/store.
static bool vectorPseudoHasAllNBitUsers(SDNode *User, unsigned UserOpNo, unsigned Bits, const TargetInstrInfo *TII)
static bool selectConstantAddr(SelectionDAG *CurDAG, const SDLoc &DL, const MVT VT, const RISCVSubtarget *Subtarget, SDValue Addr, SDValue &Base, SDValue &Offset, bool IsPrefetch=false)
#define INST_ALL_NF_CASE_WITH_FF(NAME)
#define CASE_VMSLT_OPCODES(lmulenum, suffix)
static SDValue buildGPRPair(SelectionDAG *CurDAG, const SDLoc &DL, MVT VT, SDValue Lo, SDValue Hi)
bool isRegImmLoadOrStore(SDNode *User, SDValue Add)
static SDValue findVSplat(SDValue N)
static bool isApplicableToPLIOrPLUI(int Val)
#define INST_ALL_NF_CASE(NAME)
int64_t C1
MachineInstr *& Lo
SI Fold Operands
#define LLVM_DEBUG(...)
Definition Debug.h:119
#define PASS_NAME
DEMANGLE_DUMP_METHOD void dump() const
bool isZero() const
Definition APFloat.h:1579
APInt bitcastToAPInt() const
Definition APFloat.h:1475
bool isPosZero() const
Definition APFloat.h:1594
bool isNegZero() const
Definition APFloat.h:1595
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
LLVM_ABI bool isSplat(unsigned SplatSizeInBits) const
Check if the APInt consists of a repeated bit pattern.
Definition APInt.cpp:627
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition APInt.h:215
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
const APFloat & getValueAPF() const
uint64_t getZExtValue() const
int64_t getSExtValue() const
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
FunctionPass class - This class is used to implement most global optimizations.
Definition Pass.h:312
This class is used to form a handle around another node that is persistent and is updated across invo...
const SDValue & getValue() const
static StringRef getMemConstraintName(ConstraintCode C)
Definition InlineAsm.h:485
This class is used to represent ISD::LOAD nodes.
Describe properties that are true of each instruction in the target description file.
Machine Value Type.
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
bool isFixedLengthVector() const
ElementCount getVectorElementCount() const
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
MVT getVectorElementType() const
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MONonTemporal
The memory access is non-temporal.
void setFlags(Flags f)
Bitwise OR the current flags with the given flags.
An SDNode that represents everything that will be needed to construct a MachineInstr.
RISCVDAGToDAGISelLegacy(RISCVTargetMachine &TargetMachine, CodeGenOptLevel OptLevel)
bool selectSExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectNegImm(SDValue N, SDValue &Val)
bool selectZExtBits(SDValue N, unsigned Bits, SDValue &Val)
bool selectSHXADD_UWOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD_UW.
bool areOffsetsWithinAlignment(SDValue Addr, Align Alignment)
bool hasAllNBitUsers(SDNode *Node, unsigned Bits, const unsigned Depth=0) const
bool SelectAddrRegImmLsb00000(SDValue Addr, SDValue &Base, SDValue &Offset)
Similar to SelectAddrRegImm, except that the least significant 5 bits of Offset should be all zeros.
bool SelectBrindRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
Similar to SelectAddrRegImm, but only matches a register, or a register plus a simm12 offset.
bool selectZExtImm32(SDValue N, SDValue &Val)
bool SelectAddrRegReg(SDValue Addr, SDValue &Base, SDValue &Offset)
bool selectVMNOT_VLOp(SDNode *Parent, SDValue N, SDValue &Res)
void selectVSXSEG(SDNode *Node, unsigned NF, bool IsMasked, bool IsOrdered)
void selectVLSEGFF(SDNode *Node, unsigned NF, bool IsMasked)
bool selectVSplatSimm5Plus1NoDec(SDValue N, SDValue &SplatVal)
bool SelectAddrRegImm26(SDValue Addr, SDValue &Base, SDValue &Offset)
Similar to SelectAddrRegImm, except that the offset is a 26-bit signed immediate.
bool selectSimm5Shl2(SDValue N, SDValue &Simm5, SDValue &Shl2)
void selectSF_VC_X_SE(SDNode *Node)
bool orDisjoint(const SDNode *Node) const
bool tryWideningMulAcc(SDNode *Node, const SDLoc &DL)
bool selectLow8BitsVSplat(SDValue N, SDValue &SplatVal)
bool hasAllHUsers(SDNode *Node) const
bool SelectInlineAsmMemoryOperand(const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, std::vector< SDValue > &OutOps) override
SelectInlineAsmMemoryOperand - Select the specified address as a target addressing mode,...
bool selectVSplatSimm5(SDValue N, SDValue &SplatVal)
bool selectSETCC(SDValue N, ISD::CondCode ExpectedCCVal, SDValue &Val, bool OneUse)
RISC-V doesn't have general instructions for integer setne/seteq, but we can check for equality with ...
bool selectRVVSimm5(SDValue N, unsigned Width, SDValue &Imm)
bool SelectAddrFrameIndex(SDValue Addr, SDValue &Base, SDValue &Offset)
bool tryUnsignedBitfieldInsertInZero(SDNode *Node, const SDLoc &DL, MVT VT, SDValue X, unsigned Msb, unsigned Lsb)
bool hasAllWUsers(SDNode *Node) const
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
bool selectInvLogicImm(SDValue N, SDValue &Val)
bool SelectAddrRegImm(SDValue Addr, SDValue &Base, SDValue &Offset)
bool SelectAddrRegRegScale(SDValue Addr, ArrayRef< unsigned > Amounts, SDValue &Base, SDValue &Index, SDValue &Scale)
void Select(SDNode *Node) override
Main hook for targets to transform nodes into machine nodes.
void selectXSfmmVSET(SDNode *Node)
bool trySignedBitfieldInsertInSign(SDNode *Node)
bool selectVSplat(SDValue N, SDValue &SplatVal)
void addVectorLoadStoreOperands(SDNode *Node, unsigned SEWImm, const SDLoc &DL, unsigned CurOp, bool IsMasked, bool IsStridedOrIndexed, SmallVectorImpl< SDValue > &Operands, bool IsLoad=false, MVT *IndexVT=nullptr)
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool SelectAddrRegImm9(SDValue Addr, SDValue &Base, SDValue &Offset)
Similar to SelectAddrRegImm, except that the offset is restricted to uimm9.
bool selectScalarFPAsInt(SDValue N, SDValue &Imm)
bool hasAllBUsers(SDNode *Node) const
void selectVLSEG(SDNode *Node, unsigned NF, bool IsMasked, bool IsStrided)
bool tryShrinkShlLogicImm(SDNode *Node)
void selectVSETVLI(SDNode *Node)
bool selectVLOp(SDValue N, SDValue &VL)
bool trySignedBitfieldExtract(SDNode *Node)
bool selectVSplatSimm5Plus1(SDValue N, SDValue &SplatVal)
bool SelectAddrRegZextRegScale(SDValue Addr, ArrayRef< unsigned > Amounts, unsigned Bits, SDValue &Base, SDValue &Index, SDValue &Scale)
bool selectVMNOTOp(SDValue N, SDValue &Res)
void selectVSSEG(SDNode *Node, unsigned NF, bool IsMasked, bool IsStrided)
bool selectVSplatImm64Neg(SDValue N, SDValue &SplatVal)
bool selectVSplatSimm5Plus1NonZero(SDValue N, SDValue &SplatVal)
bool tryUnsignedBitfieldExtract(SDNode *Node, const SDLoc &DL, MVT VT, SDValue X, unsigned Msb, unsigned Lsb)
void selectVLXSEG(SDNode *Node, unsigned NF, bool IsMasked, bool IsOrdered)
bool selectShiftMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt)
bool selectSHXADDOp(SDValue N, unsigned ShAmt, SDValue &Val)
Look for various patterns that can be done with a SHL that can be folded into a SHXADD.
bool tryIndexedLoad(SDNode *Node)
bool selectVSplatUimm(SDValue N, unsigned Bits, SDValue &SplatVal)
RISCVISelDAGToDAGPass(RISCVTargetMachine &TM, CodeGenOptLevel OptLevel)
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
static unsigned getRegClassIDForVecVT(MVT VT)
static RISCVVType::VLMUL getLMUL(MVT VT)
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isMachineOpcode() const
Test if this node has a post-isel opcode, directly corresponding to a MachineInstr opcode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
iterator_range< user_iterator > users()
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
EVT getValueType() const
Return the ValueType of the referenced return value.
bool isMachineOpcode() const
const SDValue & getOperand(unsigned i) const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getMachineOpcode() const
unsigned getOpcode() const
SelectionDAGISelLegacy(char &ID, std::unique_ptr< SelectionDAGISel > S)
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
const TargetLowering * TLI
const TargetInstrInfo * TII
void ReplaceUses(SDValue F, SDValue T)
ReplaceUses - replace all uses of the old node F with the use of the new node T.
virtual bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const
IsProfitableToFold - Returns true if it's profitable to fold the specific operand node N of U during ...
static bool IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, CodeGenOptLevel OptLevel, bool IgnoreChains=false)
IsLegalToFold - Returns true if the specific operand node N of U can be folded during instruction sel...
bool selectShiftAmountMask(SDValue N, unsigned ShiftWidth, SDValue &ShAmt) const
Look through a mask on the amount N of a shift that reads only the low Log2(ShiftWidth) bits of its a...
void ReplaceNode(SDNode *F, SDNode *T)
Replace all uses of F with T, then remove F from the DAG.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
static constexpr unsigned MaxRecursionDepth
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
ilist< SDNode >::iterator allnodes_iterator
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
TargetInstrInfo - Interface to description of machine instruction set.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI unsigned getOperandNo() const
Return the operand # of this use in its User.
Definition Use.cpp:35
User * getUser() const
Returns the User that contains this Use.
Definition Use.h:61
Value * getOperand(unsigned i) const
Definition User.h:207
unsigned getNumOperands() const
Definition User.h:229
iterator_range< user_iterator > users()
Definition Value.h:428
#define INT64_MIN
Definition DataTypes.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, val, ptr) This corresponds to "store atomic" instruction.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:688
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
int getIntMatCost(const APInt &Val, unsigned Size, const MCSubtargetInfo &STI, bool CompressionCost, bool FreeZeroes)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
SmallVector< Inst, 8 > InstSeq
Definition RISCVMatInt.h:43
static unsigned decodeVSEW(unsigned VSEW)
LLVM_ABI unsigned encodeXSfmmVType(unsigned SEW, unsigned Widen, bool AltFmt)
LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)
LLVM_ABI unsigned getSEWLMULRatio(unsigned SEW, VLMUL VLMul)
static unsigned decodeTWiden(unsigned TWiden)
LLVM_ABI unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic, bool AltFmt=false)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
std::optional< unsigned > getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW)
static constexpr unsigned RVVBitsPerBlock
static constexpr int64_t VLMaxSentinel
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
static const MachineMemOperand::Flags MONontemporalBit1
InstructionCost Cost
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isStrongerThanMonotonic(AtomicOrdering AO)
FunctionPass * createRISCVISelDagLegacyPass(RISCVTargetMachine &TM, CodeGenOptLevel OptLevel)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
constexpr int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
Definition MathExtras.h:224
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
constexpr bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition MathExtras.h:244
static const MachineMemOperand::Flags MONontemporalBit0
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition MathExtras.h:274
unsigned M1(unsigned Val)
Definition VE.h:394
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
int countl_zero(T Val)
Count number of 0's from the most significant bit to the least stopping at the first 1.
Definition bit.h:263
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:262
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
constexpr T maskTrailingZeros(unsigned N)
Create a bitmask with the N right-most bits set to 0, and all other bits set to 1.
Definition MathExtras.h:95
@ Add
Sum of integers.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
DWARFExpression::Operation Op
unsigned M0(unsigned Val)
Definition VE.h:393
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
Definition MathExtras.h:183
constexpr int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
Definition MathExtras.h:233
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition STLExtras.h:1963
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition Alignment.h:206
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
Definition MathExtras.h:199
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
Implement std::hash so that hash_code can be used in STL containers.
Definition BitVector.h:879
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
This class contains a discriminated union of information about pointers in memory operands,...
MachinePointerInfo getWithOffset(int64_t O) const
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.