LLVM 20.0.0git
RISCVISelLowering.cpp
Go to the documentation of this file.
1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
16#include "RISCV.h"
18#include "RISCVRegisterInfo.h"
19#include "RISCVSubtarget.h"
20#include "RISCVTargetMachine.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
36#include "llvm/IR/IRBuilder.h"
38#include "llvm/IR/IntrinsicsRISCV.h"
41#include "llvm/Support/Debug.h"
47#include <optional>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "riscv-lower"
52
53STATISTIC(NumTailCalls, "Number of tail calls");
54
56 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
57 cl::desc("Give the maximum size (in number of nodes) of the web of "
58 "instructions that we will consider for VW expansion"),
59 cl::init(18));
60
61static cl::opt<bool>
62 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
63 cl::desc("Allow the formation of VW_W operations (e.g., "
64 "VWADD_W) with splat constants"),
65 cl::init(false));
66
68 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
69 cl::desc("Set the minimum number of repetitions of a divisor to allow "
70 "transformation to multiplications by the reciprocal"),
71 cl::init(2));
72
73static cl::opt<int>
75 cl::desc("Give the maximum number of instructions that we will "
76 "use for creating a floating-point immediate value"),
77 cl::init(2));
78
80 const RISCVSubtarget &STI)
81 : TargetLowering(TM), Subtarget(STI) {
82
83 RISCVABI::ABI ABI = Subtarget.getTargetABI();
84 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
85
86 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
87 !Subtarget.hasStdExtF()) {
88 errs() << "Hard-float 'f' ABI can't be used for a target that "
89 "doesn't support the F instruction set extension (ignoring "
90 "target-abi)\n";
92 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
93 !Subtarget.hasStdExtD()) {
94 errs() << "Hard-float 'd' ABI can't be used for a target that "
95 "doesn't support the D instruction set extension (ignoring "
96 "target-abi)\n";
98 }
99
100 switch (ABI) {
101 default:
102 report_fatal_error("Don't know how to lower this ABI");
111 break;
112 }
113
114 MVT XLenVT = Subtarget.getXLenVT();
115
116 // Set up the register classes.
117 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
118
119 if (Subtarget.hasStdExtZfhmin())
120 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
121 if (Subtarget.hasStdExtZfbfmin())
122 addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
123 if (Subtarget.hasStdExtF())
124 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
125 if (Subtarget.hasStdExtD())
126 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
127 if (Subtarget.hasStdExtZhinxmin())
128 addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
129 if (Subtarget.hasStdExtZfinx())
130 addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
131 if (Subtarget.hasStdExtZdinx()) {
132 if (Subtarget.is64Bit())
133 addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
134 else
135 addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
136 }
137
138 static const MVT::SimpleValueType BoolVecVTs[] = {
139 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
140 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
141 static const MVT::SimpleValueType IntVecVTs[] = {
142 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
143 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
144 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
145 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
146 MVT::nxv4i64, MVT::nxv8i64};
147 static const MVT::SimpleValueType F16VecVTs[] = {
148 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
149 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
150 static const MVT::SimpleValueType BF16VecVTs[] = {
151 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
152 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
153 static const MVT::SimpleValueType F32VecVTs[] = {
154 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
155 static const MVT::SimpleValueType F64VecVTs[] = {
156 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
157
158 if (Subtarget.hasVInstructions()) {
159 auto addRegClassForRVV = [this](MVT VT) {
160 // Disable the smallest fractional LMUL types if ELEN is less than
161 // RVVBitsPerBlock.
162 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
163 if (VT.getVectorMinNumElements() < MinElts)
164 return;
165
166 unsigned Size = VT.getSizeInBits().getKnownMinValue();
167 const TargetRegisterClass *RC;
169 RC = &RISCV::VRRegClass;
170 else if (Size == 2 * RISCV::RVVBitsPerBlock)
171 RC = &RISCV::VRM2RegClass;
172 else if (Size == 4 * RISCV::RVVBitsPerBlock)
173 RC = &RISCV::VRM4RegClass;
174 else if (Size == 8 * RISCV::RVVBitsPerBlock)
175 RC = &RISCV::VRM8RegClass;
176 else
177 llvm_unreachable("Unexpected size");
178
179 addRegisterClass(VT, RC);
180 };
181
182 for (MVT VT : BoolVecVTs)
183 addRegClassForRVV(VT);
184 for (MVT VT : IntVecVTs) {
185 if (VT.getVectorElementType() == MVT::i64 &&
186 !Subtarget.hasVInstructionsI64())
187 continue;
188 addRegClassForRVV(VT);
189 }
190
191 if (Subtarget.hasVInstructionsF16Minimal())
192 for (MVT VT : F16VecVTs)
193 addRegClassForRVV(VT);
194
195 if (Subtarget.hasVInstructionsBF16Minimal())
196 for (MVT VT : BF16VecVTs)
197 addRegClassForRVV(VT);
198
199 if (Subtarget.hasVInstructionsF32())
200 for (MVT VT : F32VecVTs)
201 addRegClassForRVV(VT);
202
203 if (Subtarget.hasVInstructionsF64())
204 for (MVT VT : F64VecVTs)
205 addRegClassForRVV(VT);
206
207 if (Subtarget.useRVVForFixedLengthVectors()) {
208 auto addRegClassForFixedVectors = [this](MVT VT) {
209 MVT ContainerVT = getContainerForFixedLengthVector(VT);
210 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
211 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
212 addRegisterClass(VT, TRI.getRegClass(RCID));
213 };
215 if (useRVVForFixedLengthVectorVT(VT))
216 addRegClassForFixedVectors(VT);
217
219 if (useRVVForFixedLengthVectorVT(VT))
220 addRegClassForFixedVectors(VT);
221 }
222 }
223
224 // Compute derived properties from the register classes.
226
228
230 MVT::i1, Promote);
231 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
233 MVT::i1, Promote);
234
235 // TODO: add all necessary setOperationAction calls.
237
242
247 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
250 }
251
253
256
258
260
261 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
262 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
263 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
264
265 if (Subtarget.is64Bit()) {
267
270 MVT::i32, Custom);
272 if (!Subtarget.hasStdExtZbb())
275 Custom);
277 }
278 if (!Subtarget.hasStdExtZmmul()) {
280 } else if (Subtarget.is64Bit()) {
283 } else {
285 }
286
287 if (!Subtarget.hasStdExtM()) {
289 Expand);
290 } else if (Subtarget.is64Bit()) {
292 {MVT::i8, MVT::i16, MVT::i32}, Custom);
293 }
294
297 Expand);
298
300 Custom);
301
302 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
303 if (Subtarget.is64Bit())
305 } else if (Subtarget.hasVendorXTHeadBb()) {
306 if (Subtarget.is64Bit())
309 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
311 } else {
313 }
314
315 // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
316 // pattern match it directly in isel.
318 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
319 Subtarget.hasVendorXTHeadBb())
320 ? Legal
321 : Expand);
322
323 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
325 } else {
326 // Zbkb can use rev8+brev8 to implement bitreverse.
328 Subtarget.hasStdExtZbkb() ? Custom : Expand);
329 }
330
331 if (Subtarget.hasStdExtZbb() ||
332 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
334 Legal);
335 }
336
337 if (Subtarget.hasStdExtZbb() ||
338 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())) {
339 if (Subtarget.is64Bit())
341 } else {
343 }
344
345 if (Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
346 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())) {
347 // We need the custom lowering to make sure that the resulting sequence
348 // for the 32bit case is efficient on 64bit targets.
349 if (Subtarget.is64Bit())
351 } else {
353 }
354
355 if (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()) {
357 } else if (Subtarget.hasShortForwardBranchOpt()) {
358 // We can use PseudoCCSUB to implement ABS.
360 } else if (Subtarget.is64Bit()) {
362 }
363
364 if (!Subtarget.hasVendorXTHeadCondMov())
366
367 static const unsigned FPLegalNodeTypes[] = {
374
375 static const ISD::CondCode FPCCToExpand[] = {
379
380 static const unsigned FPOpToExpand[] = {
382 ISD::FREM};
383
384 static const unsigned FPRndMode[] = {
387
388 if (Subtarget.hasStdExtZfhminOrZhinxmin())
390
391 static const unsigned ZfhminZfbfminPromoteOps[] = {
401
402 if (Subtarget.hasStdExtZfbfmin()) {
411 setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
413 // FIXME: Need to promote bf16 FCOPYSIGN to f32, but the
414 // DAGCombiner::visitFP_ROUND probably needs improvements first.
416 }
417
418 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
419 if (Subtarget.hasStdExtZfhOrZhinx()) {
420 setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
421 setOperationAction(FPRndMode, MVT::f16,
422 Subtarget.hasStdExtZfa() ? Legal : Custom);
425 } else {
426 setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
429 MVT::f16, Legal);
430 // FIXME: Need to promote f16 FCOPYSIGN to f32, but the
431 // DAGCombiner::visitFP_ROUND probably needs improvements first.
433 }
434
437 setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
440
442 Subtarget.hasStdExtZfa() ? Legal : Promote);
447 MVT::f16, Promote);
448
449 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
450 // complete support for all operations in LegalizeDAG.
455 MVT::f16, Promote);
456
457 // We need to custom promote this.
458 if (Subtarget.is64Bit())
460
462 Subtarget.hasStdExtZfa() ? Legal : Custom);
463 }
464
465 if (Subtarget.hasStdExtFOrZfinx()) {
466 setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
467 setOperationAction(FPRndMode, MVT::f32,
468 Subtarget.hasStdExtZfa() ? Legal : Custom);
469 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
473 setOperationAction(FPOpToExpand, MVT::f32, Expand);
474 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
475 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
476 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
477 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
481 Subtarget.isSoftFPABI() ? LibCall : Custom);
484
485 if (Subtarget.hasStdExtZfa()) {
488 } else {
490 }
491 }
492
493 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
495
496 if (Subtarget.hasStdExtDOrZdinx()) {
497 setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
498
499 if (!Subtarget.is64Bit())
501
502 if (Subtarget.hasStdExtZfa()) {
503 setOperationAction(FPRndMode, MVT::f64, Legal);
506 } else {
507 if (Subtarget.is64Bit())
508 setOperationAction(FPRndMode, MVT::f64, Custom);
509
511 }
512
515 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
519 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
520 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
521 setOperationAction(FPOpToExpand, MVT::f64, Expand);
522 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
523 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
524 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
525 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
529 Subtarget.isSoftFPABI() ? LibCall : Custom);
532 }
533
534 if (Subtarget.is64Bit()) {
537 MVT::i32, Custom);
539 }
540
541 if (Subtarget.hasStdExtFOrZfinx()) {
543 Custom);
544
547 XLenVT, Legal);
548
551 }
552
555 XLenVT, Custom);
556
558
559 if (Subtarget.is64Bit())
561
562 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
563 // Unfortunately this can't be determined just from the ISA naming string.
565 Subtarget.is64Bit() ? Legal : Custom);
567 Subtarget.is64Bit() ? Legal : Custom);
568
571 if (Subtarget.is64Bit())
573
574 if (Subtarget.hasStdExtZicbop()) {
576 }
577
578 if (Subtarget.hasStdExtA()) {
580 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
582 else
584 } else if (Subtarget.hasForcedAtomics()) {
586 } else {
588 }
589
591
593
594 if (getTargetMachine().getTargetTriple().isOSLinux()) {
595 // Custom lowering of llvm.clear_cache.
597 }
598
599 if (Subtarget.hasVInstructions()) {
601
603
604 // RVV intrinsics may have illegal operands.
605 // We also need to custom legalize vmv.x.s.
608 {MVT::i8, MVT::i16}, Custom);
609 if (Subtarget.is64Bit())
611 MVT::i32, Custom);
612 else
614 MVT::i64, Custom);
615
617 MVT::Other, Custom);
618
619 static const unsigned IntegerVPOps[] = {
620 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
621 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
622 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
623 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
624 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
625 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
626 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
627 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
628 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
629 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
630 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
631 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
632 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
633 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF,
634 ISD::EXPERIMENTAL_VP_SPLAT};
635
636 static const unsigned FloatingPointVPOps[] = {
637 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
638 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
639 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
640 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
641 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
642 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
643 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
644 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
645 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
646 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
647 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
648 ISD::VP_LLRINT, ISD::EXPERIMENTAL_VP_REVERSE,
649 ISD::EXPERIMENTAL_VP_SPLICE, ISD::VP_REDUCE_FMINIMUM,
650 ISD::VP_REDUCE_FMAXIMUM, ISD::EXPERIMENTAL_VP_SPLAT};
651
652 static const unsigned IntegerVecReduceOps[] = {
656
657 static const unsigned FloatingPointVecReduceOps[] = {
660
661 if (!Subtarget.is64Bit()) {
662 // We must custom-lower certain vXi64 operations on RV32 due to the vector
663 // element type being illegal.
665 MVT::i64, Custom);
666
667 setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
668
669 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
670 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
671 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
672 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
673 MVT::i64, Custom);
674 }
675
676 for (MVT VT : BoolVecVTs) {
677 if (!isTypeLegal(VT))
678 continue;
679
681
682 // Mask VTs are custom-expanded into a series of standard nodes
686 VT, Custom);
687
689 Custom);
690
693 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT,
694 Expand);
695
696 setOperationAction({ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF}, VT,
697 Custom);
698
699 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);
700
703 Custom);
704
706 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
707 Custom);
708
709 // RVV has native int->float & float->int conversions where the
710 // element type sizes are within one power-of-two of each other. Any
711 // wider distances between type sizes have to be lowered as sequences
712 // which progressively narrow the gap in stages.
717 VT, Custom);
719 Custom);
720
721 // Expand all extending loads to types larger than this, and truncating
722 // stores from types larger than this.
724 setTruncStoreAction(VT, OtherVT, Expand);
726 OtherVT, Expand);
727 }
728
729 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
730 ISD::VP_TRUNCATE, ISD::VP_SETCC},
731 VT, Custom);
732
735
737
738 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
739 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
740
743 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
744 }
745
746 for (MVT VT : IntVecVTs) {
747 if (!isTypeLegal(VT))
748 continue;
749
752
753 // Vectors implement MULHS/MULHU.
755
756 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
757 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
759
761 Legal);
762
764
765 // Custom-lower extensions and truncations from/to mask types.
767 VT, Custom);
768
769 // RVV has native int->float & float->int conversions where the
770 // element type sizes are within one power-of-two of each other. Any
771 // wider distances between type sizes have to be lowered as sequences
772 // which progressively narrow the gap in stages.
777 VT, Custom);
779 Custom);
783 VT, Legal);
784
785 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
786 // nodes which truncate by one power of two at a time.
789 VT, Custom);
790
791 // Custom-lower insert/extract operations to simplify patterns.
793 Custom);
794
795 // Custom-lower reduction operations to set up the corresponding custom
796 // nodes' operands.
797 setOperationAction(IntegerVecReduceOps, VT, Custom);
798
799 setOperationAction(IntegerVPOps, VT, Custom);
800
802
804 VT, Custom);
805
807 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
808 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
809 VT, Custom);
810
813 VT, Custom);
814
817
819
821 setTruncStoreAction(VT, OtherVT, Expand);
823 OtherVT, Expand);
824 }
825
828
829 // Splice
831
832 if (Subtarget.hasStdExtZvkb()) {
834 setOperationAction(ISD::VP_BSWAP, VT, Custom);
835 } else {
836 setOperationAction({ISD::BSWAP, ISD::VP_BSWAP}, VT, Expand);
838 }
839
840 if (Subtarget.hasStdExtZvbb()) {
842 setOperationAction(ISD::VP_BITREVERSE, VT, Custom);
843 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
844 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
845 VT, Custom);
846 } else {
847 setOperationAction({ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Expand);
849 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
850 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
851 VT, Expand);
852
853 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
854 // range of f32.
855 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
856 if (isTypeLegal(FloatVT)) {
858 ISD::CTTZ_ZERO_UNDEF, ISD::VP_CTLZ,
859 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
860 VT, Custom);
861 }
862 }
863 }
864
865 // Expand various CCs to best match the RVV ISA, which natively supports UNE
866 // but no other unordered comparisons, and supports all ordered comparisons
867 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
868 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
869 // and we pattern-match those back to the "original", swapping operands once
870 // more. This way we catch both operations and both "vf" and "fv" forms with
871 // fewer patterns.
872 static const ISD::CondCode VFPCCToExpand[] = {
876 };
877
878 // TODO: support more ops.
879 static const unsigned ZvfhminPromoteOps[] = {
887
888 // TODO: support more vp ops.
889 static const unsigned ZvfhminPromoteVPOps[] = {
890 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
891 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
892 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
893 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_SQRT,
894 ISD::VP_FMINNUM, ISD::VP_FMAXNUM, ISD::VP_FCEIL,
895 ISD::VP_FFLOOR, ISD::VP_FROUND, ISD::VP_FROUNDEVEN,
896 ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO, ISD::VP_FRINT,
897 ISD::VP_FNEARBYINT, ISD::VP_SETCC, ISD::VP_FMINIMUM,
898 ISD::VP_FMAXIMUM, ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
899
900 // Sets common operation actions on RVV floating-point vector types.
901 const auto SetCommonVFPActions = [&](MVT VT) {
903 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
904 // sizes are within one power-of-two of each other. Therefore conversions
905 // between vXf16 and vXf64 must be lowered as sequences which convert via
906 // vXf32.
909 // Custom-lower insert/extract operations to simplify patterns.
911 Custom);
912 // Expand various condition codes (explained above).
913 setCondCodeAction(VFPCCToExpand, VT, Expand);
914
917
921 VT, Custom);
922
923 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
924
925 // Expand FP operations that need libcalls.
937
939
941
943 VT, Custom);
944
946 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
947 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
948 VT, Custom);
949
952
955 VT, Custom);
956
959
961
962 setOperationAction(FloatingPointVPOps, VT, Custom);
963
965 Custom);
968 VT, Legal);
973 VT, Custom);
974 };
975
976 // Sets common extload/truncstore actions on RVV floating-point vector
977 // types.
978 const auto SetCommonVFPExtLoadTruncStoreActions =
979 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
980 for (auto SmallVT : SmallerVTs) {
981 setTruncStoreAction(VT, SmallVT, Expand);
982 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
983 }
984 };
985
986 if (Subtarget.hasVInstructionsF16()) {
987 for (MVT VT : F16VecVTs) {
988 if (!isTypeLegal(VT))
989 continue;
990 SetCommonVFPActions(VT);
991 }
992 } else if (Subtarget.hasVInstructionsF16Minimal()) {
993 for (MVT VT : F16VecVTs) {
994 if (!isTypeLegal(VT))
995 continue;
998 Custom);
999 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1000 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1001 Custom);
1004 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1005 VT, Custom);
1008 VT, Custom);
1009 if (Subtarget.hasStdExtZfhmin())
1011 // load/store
1013
1014 // Custom split nxv32f16 since nxv32f32 if not legal.
1015 if (VT == MVT::nxv32f16) {
1016 setOperationAction(ZvfhminPromoteOps, VT, Custom);
1017 setOperationAction(ZvfhminPromoteVPOps, VT, Custom);
1018 continue;
1019 }
1020 // Add more promote ops.
1021 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1022 setOperationPromotedToType(ZvfhminPromoteOps, VT, F32VecVT);
1023 setOperationPromotedToType(ZvfhminPromoteVPOps, VT, F32VecVT);
1024 }
1025 }
1026
1027 // TODO: Could we merge some code with zvfhmin?
1028 if (Subtarget.hasVInstructionsBF16Minimal()) {
1029 for (MVT VT : BF16VecVTs) {
1030 if (!isTypeLegal(VT))
1031 continue;
1033 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1035 Custom);
1038 VT, Custom);
1040 if (Subtarget.hasStdExtZfbfmin())
1042 setOperationAction({ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1043 Custom);
1045 // TODO: Promote to fp32.
1046 }
1047 }
1048
1049 if (Subtarget.hasVInstructionsF32()) {
1050 for (MVT VT : F32VecVTs) {
1051 if (!isTypeLegal(VT))
1052 continue;
1053 SetCommonVFPActions(VT);
1054 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1055 }
1056 }
1057
1058 if (Subtarget.hasVInstructionsF64()) {
1059 for (MVT VT : F64VecVTs) {
1060 if (!isTypeLegal(VT))
1061 continue;
1062 SetCommonVFPActions(VT);
1063 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1064 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1065 }
1066 }
1067
1068 if (Subtarget.useRVVForFixedLengthVectors()) {
1070 if (!useRVVForFixedLengthVectorVT(VT))
1071 continue;
1072
1073 // By default everything must be expanded.
1074 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1077 setTruncStoreAction(VT, OtherVT, Expand);
1079 OtherVT, Expand);
1080 }
1081
1082 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1083 // expansion to a build_vector of 0s.
1085
1086 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1088 Custom);
1089
1091 Custom);
1092
1094 VT, Custom);
1095
1097
1099
1101
1103
1106 VT, Custom);
1107
1109
1112 Custom);
1113
1115 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1116 Custom);
1117
1119 {
1128 },
1129 VT, Custom);
1131 Custom);
1132
1134
1135 // Operations below are different for between masks and other vectors.
1136 if (VT.getVectorElementType() == MVT::i1) {
1137 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND,
1138 ISD::OR, ISD::XOR},
1139 VT, Custom);
1140
1141 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
1142 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1143 VT, Custom);
1144
1145 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1146 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1147 continue;
1148 }
1149
1150 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1151 // it before type legalization for i64 vectors on RV32. It will then be
1152 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1153 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1154 // improvements first.
1155 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1158 }
1159
1162
1163 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1164 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1165 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1166 ISD::VP_SCATTER},
1167 VT, Custom);
1168
1172 VT, Custom);
1173
1176
1178
1179 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1180 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1182
1186 VT, Custom);
1187
1189
1192
1193 // Custom-lower reduction operations to set up the corresponding custom
1194 // nodes' operands.
1198 VT, Custom);
1199
1200 setOperationAction(IntegerVPOps, VT, Custom);
1201
1202 if (Subtarget.hasStdExtZvkb())
1204
1205 if (Subtarget.hasStdExtZvbb()) {
1208 VT, Custom);
1209 } else {
1210 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
1211 // range of f32.
1212 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1213 if (isTypeLegal(FloatVT))
1216 Custom);
1217 }
1218 }
1219
1221 // There are no extending loads or truncating stores.
1222 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1223 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1224 setTruncStoreAction(VT, InnerVT, Expand);
1225 }
1226
1227 if (!useRVVForFixedLengthVectorVT(VT))
1228 continue;
1229
1230 // By default everything must be expanded.
1231 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1233
1234 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1235 // expansion to a build_vector of 0s.
1237
1240 VT, Custom);
1241
1242 // FIXME: mload, mstore, mgather, mscatter, vp_load/store,
1243 // vp_stride_load/store, vp_gather/scatter can be hoisted to here.
1245
1248 Custom);
1249
1250 if (VT.getVectorElementType() == MVT::f16 &&
1251 !Subtarget.hasVInstructionsF16()) {
1252 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1254 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1255 Custom);
1257 ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP},
1258 VT, Custom);
1260 if (Subtarget.hasStdExtZfhmin()) {
1261 // FIXME: We should prefer BUILD_VECTOR over SPLAT_VECTOR.
1263 } else {
1264 // We need to custom legalize f16 build vectors if Zfhmin isn't
1265 // available.
1267 }
1268 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1269 // Don't promote f16 vector operations to f32 if f32 vector type is
1270 // not legal.
1271 // TODO: could split the f16 vector into two vectors and do promotion.
1272 if (!isTypeLegal(F32VecVT))
1273 continue;
1274 setOperationPromotedToType(ZvfhminPromoteOps, VT, F32VecVT);
1275 setOperationPromotedToType(ZvfhminPromoteVPOps, VT, F32VecVT);
1276 continue;
1277 }
1278
1279 if (VT.getVectorElementType() == MVT::bf16) {
1280 setOperationAction({ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Custom);
1281 // FIXME: We should prefer BUILD_VECTOR over SPLAT_VECTOR.
1284 {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1285 Custom);
1286 // TODO: Promote to fp32.
1287 continue;
1288 }
1289
1292 VT, Custom);
1293
1296
1297 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1298 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1299 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1300 ISD::VP_SCATTER},
1301 VT, Custom);
1302
1307 VT, Custom);
1308
1311 VT, Custom);
1312
1313 setCondCodeAction(VFPCCToExpand, VT, Expand);
1314
1317
1319
1320 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1321
1322 setOperationAction(FloatingPointVPOps, VT, Custom);
1323
1330 VT, Custom);
1331 }
1332
1333 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1334 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32, MVT::i64},
1335 Custom);
1336 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1338 if (Subtarget.hasStdExtFOrZfinx())
1340 if (Subtarget.hasStdExtDOrZdinx())
1342 }
1343 }
1344
1345 if (Subtarget.hasStdExtA())
1347
1348 if (Subtarget.hasForcedAtomics()) {
1349 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1355 XLenVT, LibCall);
1356 }
1357
1358 if (Subtarget.hasVendorXTHeadMemIdx()) {
1359 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1360 setIndexedLoadAction(im, MVT::i8, Legal);
1361 setIndexedStoreAction(im, MVT::i8, Legal);
1362 setIndexedLoadAction(im, MVT::i16, Legal);
1363 setIndexedStoreAction(im, MVT::i16, Legal);
1364 setIndexedLoadAction(im, MVT::i32, Legal);
1365 setIndexedStoreAction(im, MVT::i32, Legal);
1366
1367 if (Subtarget.is64Bit()) {
1368 setIndexedLoadAction(im, MVT::i64, Legal);
1369 setIndexedStoreAction(im, MVT::i64, Legal);
1370 }
1371 }
1372 }
1373
1374 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1378
1382 }
1383
1384 // Function alignments.
1385 const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4);
1386 setMinFunctionAlignment(FunctionAlignment);
1387 // Set preferred alignments.
1390
1395
1396 if (Subtarget.hasStdExtFOrZfinx())
1398
1399 if (Subtarget.hasStdExtZbb())
1401
1402 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
1403 Subtarget.hasVInstructions())
1405
1406 if (Subtarget.hasStdExtZbkb())
1408 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1410 if (Subtarget.hasStdExtFOrZfinx())
1413 if (Subtarget.hasVInstructions())
1415 ISD::VP_GATHER, ISD::VP_SCATTER, ISD::SRA, ISD::SRL,
1418 ISD::EXPERIMENTAL_VP_REVERSE, ISD::MUL,
1421 if (Subtarget.hasVendorXTHeadMemPair())
1423 if (Subtarget.useRVVForFixedLengthVectors())
1425
1426 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1427 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1428
1429 // Disable strict node mutation.
1430 IsStrictFPEnabled = true;
1431 EnableExtLdPromotion = true;
1432
1433 // Let the subtarget decide if a predictable select is more expensive than the
1434 // corresponding branch. This information is used in CGP/SelectOpt to decide
1435 // when to convert selects into branches.
1436 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
1437}
1438
1440 LLVMContext &Context,
1441 EVT VT) const {
1442 if (!VT.isVector())
1443 return getPointerTy(DL);
1444 if (Subtarget.hasVInstructions() &&
1445 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1446 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1448}
1449
1450MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1451 return Subtarget.getXLenVT();
1452}
1453
1454// Return false if we can lower get_vector_length to a vsetvli intrinsic.
1455bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
1456 unsigned VF,
1457 bool IsScalable) const {
1458 if (!Subtarget.hasVInstructions())
1459 return true;
1460
1461 if (!IsScalable)
1462 return true;
1463
1464 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
1465 return true;
1466
1467 // Don't allow VF=1 if those types are't legal.
1468 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
1469 return true;
1470
1471 // VLEN=32 support is incomplete.
1472 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1473 return true;
1474
1475 // The maximum VF is for the smallest element width with LMUL=8.
1476 // VF must be a power of 2.
1477 unsigned MaxVF = (RISCV::RVVBitsPerBlock / 8) * 8;
1478 return VF > MaxVF || !isPowerOf2_32(VF);
1479}
1480
1482 return !Subtarget.hasVInstructions() ||
1483 VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
1484}
1485
1487 const CallInst &I,
1488 MachineFunction &MF,
1489 unsigned Intrinsic) const {
1490 auto &DL = I.getDataLayout();
1491
1492 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
1493 bool IsUnitStrided, bool UsePtrVal = false) {
1495 // We can't use ptrVal if the intrinsic can access memory before the
1496 // pointer. This means we can't use it for strided or indexed intrinsics.
1497 if (UsePtrVal)
1498 Info.ptrVal = I.getArgOperand(PtrOp);
1499 else
1500 Info.fallbackAddressSpace =
1501 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
1502 Type *MemTy;
1503 if (IsStore) {
1504 // Store value is the first operand.
1505 MemTy = I.getArgOperand(0)->getType();
1506 } else {
1507 // Use return type. If it's segment load, return type is a struct.
1508 MemTy = I.getType();
1509 if (MemTy->isStructTy())
1510 MemTy = MemTy->getStructElementType(0);
1511 }
1512 if (!IsUnitStrided)
1513 MemTy = MemTy->getScalarType();
1514
1515 Info.memVT = getValueType(DL, MemTy);
1516 Info.align = Align(DL.getTypeSizeInBits(MemTy->getScalarType()) / 8);
1518 Info.flags |=
1520 return true;
1521 };
1522
1523 if (I.hasMetadata(LLVMContext::MD_nontemporal))
1525
1527 switch (Intrinsic) {
1528 default:
1529 return false;
1530 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1531 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1532 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1533 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1534 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1535 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1536 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1537 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1538 case Intrinsic::riscv_masked_cmpxchg_i32:
1540 Info.memVT = MVT::i32;
1541 Info.ptrVal = I.getArgOperand(0);
1542 Info.offset = 0;
1543 Info.align = Align(4);
1546 return true;
1547 case Intrinsic::riscv_seg2_load:
1548 case Intrinsic::riscv_seg3_load:
1549 case Intrinsic::riscv_seg4_load:
1550 case Intrinsic::riscv_seg5_load:
1551 case Intrinsic::riscv_seg6_load:
1552 case Intrinsic::riscv_seg7_load:
1553 case Intrinsic::riscv_seg8_load:
1554 return SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
1555 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1556 case Intrinsic::riscv_seg2_store:
1557 case Intrinsic::riscv_seg3_store:
1558 case Intrinsic::riscv_seg4_store:
1559 case Intrinsic::riscv_seg5_store:
1560 case Intrinsic::riscv_seg6_store:
1561 case Intrinsic::riscv_seg7_store:
1562 case Intrinsic::riscv_seg8_store:
1563 // Operands are (vec, ..., vec, ptr, vl)
1564 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1565 /*IsStore*/ true,
1566 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1567 case Intrinsic::riscv_vle:
1568 case Intrinsic::riscv_vle_mask:
1569 case Intrinsic::riscv_vleff:
1570 case Intrinsic::riscv_vleff_mask:
1571 return SetRVVLoadStoreInfo(/*PtrOp*/ 1,
1572 /*IsStore*/ false,
1573 /*IsUnitStrided*/ true,
1574 /*UsePtrVal*/ true);
1575 case Intrinsic::riscv_vse:
1576 case Intrinsic::riscv_vse_mask:
1577 return SetRVVLoadStoreInfo(/*PtrOp*/ 1,
1578 /*IsStore*/ true,
1579 /*IsUnitStrided*/ true,
1580 /*UsePtrVal*/ true);
1581 case Intrinsic::riscv_vlse:
1582 case Intrinsic::riscv_vlse_mask:
1583 case Intrinsic::riscv_vloxei:
1584 case Intrinsic::riscv_vloxei_mask:
1585 case Intrinsic::riscv_vluxei:
1586 case Intrinsic::riscv_vluxei_mask:
1587 return SetRVVLoadStoreInfo(/*PtrOp*/ 1,
1588 /*IsStore*/ false,
1589 /*IsUnitStrided*/ false);
1590 case Intrinsic::riscv_vsse:
1591 case Intrinsic::riscv_vsse_mask:
1592 case Intrinsic::riscv_vsoxei:
1593 case Intrinsic::riscv_vsoxei_mask:
1594 case Intrinsic::riscv_vsuxei:
1595 case Intrinsic::riscv_vsuxei_mask:
1596 return SetRVVLoadStoreInfo(/*PtrOp*/ 1,
1597 /*IsStore*/ true,
1598 /*IsUnitStrided*/ false);
1599 case Intrinsic::riscv_vlseg2:
1600 case Intrinsic::riscv_vlseg3:
1601 case Intrinsic::riscv_vlseg4:
1602 case Intrinsic::riscv_vlseg5:
1603 case Intrinsic::riscv_vlseg6:
1604 case Intrinsic::riscv_vlseg7:
1605 case Intrinsic::riscv_vlseg8:
1606 case Intrinsic::riscv_vlseg2ff:
1607 case Intrinsic::riscv_vlseg3ff:
1608 case Intrinsic::riscv_vlseg4ff:
1609 case Intrinsic::riscv_vlseg5ff:
1610 case Intrinsic::riscv_vlseg6ff:
1611 case Intrinsic::riscv_vlseg7ff:
1612 case Intrinsic::riscv_vlseg8ff:
1613 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1614 /*IsStore*/ false,
1615 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1616 case Intrinsic::riscv_vlseg2_mask:
1617 case Intrinsic::riscv_vlseg3_mask:
1618 case Intrinsic::riscv_vlseg4_mask:
1619 case Intrinsic::riscv_vlseg5_mask:
1620 case Intrinsic::riscv_vlseg6_mask:
1621 case Intrinsic::riscv_vlseg7_mask:
1622 case Intrinsic::riscv_vlseg8_mask:
1623 case Intrinsic::riscv_vlseg2ff_mask:
1624 case Intrinsic::riscv_vlseg3ff_mask:
1625 case Intrinsic::riscv_vlseg4ff_mask:
1626 case Intrinsic::riscv_vlseg5ff_mask:
1627 case Intrinsic::riscv_vlseg6ff_mask:
1628 case Intrinsic::riscv_vlseg7ff_mask:
1629 case Intrinsic::riscv_vlseg8ff_mask:
1630 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
1631 /*IsStore*/ false,
1632 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
1633 case Intrinsic::riscv_vlsseg2:
1634 case Intrinsic::riscv_vlsseg3:
1635 case Intrinsic::riscv_vlsseg4:
1636 case Intrinsic::riscv_vlsseg5:
1637 case Intrinsic::riscv_vlsseg6:
1638 case Intrinsic::riscv_vlsseg7:
1639 case Intrinsic::riscv_vlsseg8:
1640 case Intrinsic::riscv_vloxseg2:
1641 case Intrinsic::riscv_vloxseg3:
1642 case Intrinsic::riscv_vloxseg4:
1643 case Intrinsic::riscv_vloxseg5:
1644 case Intrinsic::riscv_vloxseg6:
1645 case Intrinsic::riscv_vloxseg7:
1646 case Intrinsic::riscv_vloxseg8:
1647 case Intrinsic::riscv_vluxseg2:
1648 case Intrinsic::riscv_vluxseg3:
1649 case Intrinsic::riscv_vluxseg4:
1650 case Intrinsic::riscv_vluxseg5:
1651 case Intrinsic::riscv_vluxseg6:
1652 case Intrinsic::riscv_vluxseg7:
1653 case Intrinsic::riscv_vluxseg8:
1654 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1655 /*IsStore*/ false,
1656 /*IsUnitStrided*/ false);
1657 case Intrinsic::riscv_vlsseg2_mask:
1658 case Intrinsic::riscv_vlsseg3_mask:
1659 case Intrinsic::riscv_vlsseg4_mask:
1660 case Intrinsic::riscv_vlsseg5_mask:
1661 case Intrinsic::riscv_vlsseg6_mask:
1662 case Intrinsic::riscv_vlsseg7_mask:
1663 case Intrinsic::riscv_vlsseg8_mask:
1664 case Intrinsic::riscv_vloxseg2_mask:
1665 case Intrinsic::riscv_vloxseg3_mask:
1666 case Intrinsic::riscv_vloxseg4_mask:
1667 case Intrinsic::riscv_vloxseg5_mask:
1668 case Intrinsic::riscv_vloxseg6_mask:
1669 case Intrinsic::riscv_vloxseg7_mask:
1670 case Intrinsic::riscv_vloxseg8_mask:
1671 case Intrinsic::riscv_vluxseg2_mask:
1672 case Intrinsic::riscv_vluxseg3_mask:
1673 case Intrinsic::riscv_vluxseg4_mask:
1674 case Intrinsic::riscv_vluxseg5_mask:
1675 case Intrinsic::riscv_vluxseg6_mask:
1676 case Intrinsic::riscv_vluxseg7_mask:
1677 case Intrinsic::riscv_vluxseg8_mask:
1678 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
1679 /*IsStore*/ false,
1680 /*IsUnitStrided*/ false);
1681 case Intrinsic::riscv_vsseg2:
1682 case Intrinsic::riscv_vsseg3:
1683 case Intrinsic::riscv_vsseg4:
1684 case Intrinsic::riscv_vsseg5:
1685 case Intrinsic::riscv_vsseg6:
1686 case Intrinsic::riscv_vsseg7:
1687 case Intrinsic::riscv_vsseg8:
1688 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 2,
1689 /*IsStore*/ true,
1690 /*IsUnitStrided*/ false);
1691 case Intrinsic::riscv_vsseg2_mask:
1692 case Intrinsic::riscv_vsseg3_mask:
1693 case Intrinsic::riscv_vsseg4_mask:
1694 case Intrinsic::riscv_vsseg5_mask:
1695 case Intrinsic::riscv_vsseg6_mask:
1696 case Intrinsic::riscv_vsseg7_mask:
1697 case Intrinsic::riscv_vsseg8_mask:
1698 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1699 /*IsStore*/ true,
1700 /*IsUnitStrided*/ false);
1701 case Intrinsic::riscv_vssseg2:
1702 case Intrinsic::riscv_vssseg3:
1703 case Intrinsic::riscv_vssseg4:
1704 case Intrinsic::riscv_vssseg5:
1705 case Intrinsic::riscv_vssseg6:
1706 case Intrinsic::riscv_vssseg7:
1707 case Intrinsic::riscv_vssseg8:
1708 case Intrinsic::riscv_vsoxseg2:
1709 case Intrinsic::riscv_vsoxseg3:
1710 case Intrinsic::riscv_vsoxseg4:
1711 case Intrinsic::riscv_vsoxseg5:
1712 case Intrinsic::riscv_vsoxseg6:
1713 case Intrinsic::riscv_vsoxseg7:
1714 case Intrinsic::riscv_vsoxseg8:
1715 case Intrinsic::riscv_vsuxseg2:
1716 case Intrinsic::riscv_vsuxseg3:
1717 case Intrinsic::riscv_vsuxseg4:
1718 case Intrinsic::riscv_vsuxseg5:
1719 case Intrinsic::riscv_vsuxseg6:
1720 case Intrinsic::riscv_vsuxseg7:
1721 case Intrinsic::riscv_vsuxseg8:
1722 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
1723 /*IsStore*/ true,
1724 /*IsUnitStrided*/ false);
1725 case Intrinsic::riscv_vssseg2_mask:
1726 case Intrinsic::riscv_vssseg3_mask:
1727 case Intrinsic::riscv_vssseg4_mask:
1728 case Intrinsic::riscv_vssseg5_mask:
1729 case Intrinsic::riscv_vssseg6_mask:
1730 case Intrinsic::riscv_vssseg7_mask:
1731 case Intrinsic::riscv_vssseg8_mask:
1732 case Intrinsic::riscv_vsoxseg2_mask:
1733 case Intrinsic::riscv_vsoxseg3_mask:
1734 case Intrinsic::riscv_vsoxseg4_mask:
1735 case Intrinsic::riscv_vsoxseg5_mask:
1736 case Intrinsic::riscv_vsoxseg6_mask:
1737 case Intrinsic::riscv_vsoxseg7_mask:
1738 case Intrinsic::riscv_vsoxseg8_mask:
1739 case Intrinsic::riscv_vsuxseg2_mask:
1740 case Intrinsic::riscv_vsuxseg3_mask:
1741 case Intrinsic::riscv_vsuxseg4_mask:
1742 case Intrinsic::riscv_vsuxseg5_mask:
1743 case Intrinsic::riscv_vsuxseg6_mask:
1744 case Intrinsic::riscv_vsuxseg7_mask:
1745 case Intrinsic::riscv_vsuxseg8_mask:
1746 return SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
1747 /*IsStore*/ true,
1748 /*IsUnitStrided*/ false);
1749 }
1750}
1751
1753 const AddrMode &AM, Type *Ty,
1754 unsigned AS,
1755 Instruction *I) const {
1756 // No global is ever allowed as a base.
1757 if (AM.BaseGV)
1758 return false;
1759
1760 // None of our addressing modes allows a scalable offset
1761 if (AM.ScalableOffset)
1762 return false;
1763
1764 // RVV instructions only support register addressing.
1765 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
1766 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
1767
1768 // Require a 12-bit signed offset.
1769 if (!isInt<12>(AM.BaseOffs))
1770 return false;
1771
1772 switch (AM.Scale) {
1773 case 0: // "r+i" or just "i", depending on HasBaseReg.
1774 break;
1775 case 1:
1776 if (!AM.HasBaseReg) // allow "r+i".
1777 break;
1778 return false; // disallow "r+r" or "r+r+i".
1779 default:
1780 return false;
1781 }
1782
1783 return true;
1784}
1785
1787 return isInt<12>(Imm);
1788}
1789
1791 return isInt<12>(Imm);
1792}
1793
1794// On RV32, 64-bit integers are split into their high and low parts and held
1795// in two different registers, so the trunc is free since the low register can
1796// just be used.
1797// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
1798// isTruncateFree?
1800 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1801 return false;
1802 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1803 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1804 return (SrcBits == 64 && DestBits == 32);
1805}
1806
1808 // We consider i64->i32 free on RV64 since we have good selection of W
1809 // instructions that make promoting operations back to i64 free in many cases.
1810 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
1811 !DstVT.isInteger())
1812 return false;
1813 unsigned SrcBits = SrcVT.getSizeInBits();
1814 unsigned DestBits = DstVT.getSizeInBits();
1815 return (SrcBits == 64 && DestBits == 32);
1816}
1817
1819 EVT SrcVT = Val.getValueType();
1820 // free truncate from vnsrl and vnsra
1821 if (Subtarget.hasVInstructions() &&
1822 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
1823 SrcVT.isVector() && VT2.isVector()) {
1824 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
1825 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
1826 if (SrcBits == DestBits * 2) {
1827 return true;
1828 }
1829 }
1830 return TargetLowering::isTruncateFree(Val, VT2);
1831}
1832
1834 // Zexts are free if they can be combined with a load.
1835 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1836 // poorly with type legalization of compares preferring sext.
1837 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1838 EVT MemVT = LD->getMemoryVT();
1839 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1840 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1841 LD->getExtensionType() == ISD::ZEXTLOAD))
1842 return true;
1843 }
1844
1845 return TargetLowering::isZExtFree(Val, VT2);
1846}
1847
1849 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1850}
1851
1853 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
1854}
1855
1857 return Subtarget.hasStdExtZbb() ||
1858 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit());
1859}
1860
1862 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb() ||
1863 (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit());
1864}
1865
1867 const Instruction &AndI) const {
1868 // We expect to be able to match a bit extraction instruction if the Zbs
1869 // extension is supported and the mask is a power of two. However, we
1870 // conservatively return false if the mask would fit in an ANDI instruction,
1871 // on the basis that it's possible the sinking+duplication of the AND in
1872 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
1873 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
1874 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1875 return false;
1876 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
1877 if (!Mask)
1878 return false;
1879 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1880}
1881
1883 EVT VT = Y.getValueType();
1884
1885 // FIXME: Support vectors once we have tests.
1886 if (VT.isVector())
1887 return false;
1888
1889 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1890 (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
1891}
1892
1894 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
1895 if (Subtarget.hasStdExtZbs())
1896 return X.getValueType().isScalarInteger();
1897 auto *C = dyn_cast<ConstantSDNode>(Y);
1898 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
1899 if (Subtarget.hasVendorXTHeadBs())
1900 return C != nullptr;
1901 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
1902 return C && C->getAPIntValue().ule(10);
1903}
1904
1906 EVT VT) const {
1907 // Only enable for rvv.
1908 if (!VT.isVector() || !Subtarget.hasVInstructions())
1909 return false;
1910
1911 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
1912 return false;
1913
1914 return true;
1915}
1916
1918 Type *Ty) const {
1919 assert(Ty->isIntegerTy());
1920
1921 unsigned BitSize = Ty->getIntegerBitWidth();
1922 if (BitSize > Subtarget.getXLen())
1923 return false;
1924
1925 // Fast path, assume 32-bit immediates are cheap.
1926 int64_t Val = Imm.getSExtValue();
1927 if (isInt<32>(Val))
1928 return true;
1929
1930 // A constant pool entry may be more aligned thant he load we're trying to
1931 // replace. If we don't support unaligned scalar mem, prefer the constant
1932 // pool.
1933 // TODO: Can the caller pass down the alignment?
1934 if (!Subtarget.enableUnalignedScalarMem())
1935 return true;
1936
1937 // Prefer to keep the load if it would require many instructions.
1938 // This uses the same threshold we use for constant pools but doesn't
1939 // check useConstantPoolForLargeInts.
1940 // TODO: Should we keep the load only when we're definitely going to emit a
1941 // constant pool?
1942
1944 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
1945}
1946
1950 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
1951 SelectionDAG &DAG) const {
1952 // One interesting pattern that we'd want to form is 'bit extract':
1953 // ((1 >> Y) & 1) ==/!= 0
1954 // But we also need to be careful not to try to reverse that fold.
1955
1956 // Is this '((1 >> Y) & 1)'?
1957 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
1958 return false; // Keep the 'bit extract' pattern.
1959
1960 // Will this be '((1 >> Y) & 1)' after the transform?
1961 if (NewShiftOpcode == ISD::SRL && CC->isOne())
1962 return true; // Do form the 'bit extract' pattern.
1963
1964 // If 'X' is a constant, and we transform, then we will immediately
1965 // try to undo the fold, thus causing endless combine loop.
1966 // So only do the transform if X is not a constant. This matches the default
1967 // implementation of this function.
1968 return !XC;
1969}
1970
1971bool RISCVTargetLowering::canSplatOperand(unsigned Opcode, int Operand) const {
1972 switch (Opcode) {
1973 case Instruction::Add:
1974 case Instruction::Sub:
1975 case Instruction::Mul:
1976 case Instruction::And:
1977 case Instruction::Or:
1978 case Instruction::Xor:
1979 case Instruction::FAdd:
1980 case Instruction::FSub:
1981 case Instruction::FMul:
1982 case Instruction::FDiv:
1983 case Instruction::ICmp:
1984 case Instruction::FCmp:
1985 return true;
1986 case Instruction::Shl:
1987 case Instruction::LShr:
1988 case Instruction::AShr:
1989 case Instruction::UDiv:
1990 case Instruction::SDiv:
1991 case Instruction::URem:
1992 case Instruction::SRem:
1993 case Instruction::Select:
1994 return Operand == 1;
1995 default:
1996 return false;
1997 }
1998}
1999
2000
2002 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
2003 return false;
2004
2005 if (canSplatOperand(I->getOpcode(), Operand))
2006 return true;
2007
2008 auto *II = dyn_cast<IntrinsicInst>(I);
2009 if (!II)
2010 return false;
2011
2012 switch (II->getIntrinsicID()) {
2013 case Intrinsic::fma:
2014 case Intrinsic::vp_fma:
2015 return Operand == 0 || Operand == 1;
2016 case Intrinsic::vp_shl:
2017 case Intrinsic::vp_lshr:
2018 case Intrinsic::vp_ashr:
2019 case Intrinsic::vp_udiv:
2020 case Intrinsic::vp_sdiv:
2021 case Intrinsic::vp_urem:
2022 case Intrinsic::vp_srem:
2023 case Intrinsic::ssub_sat:
2024 case Intrinsic::vp_ssub_sat:
2025 case Intrinsic::usub_sat:
2026 case Intrinsic::vp_usub_sat:
2027 return Operand == 1;
2028 // These intrinsics are commutative.
2029 case Intrinsic::vp_add:
2030 case Intrinsic::vp_mul:
2031 case Intrinsic::vp_and:
2032 case Intrinsic::vp_or:
2033 case Intrinsic::vp_xor:
2034 case Intrinsic::vp_fadd:
2035 case Intrinsic::vp_fmul:
2036 case Intrinsic::vp_icmp:
2037 case Intrinsic::vp_fcmp:
2038 case Intrinsic::smin:
2039 case Intrinsic::vp_smin:
2040 case Intrinsic::umin:
2041 case Intrinsic::vp_umin:
2042 case Intrinsic::smax:
2043 case Intrinsic::vp_smax:
2044 case Intrinsic::umax:
2045 case Intrinsic::vp_umax:
2046 case Intrinsic::sadd_sat:
2047 case Intrinsic::vp_sadd_sat:
2048 case Intrinsic::uadd_sat:
2049 case Intrinsic::vp_uadd_sat:
2050 // These intrinsics have 'vr' versions.
2051 case Intrinsic::vp_sub:
2052 case Intrinsic::vp_fsub:
2053 case Intrinsic::vp_fdiv:
2054 return Operand == 0 || Operand == 1;
2055 default:
2056 return false;
2057 }
2058}
2059
2060/// Check if sinking \p I's operands to I's basic block is profitable, because
2061/// the operands can be folded into a target instruction, e.g.
2062/// splats of scalars can fold into vector instructions.
2064 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
2065 using namespace llvm::PatternMatch;
2066
2067 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
2068 return false;
2069
2070 // Don't sink splat operands if the target prefers it. Some targets requires
2071 // S2V transfer buffers and we can run out of them copying the same value
2072 // repeatedly.
2073 // FIXME: It could still be worth doing if it would improve vector register
2074 // pressure and prevent a vector spill.
2075 if (!Subtarget.sinkSplatOperands())
2076 return false;
2077
2078 for (auto OpIdx : enumerate(I->operands())) {
2079 if (!canSplatOperand(I, OpIdx.index()))
2080 continue;
2081
2082 Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
2083 // Make sure we are not already sinking this operand
2084 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
2085 continue;
2086
2087 // We are looking for a splat that can be sunk.
2089 m_Undef(), m_ZeroMask())))
2090 continue;
2091
2092 // Don't sink i1 splats.
2093 if (cast<VectorType>(Op->getType())->getElementType()->isIntegerTy(1))
2094 continue;
2095
2096 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
2097 // and vector registers
2098 for (Use &U : Op->uses()) {
2099 Instruction *Insn = cast<Instruction>(U.getUser());
2100 if (!canSplatOperand(Insn, U.getOperandNo()))
2101 return false;
2102 }
2103
2104 Ops.push_back(&Op->getOperandUse(0));
2105 Ops.push_back(&OpIdx.value());
2106 }
2107 return true;
2108}
2109
2111 unsigned Opc = VecOp.getOpcode();
2112
2113 // Assume target opcodes can't be scalarized.
2114 // TODO - do we have any exceptions?
2115 if (Opc >= ISD::BUILTIN_OP_END)
2116 return false;
2117
2118 // If the vector op is not supported, try to convert to scalar.
2119 EVT VecVT = VecOp.getValueType();
2120 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
2121 return true;
2122
2123 // If the vector op is supported, but the scalar op is not, the transform may
2124 // not be worthwhile.
2125 // Permit a vector binary operation can be converted to scalar binary
2126 // operation which is custom lowered with illegal type.
2127 EVT ScalarVT = VecVT.getScalarType();
2128 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
2129 isOperationCustom(Opc, ScalarVT);
2130}
2131
2133 const GlobalAddressSDNode *GA) const {
2134 // In order to maximise the opportunity for common subexpression elimination,
2135 // keep a separate ADD node for the global address offset instead of folding
2136 // it in the global address node. Later peephole optimisations may choose to
2137 // fold it back in when profitable.
2138 return false;
2139}
2140
2141// Return one of the followings:
2142// (1) `{0-31 value, false}` if FLI is available for Imm's type and FP value.
2143// (2) `{0-31 value, true}` if Imm is negative and FLI is available for its
2144// positive counterpart, which will be materialized from the first returned
2145// element. The second returned element indicated that there should be a FNEG
2146// followed.
2147// (3) `{-1, _}` if there is no way FLI can be used to materialize Imm.
2148std::pair<int, bool> RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm,
2149 EVT VT) const {
2150 if (!Subtarget.hasStdExtZfa())
2151 return std::make_pair(-1, false);
2152
2153 bool IsSupportedVT = false;
2154 if (VT == MVT::f16) {
2155 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2156 } else if (VT == MVT::f32) {
2157 IsSupportedVT = true;
2158 } else if (VT == MVT::f64) {
2159 assert(Subtarget.hasStdExtD() && "Expect D extension");
2160 IsSupportedVT = true;
2161 }
2162
2163 if (!IsSupportedVT)
2164 return std::make_pair(-1, false);
2165
2167 if (Index < 0 && Imm.isNegative())
2168 // Try the combination of its positive counterpart + FNEG.
2169 return std::make_pair(RISCVLoadFPImm::getLoadFPImm(-Imm), true);
2170 else
2171 return std::make_pair(Index, false);
2172}
2173
2175 bool ForCodeSize) const {
2176 bool IsLegalVT = false;
2177 if (VT == MVT::f16)
2178 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2179 else if (VT == MVT::f32)
2180 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2181 else if (VT == MVT::f64)
2182 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2183 else if (VT == MVT::bf16)
2184 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2185
2186 if (!IsLegalVT)
2187 return false;
2188
2189 if (getLegalZfaFPImm(Imm, VT).first >= 0)
2190 return true;
2191
2192 // Cannot create a 64 bit floating-point immediate value for rv32.
2193 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2194 // td can handle +0.0 or -0.0 already.
2195 // -0.0 can be created by fmv + fneg.
2196 return Imm.isZero();
2197 }
2198
2199 // Special case: fmv + fneg
2200 if (Imm.isNegZero())
2201 return true;
2202
2203 // Building an integer and then converting requires a fmv at the end of
2204 // the integer sequence.
2205 const int Cost =
2206 1 + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(), Subtarget.getXLen(),
2207 Subtarget);
2208 return Cost <= FPImmCost;
2209}
2210
2211// TODO: This is very conservative.
2213 unsigned Index) const {
2215 return false;
2216
2217 // Only support extracting a fixed from a fixed vector for now.
2218 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2219 return false;
2220
2221 EVT EltVT = ResVT.getVectorElementType();
2222 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2223
2224 // The smallest type we can slide is i8.
2225 // TODO: We can extract index 0 from a mask vector without a slide.
2226 if (EltVT == MVT::i1)
2227 return false;
2228
2229 unsigned ResElts = ResVT.getVectorNumElements();
2230 unsigned SrcElts = SrcVT.getVectorNumElements();
2231
2232 unsigned MinVLen = Subtarget.getRealMinVLen();
2233 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2234
2235 // If we're extracting only data from the first VLEN bits of the source
2236 // then we can always do this with an m1 vslidedown.vx. Restricting the
2237 // Index ensures we can use a vslidedown.vi.
2238 // TODO: We can generalize this when the exact VLEN is known.
2239 if (Index + ResElts <= MinVLMAX && Index < 31)
2240 return true;
2241
2242 // Convervatively only handle extracting half of a vector.
2243 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2244 // a cheap extract. However, this case is important in practice for
2245 // shuffled extracts of longer vectors. How resolve?
2246 if ((ResElts * 2) != SrcElts)
2247 return false;
2248
2249 // Slide can support arbitrary index, but we only treat vslidedown.vi as
2250 // cheap.
2251 if (Index >= 32)
2252 return false;
2253
2254 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2255 // the upper half of a vector until we have more test coverage.
2256 return Index == 0 || Index == ResElts;
2257}
2258
2261 EVT VT) const {
2262 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2263 // We might still end up using a GPR but that will be decided based on ABI.
2264 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2265 !Subtarget.hasStdExtZfhminOrZhinxmin())
2266 return MVT::f32;
2267
2269
2270 return PartVT;
2271}
2272
2275 EVT VT) const {
2276 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2277 // We might still end up using a GPR but that will be decided based on ABI.
2278 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2279 !Subtarget.hasStdExtZfhminOrZhinxmin())
2280 return 1;
2281
2283}
2284
2286 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
2287 unsigned &NumIntermediates, MVT &RegisterVT) const {
2289 Context, CC, VT, IntermediateVT, NumIntermediates, RegisterVT);
2290
2291 return NumRegs;
2292}
2293
2294// Changes the condition code and swaps operands if necessary, so the SetCC
2295// operation matches one of the comparisons supported directly by branches
2296// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2297// with 1/-1.
2298static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2299 ISD::CondCode &CC, SelectionDAG &DAG) {
2300 // If this is a single bit test that can't be handled by ANDI, shift the
2301 // bit to be tested to the MSB and perform a signed compare with 0.
2302 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
2303 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2304 isa<ConstantSDNode>(LHS.getOperand(1))) {
2305 uint64_t Mask = LHS.getConstantOperandVal(1);
2306 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
2307 unsigned ShAmt = 0;
2308 if (isPowerOf2_64(Mask)) {
2310 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
2311 } else {
2312 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
2313 }
2314
2315 LHS = LHS.getOperand(0);
2316 if (ShAmt != 0)
2317 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
2318 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
2319 return;
2320 }
2321 }
2322
2323 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2324 int64_t C = RHSC->getSExtValue();
2325 switch (CC) {
2326 default: break;
2327 case ISD::SETGT:
2328 // Convert X > -1 to X >= 0.
2329 if (C == -1) {
2330 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2331 CC = ISD::SETGE;
2332 return;
2333 }
2334 break;
2335 case ISD::SETLT:
2336 // Convert X < 1 to 0 >= X.
2337 if (C == 1) {
2338 RHS = LHS;
2339 LHS = DAG.getConstant(0, DL, RHS.getValueType());
2340 CC = ISD::SETGE;
2341 return;
2342 }
2343 break;
2344 }
2345 }
2346
2347 switch (CC) {
2348 default:
2349 break;
2350 case ISD::SETGT:
2351 case ISD::SETLE:
2352 case ISD::SETUGT:
2353 case ISD::SETULE:
2355 std::swap(LHS, RHS);
2356 break;
2357 }
2358}
2359
2361 assert(VT.isScalableVector() && "Expecting a scalable vector type");
2362 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
2363 if (VT.getVectorElementType() == MVT::i1)
2364 KnownSize *= 8;
2365
2366 switch (KnownSize) {
2367 default:
2368 llvm_unreachable("Invalid LMUL.");
2369 case 8:
2371 case 16:
2373 case 32:
2375 case 64:
2377 case 128:
2379 case 256:
2381 case 512:
2383 }
2384}
2385
2387 switch (LMul) {
2388 default:
2389 llvm_unreachable("Invalid LMUL.");
2394 return RISCV::VRRegClassID;
2396 return RISCV::VRM2RegClassID;
2398 return RISCV::VRM4RegClassID;
2400 return RISCV::VRM8RegClassID;
2401 }
2402}
2403
2405 RISCVII::VLMUL LMUL = getLMUL(VT);
2406 if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
2407 LMUL == RISCVII::VLMUL::LMUL_F4 ||
2408 LMUL == RISCVII::VLMUL::LMUL_F2 ||
2409 LMUL == RISCVII::VLMUL::LMUL_1) {
2410 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2411 "Unexpected subreg numbering");
2412 return RISCV::sub_vrm1_0 + Index;
2413 }
2414 if (LMUL == RISCVII::VLMUL::LMUL_2) {
2415 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2416 "Unexpected subreg numbering");
2417 return RISCV::sub_vrm2_0 + Index;
2418 }
2419 if (LMUL == RISCVII::VLMUL::LMUL_4) {
2420 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2421 "Unexpected subreg numbering");
2422 return RISCV::sub_vrm4_0 + Index;
2423 }
2424 llvm_unreachable("Invalid vector type.");
2425}
2426
2428 if (VT.getVectorElementType() == MVT::i1)
2429 return RISCV::VRRegClassID;
2430 return getRegClassIDForLMUL(getLMUL(VT));
2431}
2432
2433// Attempt to decompose a subvector insert/extract between VecVT and
2434// SubVecVT via subregister indices. Returns the subregister index that
2435// can perform the subvector insert/extract with the given element index, as
2436// well as the index corresponding to any leftover subvectors that must be
2437// further inserted/extracted within the register class for SubVecVT.
2438std::pair<unsigned, unsigned>
2440 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
2441 const RISCVRegisterInfo *TRI) {
2442 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
2443 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
2444 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
2445 "Register classes not ordered");
2446 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
2447 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
2448 // Try to compose a subregister index that takes us from the incoming
2449 // LMUL>1 register class down to the outgoing one. At each step we half
2450 // the LMUL:
2451 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
2452 // Note that this is not guaranteed to find a subregister index, such as
2453 // when we are extracting from one VR type to another.
2454 unsigned SubRegIdx = RISCV::NoSubRegister;
2455 for (const unsigned RCID :
2456 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
2457 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
2458 VecVT = VecVT.getHalfNumVectorElementsVT();
2459 bool IsHi =
2460 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
2461 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
2462 getSubregIndexByMVT(VecVT, IsHi));
2463 if (IsHi)
2464 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
2465 }
2466 return {SubRegIdx, InsertExtractIdx};
2467}
2468
2469// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
2470// stores for those types.
2471bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
2472 return !Subtarget.useRVVForFixedLengthVectors() ||
2473 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
2474}
2475
2477 if (!ScalarTy.isSimple())
2478 return false;
2479 switch (ScalarTy.getSimpleVT().SimpleTy) {
2480 case MVT::iPTR:
2481 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
2482 case MVT::i8:
2483 case MVT::i16:
2484 case MVT::i32:
2485 return true;
2486 case MVT::i64:
2487 return Subtarget.hasVInstructionsI64();
2488 case MVT::f16:
2489 return Subtarget.hasVInstructionsF16();
2490 case MVT::f32:
2491 return Subtarget.hasVInstructionsF32();
2492 case MVT::f64:
2493 return Subtarget.hasVInstructionsF64();
2494 default:
2495 return false;
2496 }
2497}
2498
2499
2500unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
2501 return NumRepeatedDivisors;
2502}
2503
2505 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
2506 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
2507 "Unexpected opcode");
2508 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
2509 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
2511 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
2512 if (!II)
2513 return SDValue();
2514 return Op.getOperand(II->VLOperand + 1 + HasChain);
2515}
2516
2518 const RISCVSubtarget &Subtarget) {
2519 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
2520 if (!Subtarget.useRVVForFixedLengthVectors())
2521 return false;
2522
2523 // We only support a set of vector types with a consistent maximum fixed size
2524 // across all supported vector element types to avoid legalization issues.
2525 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
2526 // fixed-length vector type we support is 1024 bytes.
2527 if (VT.getFixedSizeInBits() > 1024 * 8)
2528 return false;
2529
2530 unsigned MinVLen = Subtarget.getRealMinVLen();
2531
2532 MVT EltVT = VT.getVectorElementType();
2533
2534 // Don't use RVV for vectors we cannot scalarize if required.
2535 switch (EltVT.SimpleTy) {
2536 // i1 is supported but has different rules.
2537 default:
2538 return false;
2539 case MVT::i1:
2540 // Masks can only use a single register.
2541 if (VT.getVectorNumElements() > MinVLen)
2542 return false;
2543 MinVLen /= 8;
2544 break;
2545 case MVT::i8:
2546 case MVT::i16:
2547 case MVT::i32:
2548 break;
2549 case MVT::i64:
2550 if (!Subtarget.hasVInstructionsI64())
2551 return false;
2552 break;
2553 case MVT::f16:
2554 if (!Subtarget.hasVInstructionsF16Minimal())
2555 return false;
2556 break;
2557 case MVT::bf16:
2558 if (!Subtarget.hasVInstructionsBF16Minimal())
2559 return false;
2560 break;
2561 case MVT::f32:
2562 if (!Subtarget.hasVInstructionsF32())
2563 return false;
2564 break;
2565 case MVT::f64:
2566 if (!Subtarget.hasVInstructionsF64())
2567 return false;
2568 break;
2569 }
2570
2571 // Reject elements larger than ELEN.
2572 if (EltVT.getSizeInBits() > Subtarget.getELen())
2573 return false;
2574
2575 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
2576 // Don't use RVV for types that don't fit.
2577 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
2578 return false;
2579
2580 // TODO: Perhaps an artificial restriction, but worth having whilst getting
2581 // the base fixed length RVV support in place.
2582 if (!VT.isPow2VectorType())
2583 return false;
2584
2585 return true;
2586}
2587
2588bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
2589 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
2590}
2591
2592// Return the largest legal scalable vector type that matches VT's element type.
2594 const RISCVSubtarget &Subtarget) {
2595 // This may be called before legal types are setup.
2596 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
2597 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
2598 "Expected legal fixed length vector!");
2599
2600 unsigned MinVLen = Subtarget.getRealMinVLen();
2601 unsigned MaxELen = Subtarget.getELen();
2602
2603 MVT EltVT = VT.getVectorElementType();
2604 switch (EltVT.SimpleTy) {
2605 default:
2606 llvm_unreachable("unexpected element type for RVV container");
2607 case MVT::i1:
2608 case MVT::i8:
2609 case MVT::i16:
2610 case MVT::i32:
2611 case MVT::i64:
2612 case MVT::bf16:
2613 case MVT::f16:
2614 case MVT::f32:
2615 case MVT::f64: {
2616 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
2617 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
2618 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
2619 unsigned NumElts =
2621 NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
2622 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
2623 return MVT::getScalableVectorVT(EltVT, NumElts);
2624 }
2625 }
2626}
2627
2629 const RISCVSubtarget &Subtarget) {
2631 Subtarget);
2632}
2633
2635 return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
2636}
2637
2638// Grow V to consume an entire RVV register.
2640 const RISCVSubtarget &Subtarget) {
2641 assert(VT.isScalableVector() &&
2642 "Expected to convert into a scalable vector!");
2643 assert(V.getValueType().isFixedLengthVector() &&
2644 "Expected a fixed length vector operand!");
2645 SDLoc DL(V);
2646 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
2647 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2648}
2649
2650// Shrink V so it's just big enough to maintain a VT's worth of data.
2652 const RISCVSubtarget &Subtarget) {
2654 "Expected to convert into a fixed length vector!");
2655 assert(V.getValueType().isScalableVector() &&
2656 "Expected a scalable vector operand!");
2657 SDLoc DL(V);
2658 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
2659 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2660}
2661
2662/// Return the type of the mask type suitable for masking the provided
2663/// vector type. This is simply an i1 element type vector of the same
2664/// (possibly scalable) length.
2665static MVT getMaskTypeFor(MVT VecVT) {
2666 assert(VecVT.isVector());
2668 return MVT::getVectorVT(MVT::i1, EC);
2669}
2670
2671/// Creates an all ones mask suitable for masking a vector of type VecTy with
2672/// vector length VL. .
2673static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
2674 SelectionDAG &DAG) {
2675 MVT MaskVT = getMaskTypeFor(VecVT);
2676 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2677}
2678
2679static std::pair<SDValue, SDValue>
2681 const RISCVSubtarget &Subtarget) {
2682 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
2683 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
2684 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
2685 return {Mask, VL};
2686}
2687
2688static std::pair<SDValue, SDValue>
2689getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
2690 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
2691 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2692 SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
2693 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2694 return {Mask, VL};
2695}
2696
2697// Gets the two common "VL" operands: an all-ones mask and the vector length.
2698// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
2699// the vector type that the fixed-length vector is contained in. Otherwise if
2700// VecVT is scalable, then ContainerVT should be the same as VecVT.
2701static std::pair<SDValue, SDValue>
2702getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
2703 const RISCVSubtarget &Subtarget) {
2704 if (VecVT.isFixedLengthVector())
2705 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
2706 Subtarget);
2707 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2708 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
2709}
2710
2712 SelectionDAG &DAG) const {
2713 assert(VecVT.isScalableVector() && "Expected scalable vector");
2714 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
2715 VecVT.getVectorElementCount());
2716}
2717
2718std::pair<unsigned, unsigned>
2720 const RISCVSubtarget &Subtarget) {
2721 assert(VecVT.isScalableVector() && "Expected scalable vector");
2722
2723 unsigned EltSize = VecVT.getScalarSizeInBits();
2724 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
2725
2726 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
2727 unsigned MaxVLMAX =
2728 RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
2729
2730 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
2731 unsigned MinVLMAX =
2732 RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
2733
2734 return std::make_pair(MinVLMAX, MaxVLMAX);
2735}
2736
2737// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
2738// of either is (currently) supported. This can get us into an infinite loop
2739// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
2740// as a ..., etc.
2741// Until either (or both) of these can reliably lower any node, reporting that
2742// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
2743// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
2744// which is not desirable.
2746 EVT VT, unsigned DefinedValues) const {
2747 return false;
2748}
2749
2751 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
2752 // implementation-defined.
2753 if (!VT.isVector())
2755 unsigned DLenFactor = Subtarget.getDLenFactor();
2756 unsigned Cost;
2757 if (VT.isScalableVector()) {
2758 unsigned LMul;
2759 bool Fractional;
2760 std::tie(LMul, Fractional) =
2762 if (Fractional)
2763 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
2764 else
2765 Cost = (LMul * DLenFactor);
2766 } else {
2767 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
2768 }
2769 return Cost;
2770}
2771
2772
2773/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
2774/// is generally quadratic in the number of vreg implied by LMUL. Note that
2775/// operand (index and possibly mask) are handled separately.
2777 return getLMULCost(VT) * getLMULCost(VT);
2778}
2779
2780/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
2781/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
2782/// or may track the vrgather.vv cost. It is implementation-dependent.
2784 return getLMULCost(VT);
2785}
2786
2787/// Return the cost of a vslidedown.vx or vslideup.vx instruction
2788/// for the type VT. (This does not cover the vslide1up or vslide1down
2789/// variants.) Slides may be linear in the number of vregs implied by LMUL,
2790/// or may track the vrgather.vv cost. It is implementation-dependent.
2792 return getLMULCost(VT);
2793}
2794
2795/// Return the cost of a vslidedown.vi or vslideup.vi instruction
2796/// for the type VT. (This does not cover the vslide1up or vslide1down
2797/// variants.) Slides may be linear in the number of vregs implied by LMUL,
2798/// or may track the vrgather.vv cost. It is implementation-dependent.
2800 return getLMULCost(VT);
2801}
2802
2804 const RISCVSubtarget &Subtarget) {
2805 // RISC-V FP-to-int conversions saturate to the destination register size, but
2806 // don't produce 0 for nan. We can use a conversion instruction and fix the
2807 // nan case with a compare and a select.
2808 SDValue Src = Op.getOperand(0);
2809
2810 MVT DstVT = Op.getSimpleValueType();
2811 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2812
2813 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
2814
2815 if (!DstVT.isVector()) {
2816 // For bf16 or for f16 in absense of Zfh, promote to f32, then saturate
2817 // the result.
2818 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
2819 Src.getValueType() == MVT::bf16) {
2820 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
2821 }
2822
2823 unsigned Opc;
2824 if (SatVT == DstVT)
2825 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
2826 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2828 else
2829 return SDValue();
2830 // FIXME: Support other SatVTs by clamping before or after the conversion.
2831
2832 SDLoc DL(Op);
2833 SDValue FpToInt = DAG.getNode(
2834 Opc, DL, DstVT, Src,
2836
2837 if (Opc == RISCVISD::FCVT_WU_RV64)
2838 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
2839
2840 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
2841 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
2843 }
2844
2845 // Vectors.
2846
2847 MVT DstEltVT = DstVT.getVectorElementType();
2848 MVT SrcVT = Src.getSimpleValueType();
2849 MVT SrcEltVT = SrcVT.getVectorElementType();
2850 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
2851 unsigned DstEltSize = DstEltVT.getSizeInBits();
2852
2853 // Only handle saturating to the destination type.
2854 if (SatVT != DstEltVT)
2855 return SDValue();
2856
2857 MVT DstContainerVT = DstVT;
2858 MVT SrcContainerVT = SrcVT;
2859 if (DstVT.isFixedLengthVector()) {
2860 DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget);
2861 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
2862 assert(DstContainerVT.getVectorElementCount() ==
2863 SrcContainerVT.getVectorElementCount() &&
2864 "Expected same element count");
2865 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2866 }
2867
2868 SDLoc DL(Op);
2869
2870 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
2871
2872 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
2873 {Src, Src, DAG.getCondCode(ISD::SETNE),
2874 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2875
2876 // Need to widen by more than 1 step, promote the FP type, then do a widening
2877 // convert.
2878 if (DstEltSize > (2 * SrcEltSize)) {
2879 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
2880 MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
2881 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
2882 }
2883
2884 MVT CvtContainerVT = DstContainerVT;
2885 MVT CvtEltVT = DstEltVT;
2886 if (SrcEltSize > (2 * DstEltSize)) {
2887 CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
2888 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
2889 }
2890
2891 unsigned RVVOpc =
2893 SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
2894
2895 while (CvtContainerVT != DstContainerVT) {
2896 CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
2897 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
2898 // Rounding mode here is arbitrary since we aren't shifting out any bits.
2899 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
2901 Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
2902 }
2903
2904 SDValue SplatZero = DAG.getNode(
2905 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
2906 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
2907 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
2908 Res, DAG.getUNDEF(DstContainerVT), VL);
2909
2910 if (DstVT.isFixedLengthVector())
2911 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
2912
2913 return Res;
2914}
2915
2917 switch (Opc) {
2918 case ISD::FROUNDEVEN:
2920 case ISD::VP_FROUNDEVEN:
2921 return RISCVFPRndMode::RNE;
2922 case ISD::FTRUNC:
2923 case ISD::STRICT_FTRUNC:
2924 case ISD::VP_FROUNDTOZERO:
2925 return RISCVFPRndMode::RTZ;
2926 case ISD::FFLOOR:
2927 case ISD::STRICT_FFLOOR:
2928 case ISD::VP_FFLOOR:
2929 return RISCVFPRndMode::RDN;
2930 case ISD::FCEIL:
2931 case ISD::STRICT_FCEIL:
2932 case ISD::VP_FCEIL:
2933 return RISCVFPRndMode::RUP;
2934 case ISD::FROUND:
2935 case ISD::STRICT_FROUND:
2936 case ISD::VP_FROUND:
2937 return RISCVFPRndMode::RMM;
2938 case ISD::FRINT:
2939 return RISCVFPRndMode::DYN;
2940 }
2941
2943}
2944
2945// Expand vector FTRUNC, FCEIL, FFLOOR, FROUND, VP_FCEIL, VP_FFLOOR, VP_FROUND
2946// VP_FROUNDEVEN, VP_FROUNDTOZERO, VP_FRINT and VP_FNEARBYINT by converting to
2947// the integer domain and back. Taking care to avoid converting values that are
2948// nan or already correct.
2949static SDValue
2951 const RISCVSubtarget &Subtarget) {
2952 MVT VT = Op.getSimpleValueType();
2953 assert(VT.isVector() && "Unexpected type");
2954
2955 SDLoc DL(Op);
2956
2957 SDValue Src = Op.getOperand(0);
2958
2959 MVT ContainerVT = VT;
2960 if (VT.isFixedLengthVector()) {
2961 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2962 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
2963 }
2964
2965 SDValue Mask, VL;
2966 if (Op->isVPOpcode()) {
2967 Mask = Op.getOperand(1);
2968 if (VT.isFixedLengthVector())
2969 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
2970 Subtarget);
2971 VL = Op.getOperand(2);
2972 } else {
2973 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2974 }
2975
2976 // Freeze the source since we are increasing the number of uses.
2977 Src = DAG.getFreeze(Src);
2978
2979 // We do the conversion on the absolute value and fix the sign at the end.
2980 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
2981
2982 // Determine the largest integer that can be represented exactly. This and
2983 // values larger than it don't have any fractional bits so don't need to
2984 // be converted.
2985 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
2986 unsigned Precision = APFloat::semanticsPrecision(FltSem);
2987 APFloat MaxVal = APFloat(FltSem);
2988 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
2989 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
2990 SDValue MaxValNode =
2991 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
2992 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
2993 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
2994
2995 // If abs(Src) was larger than MaxVal or nan, keep it.
2996 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
2997 Mask =
2998 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
2999 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3000 Mask, Mask, VL});
3001
3002 // Truncate to integer and convert back to FP.
3003 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3004 MVT XLenVT = Subtarget.getXLenVT();
3005 SDValue Truncated;
3006
3007 switch (Op.getOpcode()) {
3008 default:
3009 llvm_unreachable("Unexpected opcode");
3010 case ISD::FCEIL:
3011 case ISD::VP_FCEIL:
3012 case ISD::FFLOOR:
3013 case ISD::VP_FFLOOR:
3014 case ISD::FROUND:
3015 case ISD::FROUNDEVEN:
3016 case ISD::VP_FROUND:
3017 case ISD::VP_FROUNDEVEN:
3018 case ISD::VP_FROUNDTOZERO: {
3021 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3022 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3023 break;
3024 }
3025 case ISD::FTRUNC:
3026 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3027 Mask, VL);
3028 break;
3029 case ISD::FRINT:
3030 case ISD::VP_FRINT:
3031 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
3032 break;
3033 case ISD::FNEARBYINT:
3034 case ISD::VP_FNEARBYINT:
3035 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3036 Mask, VL);
3037 break;
3038 }
3039
3040 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3041 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3042 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3043 Mask, VL);
3044
3045 // Restore the original sign so that -0.0 is preserved.
3046 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3047 Src, Src, Mask, VL);
3048
3049 if (!VT.isFixedLengthVector())
3050 return Truncated;
3051
3052 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3053}
3054
3055// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3056// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3057// qNan and coverting the new source to integer and back to FP.
3058static SDValue
3060 const RISCVSubtarget &Subtarget) {
3061 SDLoc DL(Op);
3062 MVT VT = Op.getSimpleValueType();
3063 SDValue Chain = Op.getOperand(0);
3064 SDValue Src = Op.getOperand(1);
3065
3066 MVT ContainerVT = VT;
3067 if (VT.isFixedLengthVector()) {
3068 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3069 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3070 }
3071
3072 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3073
3074 // Freeze the source since we are increasing the number of uses.
3075 Src = DAG.getFreeze(Src);
3076
3077 // Covert sNan to qNan by executing x + x for all unordered elemenet x in Src.
3078 MVT MaskVT = Mask.getSimpleValueType();
3080 DAG.getVTList(MaskVT, MVT::Other),
3081 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3082 DAG.getUNDEF(MaskVT), Mask, VL});
3083 Chain = Unorder.getValue(1);
3085 DAG.getVTList(ContainerVT, MVT::Other),
3086 {Chain, Src, Src, Src, Unorder, VL});
3087 Chain = Src.getValue(1);
3088
3089 // We do the conversion on the absolute value and fix the sign at the end.
3090 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3091
3092 // Determine the largest integer that can be represented exactly. This and
3093 // values larger than it don't have any fractional bits so don't need to
3094 // be converted.
3095 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3096 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3097 APFloat MaxVal = APFloat(FltSem);
3098 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3099 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3100 SDValue MaxValNode =
3101 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3102 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3103 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3104
3105 // If abs(Src) was larger than MaxVal or nan, keep it.
3106 Mask = DAG.getNode(
3107 RISCVISD::SETCC_VL, DL, MaskVT,
3108 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3109
3110 // Truncate to integer and convert back to FP.
3111 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3112 MVT XLenVT = Subtarget.getXLenVT();
3113 SDValue Truncated;
3114
3115 switch (Op.getOpcode()) {
3116 default:
3117 llvm_unreachable("Unexpected opcode");
3118 case ISD::STRICT_FCEIL:
3119 case ISD::STRICT_FFLOOR:
3120 case ISD::STRICT_FROUND:
3124 Truncated = DAG.getNode(
3125 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3126 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3127 break;
3128 }
3129 case ISD::STRICT_FTRUNC:
3130 Truncated =
3132 DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3133 break;
3136 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3137 Mask, VL);
3138 break;
3139 }
3140 Chain = Truncated.getValue(1);
3141
3142 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3143 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3144 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3145 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3146 Truncated, Mask, VL);
3147 Chain = Truncated.getValue(1);
3148 }
3149
3150 // Restore the original sign so that -0.0 is preserved.
3151 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3152 Src, Src, Mask, VL);
3153
3154 if (VT.isFixedLengthVector())
3155 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3156 return DAG.getMergeValues({Truncated, Chain}, DL);
3157}
3158
3159static SDValue
3161 const RISCVSubtarget &Subtarget) {
3162 MVT VT = Op.getSimpleValueType();
3163 if (VT.isVector())
3164 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
3165
3166 if (DAG.shouldOptForSize())
3167 return SDValue();
3168
3169 SDLoc DL(Op);
3170 SDValue Src = Op.getOperand(0);
3171
3172 // Create an integer the size of the mantissa with the MSB set. This and all
3173 // values larger than it don't have any fractional bits so don't need to be
3174 // converted.
3175 const fltSemantics &FltSem = VT.getFltSemantics();
3176 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3177 APFloat MaxVal = APFloat(FltSem);
3178 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3179 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3180 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
3181
3183 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3184 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
3185}
3186
3187// Expand vector LRINT and LLRINT by converting to the integer domain.
3189 const RISCVSubtarget &Subtarget) {
3190 MVT VT = Op.getSimpleValueType();
3191 assert(VT.isVector() && "Unexpected type");
3192
3193 SDLoc DL(Op);
3194 SDValue Src = Op.getOperand(0);
3195 MVT ContainerVT = VT;
3196
3197 if (VT.isFixedLengthVector()) {
3198 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3199 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3200 }
3201
3202 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3203 SDValue Truncated =
3204 DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, ContainerVT, Src, Mask, VL);
3205
3206 if (!VT.isFixedLengthVector())
3207 return Truncated;
3208
3209 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3210}
3211
3212static SDValue
3214 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
3215 SDValue Offset, SDValue Mask, SDValue VL,
3217 if (Passthru.isUndef())
3219 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3220 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3221 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3222}
3223
3224static SDValue
3225getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
3226 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
3227 SDValue VL,
3229 if (Passthru.isUndef())
3231 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3232 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3233 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3234}
3235
3236static MVT getLMUL1VT(MVT VT) {
3238 "Unexpected vector MVT");
3242}
3243
3247 int64_t Addend;
3248};
3249
3250static std::optional<APInt> getExactInteger(const APFloat &APF,
3252 // We will use a SINT_TO_FP to materialize this constant so we should use a
3253 // signed APSInt here.
3254 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
3255 // We use an arbitrary rounding mode here. If a floating-point is an exact
3256 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
3257 // the rounding mode changes the output value, then it is not an exact
3258 // integer.
3260 bool IsExact;
3261 // If it is out of signed integer range, it will return an invalid operation.
3262 // If it is not an exact integer, IsExact is false.
3263 if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
3265 !IsExact)
3266 return std::nullopt;
3267 return ValInt.extractBits(BitWidth, 0);
3268}
3269
3270// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
3271// to the (non-zero) step S and start value X. This can be then lowered as the
3272// RVV sequence (VID * S) + X, for example.
3273// The step S is represented as an integer numerator divided by a positive
3274// denominator. Note that the implementation currently only identifies
3275// sequences in which either the numerator is +/- 1 or the denominator is 1. It
3276// cannot detect 2/3, for example.
3277// Note that this method will also match potentially unappealing index
3278// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
3279// determine whether this is worth generating code for.
3280//
3281// EltSizeInBits is the size of the type that the sequence will be calculated
3282// in, i.e. SEW for build_vectors or XLEN for address calculations.
3283static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
3284 unsigned EltSizeInBits) {
3285 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
3286 if (!cast<BuildVectorSDNode>(Op)->isConstant())
3287 return std::nullopt;
3288 bool IsInteger = Op.getValueType().isInteger();
3289
3290 std::optional<unsigned> SeqStepDenom;
3291 std::optional<APInt> SeqStepNum;
3292 std::optional<APInt> SeqAddend;
3293 std::optional<std::pair<APInt, unsigned>> PrevElt;
3294 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
3295
3296 // First extract the ops into a list of constant integer values. This may not
3297 // be possible for floats if they're not all representable as integers.
3299 const unsigned OpSize = Op.getScalarValueSizeInBits();
3300 for (auto [Idx, Elt] : enumerate(Op->op_values())) {
3301 if (Elt.isUndef()) {
3302 Elts[Idx] = std::nullopt;
3303 continue;
3304 }
3305 if (IsInteger) {
3306 Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
3307 } else {
3308 auto ExactInteger =
3309 getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
3310 if (!ExactInteger)
3311 return std::nullopt;
3312 Elts[Idx] = *ExactInteger;
3313 }
3314 }
3315
3316 for (auto [Idx, Elt] : enumerate(Elts)) {
3317 // Assume undef elements match the sequence; we just have to be careful
3318 // when interpolating across them.
3319 if (!Elt)
3320 continue;
3321
3322 if (PrevElt) {
3323 // Calculate the step since the last non-undef element, and ensure
3324 // it's consistent across the entire sequence.
3325 unsigned IdxDiff = Idx - PrevElt->second;
3326 APInt ValDiff = *Elt - PrevElt->first;
3327
3328 // A zero-value value difference means that we're somewhere in the middle
3329 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
3330 // step change before evaluating the sequence.
3331 if (ValDiff == 0)
3332 continue;
3333
3334 int64_t Remainder = ValDiff.srem(IdxDiff);
3335 // Normalize the step if it's greater than 1.
3336 if (Remainder != ValDiff.getSExtValue()) {
3337 // The difference must cleanly divide the element span.
3338 if (Remainder != 0)
3339 return std::nullopt;
3340 ValDiff = ValDiff.sdiv(IdxDiff);
3341 IdxDiff = 1;
3342 }
3343
3344 if (!SeqStepNum)
3345 SeqStepNum = ValDiff;
3346 else if (ValDiff != SeqStepNum)
3347 return std::nullopt;
3348
3349 if (!SeqStepDenom)
3350 SeqStepDenom = IdxDiff;
3351 else if (IdxDiff != *SeqStepDenom)
3352 return std::nullopt;
3353 }
3354
3355 // Record this non-undef element for later.
3356 if (!PrevElt || PrevElt->first != *Elt)
3357 PrevElt = std::make_pair(*Elt, Idx);
3358 }
3359
3360 // We need to have logged a step for this to count as a legal index sequence.
3361 if (!SeqStepNum || !SeqStepDenom)
3362 return std::nullopt;
3363
3364 // Loop back through the sequence and validate elements we might have skipped
3365 // while waiting for a valid step. While doing this, log any sequence addend.
3366 for (auto [Idx, Elt] : enumerate(Elts)) {
3367 if (!Elt)
3368 continue;
3369 APInt ExpectedVal =
3370 (APInt(EltSizeInBits, Idx) * *SeqStepNum).sdiv(*SeqStepDenom);
3371
3372 APInt Addend = *Elt - ExpectedVal;
3373 if (!SeqAddend)
3374 SeqAddend = Addend;
3375 else if (Addend != SeqAddend)
3376 return std::nullopt;
3377 }
3378
3379 assert(SeqAddend && "Must have an addend if we have a step");
3380
3381 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
3382 SeqAddend->getSExtValue()};
3383}
3384
3385// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
3386// and lower it as a VRGATHER_VX_VL from the source vector.
3387static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
3388 SelectionDAG &DAG,
3389 const RISCVSubtarget &Subtarget) {
3390 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
3391 return SDValue();
3392 SDValue Vec = SplatVal.getOperand(0);
3393 // Only perform this optimization on vectors of the same size for simplicity.
3394 // Don't perform this optimization for i1 vectors.
3395 // FIXME: Support i1 vectors, maybe by promoting to i8?
3396 if (Vec.getValueType() != VT || VT.getVectorElementType() == MVT::i1)
3397 return SDValue();
3398 SDValue Idx = SplatVal.getOperand(1);
3399 // The index must be a legal type.
3400 if (Idx.getValueType() != Subtarget.getXLenVT())
3401 return SDValue();
3402
3403 MVT ContainerVT = VT;
3404 if (VT.isFixedLengthVector()) {
3405 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3406 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3407 }
3408
3409 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3410
3411 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
3412 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
3413
3414 if (!VT.isFixedLengthVector())
3415 return Gather;
3416
3417 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
3418}
3419
3420
3421/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
3422/// which constitute a large proportion of the elements. In such cases we can
3423/// splat a vector with the dominant element and make up the shortfall with
3424/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
3425/// Note that this includes vectors of 2 elements by association. The
3426/// upper-most element is the "dominant" one, allowing us to use a splat to
3427/// "insert" the upper element, and an insert of the lower element at position
3428/// 0, which improves codegen.
3430 const RISCVSubtarget &Subtarget) {
3431 MVT VT = Op.getSimpleValueType();
3432 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3433
3434 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3435
3436 SDLoc DL(Op);
3437 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3438
3439 MVT XLenVT = Subtarget.getXLenVT();
3440 unsigned NumElts = Op.getNumOperands();
3441
3442 SDValue DominantValue;
3443 unsigned MostCommonCount = 0;
3444 DenseMap<SDValue, unsigned> ValueCounts;
3445 unsigned NumUndefElts =
3446 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
3447
3448 // Track the number of scalar loads we know we'd be inserting, estimated as
3449 // any non-zero floating-point constant. Other kinds of element are either
3450 // already in registers or are materialized on demand. The threshold at which
3451 // a vector load is more desirable than several scalar materializion and
3452 // vector-insertion instructions is not known.
3453 unsigned NumScalarLoads = 0;
3454
3455 for (SDValue V : Op->op_values()) {
3456 if (V.isUndef())
3457 continue;
3458
3459 ValueCounts.insert(std::make_pair(V, 0));
3460 unsigned &Count = ValueCounts[V];
3461 if (0 == Count)
3462 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
3463 NumScalarLoads += !CFP->isExactlyValue(+0.0);
3464
3465 // Is this value dominant? In case of a tie, prefer the highest element as
3466 // it's cheaper to insert near the beginning of a vector than it is at the
3467 // end.
3468 if (++Count >= MostCommonCount) {
3469 DominantValue = V;
3470 MostCommonCount = Count;
3471 }
3472 }
3473
3474 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
3475 unsigned NumDefElts = NumElts - NumUndefElts;
3476 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
3477
3478 // Don't perform this optimization when optimizing for size, since
3479 // materializing elements and inserting them tends to cause code bloat.
3480 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
3481 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
3482 ((MostCommonCount > DominantValueCountThreshold) ||
3483 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
3484 // Start by splatting the most common element.
3485 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
3486
3487 DenseSet<SDValue> Processed{DominantValue};
3488
3489 // We can handle an insert into the last element (of a splat) via
3490 // v(f)slide1down. This is slightly better than the vslideup insert
3491 // lowering as it avoids the need for a vector group temporary. It
3492 // is also better than using vmerge.vx as it avoids the need to
3493 // materialize the mask in a vector register.
3494 if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
3495 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
3496 LastOp != DominantValue) {
3497 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3498 auto OpCode =
3500 if (!VT.isFloatingPoint())
3501 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
3502 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
3503 LastOp, Mask, VL);
3504 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
3505 Processed.insert(LastOp);
3506 }
3507
3508 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
3509 for (const auto &OpIdx : enumerate(Op->ops())) {
3510 const SDValue &V = OpIdx.value();
3511 if (V.isUndef() || !Processed.insert(V).second)
3512 continue;
3513 if (ValueCounts[V] == 1) {
3514 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
3515 DAG.getVectorIdxConstant(OpIdx.index(), DL));
3516 } else {
3517 // Blend in all instances of this value using a VSELECT, using a
3518 // mask where each bit signals whether that element is the one
3519 // we're after.
3521 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
3522 return DAG.getConstant(V == V1, DL, XLenVT);
3523 });
3524 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
3525 DAG.getBuildVector(SelMaskTy, DL, Ops),
3526 DAG.getSplatBuildVector(VT, DL, V), Vec);
3527 }
3528 }
3529
3530 return Vec;
3531 }
3532
3533 return SDValue();
3534}
3535
3537 const RISCVSubtarget &Subtarget) {
3538 MVT VT = Op.getSimpleValueType();
3539 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3540
3541 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3542
3543 SDLoc DL(Op);
3544 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3545
3546 MVT XLenVT = Subtarget.getXLenVT();
3547 unsigned NumElts = Op.getNumOperands();
3548
3549 if (VT.getVectorElementType() == MVT::i1) {
3550 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
3551 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
3552 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
3553 }
3554
3555 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
3556 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
3557 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
3558 }
3559
3560 // Lower constant mask BUILD_VECTORs via an integer vector type, in
3561 // scalar integer chunks whose bit-width depends on the number of mask
3562 // bits and XLEN.
3563 // First, determine the most appropriate scalar integer type to use. This
3564 // is at most XLenVT, but may be shrunk to a smaller vector element type
3565 // according to the size of the final vector - use i8 chunks rather than
3566 // XLenVT if we're producing a v8i1. This results in more consistent
3567 // codegen across RV32 and RV64.
3568 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
3569 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
3570 // If we have to use more than one INSERT_VECTOR_ELT then this
3571 // optimization is likely to increase code size; avoid peforming it in
3572 // such a case. We can use a load from a constant pool in this case.
3573 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
3574 return SDValue();
3575 // Now we can create our integer vector type. Note that it may be larger
3576 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
3577 unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
3578 MVT IntegerViaVecVT =
3579 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
3580 IntegerViaVecElts);
3581
3582 uint64_t Bits = 0;
3583 unsigned BitPos = 0, IntegerEltIdx = 0;
3584 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
3585
3586 for (unsigned I = 0; I < NumElts;) {
3587 SDValue V = Op.getOperand(I);
3588 bool BitValue = !V.isUndef() && V->getAsZExtVal();
3589 Bits |= ((uint64_t)BitValue << BitPos);
3590 ++BitPos;
3591 ++I;
3592
3593 // Once we accumulate enough bits to fill our scalar type or process the
3594 // last element, insert into our vector and clear our accumulated data.
3595 if (I % NumViaIntegerBits == 0 || I == NumElts) {
3596 if (NumViaIntegerBits <= 32)
3597 Bits = SignExtend64<32>(Bits);
3598 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
3599 Elts[IntegerEltIdx] = Elt;
3600 Bits = 0;
3601 BitPos = 0;
3602 IntegerEltIdx++;
3603 }
3604 }
3605
3606 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
3607
3608 if (NumElts < NumViaIntegerBits) {
3609 // If we're producing a smaller vector than our minimum legal integer
3610 // type, bitcast to the equivalent (known-legal) mask type, and extract
3611 // our final mask.
3612 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
3613 Vec = DAG.getBitcast(MVT::v8i1, Vec);
3614 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
3615 DAG.getConstant(0, DL, XLenVT));
3616 } else {
3617 // Else we must have produced an integer type with the same size as the
3618 // mask type; bitcast for the final result.
3619 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
3620 Vec = DAG.getBitcast(VT, Vec);
3621 }
3622
3623 return Vec;
3624 }
3625
3626 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
3627 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
3629 if (!VT.isFloatingPoint())
3630 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
3631 Splat =
3632 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
3633 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
3634 }
3635
3636 // Try and match index sequences, which we can lower to the vid instruction
3637 // with optional modifications. An all-undef vector is matched by
3638 // getSplatValue, above.
3639 if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
3640 int64_t StepNumerator = SimpleVID->StepNumerator;
3641 unsigned StepDenominator = SimpleVID->StepDenominator;
3642 int64_t Addend = SimpleVID->Addend;
3643
3644 assert(StepNumerator != 0 && "Invalid step");
3645 bool Negate = false;
3646 int64_t SplatStepVal = StepNumerator;
3647 unsigned StepOpcode = ISD::MUL;
3648 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
3649 // anyway as the shift of 63 won't fit in uimm5.
3650 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
3651 isPowerOf2_64(std::abs(StepNumerator))) {
3652 Negate = StepNumerator < 0;
3653 StepOpcode = ISD::SHL;
3654 SplatStepVal = Log2_64(std::abs(StepNumerator));
3655 }
3656
3657 // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
3658 // threshold since it's the immediate value many RVV instructions accept.
3659 // There is no vmul.vi instruction so ensure multiply constant can fit in
3660 // a single addi instruction.
3661 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
3662 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
3663 isPowerOf2_32(StepDenominator) &&
3664 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
3665 MVT VIDVT =
3667 MVT VIDContainerVT =
3668 getContainerForFixedLengthVector(DAG, VIDVT, Subtarget);
3669 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
3670 // Convert right out of the scalable type so we can use standard ISD
3671 // nodes for the rest of the computation. If we used scalable types with
3672 // these, we'd lose the fixed-length vector info and generate worse
3673 // vsetvli code.
3674 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
3675 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
3676 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
3677 SDValue SplatStep = DAG.getConstant(SplatStepVal, DL, VIDVT);
3678 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
3679 }
3680 if (StepDenominator != 1) {
3681 SDValue SplatStep =
3682 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
3683 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
3684 }
3685 if (Addend != 0 || Negate) {
3686 SDValue SplatAddend = DAG.getConstant(Addend, DL, VIDVT);
3687 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
3688 VID);
3689 }
3690 if (VT.isFloatingPoint()) {
3691 // TODO: Use vfwcvt to reduce register pressure.
3692 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
3693 }
3694 return VID;
3695 }
3696 }
3697
3698 // For very small build_vectors, use a single scalar insert of a constant.
3699 // TODO: Base this on constant rematerialization cost, not size.
3700 const unsigned EltBitSize = VT.getScalarSizeInBits();
3701 if (VT.getSizeInBits() <= 32 &&
3703 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
3704 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
3705 "Unexpected sequence type");
3706 // If we can use the original VL with the modified element type, this
3707 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
3708 // be moved into InsertVSETVLI?
3709 unsigned ViaVecLen =
3710 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
3711 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
3712
3713 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3714 uint64_t SplatValue = 0;
3715 // Construct the amalgamated value at this larger vector type.
3716 for (const auto &OpIdx : enumerate(Op->op_values())) {
3717 const auto &SeqV = OpIdx.value();
3718 if (!SeqV.isUndef())
3719 SplatValue |=
3720 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
3721 }
3722
3723 // On RV64, sign-extend from 32 to 64 bits where possible in order to
3724 // achieve better constant materializion.
3725 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
3726 SplatValue = SignExtend64<32>(SplatValue);
3727
3728 SDValue Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ViaVecVT,
3729 DAG.getUNDEF(ViaVecVT),
3730 DAG.getConstant(SplatValue, DL, XLenVT),
3731 DAG.getVectorIdxConstant(0, DL));
3732 if (ViaVecLen != 1)
3734 MVT::getVectorVT(ViaIntVT, 1), Vec,
3735 DAG.getConstant(0, DL, XLenVT));
3736 return DAG.getBitcast(VT, Vec);
3737 }
3738
3739
3740 // Attempt to detect "hidden" splats, which only reveal themselves as splats
3741 // when re-interpreted as a vector with a larger element type. For example,
3742 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
3743 // could be instead splat as
3744 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
3745 // TODO: This optimization could also work on non-constant splats, but it
3746 // would require bit-manipulation instructions to construct the splat value.
3747 SmallVector<SDValue> Sequence;
3748 const auto *BV = cast<BuildVectorSDNode>(Op);
3749 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
3751 BV->getRepeatedSequence(Sequence) &&
3752 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
3753 unsigned SeqLen = Sequence.size();
3754 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
3755 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
3756 ViaIntVT == MVT::i64) &&
3757 "Unexpected sequence type");
3758
3759 // If we can use the original VL with the modified element type, this
3760 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
3761 // be moved into InsertVSETVLI?
3762 const unsigned RequiredVL = NumElts / SeqLen;
3763 const unsigned ViaVecLen =
3764 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
3765 NumElts : RequiredVL;
3766 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
3767
3768 unsigned EltIdx = 0;
3769 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
3770 uint64_t SplatValue = 0;
3771 // Construct the amalgamated value which can be splatted as this larger
3772 // vector type.
3773 for (const auto &SeqV : Sequence) {
3774 if (!SeqV.isUndef())
3775 SplatValue |=
3776 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
3777 EltIdx++;
3778 }
3779
3780 // On RV64, sign-extend from 32 to 64 bits where possible in order to
3781 // achieve better constant materializion.
3782 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
3783 SplatValue = SignExtend64<32>(SplatValue);
3784
3785 // Since we can't introduce illegal i64 types at this stage, we can only
3786 // perform an i64 splat on RV32 if it is its own sign-extended value. That
3787 // way we can use RVV instructions to splat.
3788 assert((ViaIntVT.bitsLE(XLenVT) ||
3789 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
3790 "Unexpected bitcast sequence");
3791 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
3792 SDValue ViaVL =
3793 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
3794 MVT ViaContainerVT =
3795 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
3796 SDValue Splat =
3797 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
3798 DAG.getUNDEF(ViaContainerVT),
3799 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
3800 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
3801 if (ViaVecLen != RequiredVL)
3803 MVT::getVectorVT(ViaIntVT, RequiredVL), Splat,
3804 DAG.getConstant(0, DL, XLenVT));
3805 return DAG.getBitcast(VT, Splat);
3806 }
3807 }
3808
3809 // If the number of signbits allows, see if we can lower as a <N x i8>.
3810 // Our main goal here is to reduce LMUL (and thus work) required to
3811 // build the constant, but we will also narrow if the resulting
3812 // narrow vector is known to materialize cheaply.
3813 // TODO: We really should be costing the smaller vector. There are
3814 // profitable cases this misses.
3815 if (EltBitSize > 8 && VT.isInteger() &&
3816 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
3817 DAG.ComputeMaxSignificantBits(Op) <= 8) {
3818 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
3819 DL, Op->ops());
3820 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
3821 Source, DAG, Subtarget);
3822 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
3823 return convertFromScalableVector(VT, Res, DAG, Subtarget);
3824 }
3825
3826 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
3827 return Res;
3828
3829 // For constant vectors, use generic constant pool lowering. Otherwise,
3830 // we'd have to materialize constants in GPRs just to move them into the
3831 // vector.
3832 return SDValue();
3833}
3834
3835static unsigned getPACKOpcode(unsigned DestBW,
3836 const RISCVSubtarget &Subtarget) {
3837 switch (DestBW) {
3838 default:
3839 llvm_unreachable("Unsupported pack size");
3840 case 16:
3841 return RISCV::PACKH;
3842 case 32:
3843 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
3844 case 64:
3845 assert(Subtarget.is64Bit());
3846 return RISCV::PACK;
3847 }
3848}
3849
3850/// Double the element size of the build vector to reduce the number
3851/// of vslide1down in the build vector chain. In the worst case, this
3852/// trades three scalar operations for 1 vector operation. Scalar
3853/// operations are generally lower latency, and for out-of-order cores
3854/// we also benefit from additional parallelism.
3856 const RISCVSubtarget &Subtarget) {
3857 SDLoc DL(Op);
3858 MVT VT = Op.getSimpleValueType();
3859 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3860 MVT ElemVT = VT.getVectorElementType();
3861 if (!ElemVT.isInteger())
3862 return SDValue();
3863
3864 // TODO: Relax these architectural restrictions, possibly with costing
3865 // of the actual instructions required.
3866 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
3867 return SDValue();
3868
3869 unsigned NumElts = VT.getVectorNumElements();
3870 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
3871 if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
3872 NumElts % 2 != 0)
3873 return SDValue();
3874
3875 // Produce [B,A] packed into a type twice as wide. Note that all
3876 // scalars are XLenVT, possibly masked (see below).
3877 MVT XLenVT = Subtarget.getXLenVT();
3878 SDValue Mask = DAG.getConstant(
3879 APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
3880 auto pack = [&](SDValue A, SDValue B) {
3881 // Bias the scheduling of the inserted operations to near the
3882 // definition of the element - this tends to reduce register
3883 // pressure overall.
3884 SDLoc ElemDL(B);
3885 if (Subtarget.hasStdExtZbkb())
3886 // Note that we're relying on the high bits of the result being
3887 // don't care. For PACKW, the result is *sign* extended.
3888 return SDValue(
3889 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
3890 ElemDL, XLenVT, A, B),
3891 0);
3892
3893 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
3894 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
3895 SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
3896 SDNodeFlags Flags;
3897 Flags.setDisjoint(true);
3898 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
3899 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt), Flags);
3900 };
3901
3902 SmallVector<SDValue> NewOperands;
3903 NewOperands.reserve(NumElts / 2);
3904 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
3905 NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
3906 assert(NumElts == NewOperands.size() * 2);
3907 MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
3908 MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
3909 return DAG.getNode(ISD::BITCAST, DL, VT,
3910 DAG.getBuildVector(WideVecVT, DL, NewOperands));
3911}
3912
3913// Convert to an vXf16 build_vector to vXi16 with bitcasts.
3915 MVT VT = Op.getSimpleValueType();
3916 MVT IVT = VT.changeVectorElementType(MVT::i16);
3918 for (unsigned I = 0, E = Op.getNumOperands(); I != E; ++I)
3919 NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
3920 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op), IVT, NewOps);
3921 return DAG.getBitcast(VT, Res);
3922}
3923
3925 const RISCVSubtarget &Subtarget) {
3926 MVT VT = Op.getSimpleValueType();
3927 assert(VT.isFixedLengthVector() && "Unexpected vector!");
3928
3929 // If we don't have scalar f16, we need to bitcast to an i16 vector.
3930 if (VT.getVectorElementType() == MVT::f16 &&
3931 !Subtarget.hasStdExtZfhmin())
3932 return lowerBUILD_VECTORvXf16(Op, DAG);
3933
3934 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
3936 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
3937
3938 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3939
3940 SDLoc DL(Op);
3941 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3942
3943 MVT XLenVT = Subtarget.getXLenVT();
3944
3945 if (VT.getVectorElementType() == MVT::i1) {
3946 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
3947 // vector type, we have a legal equivalently-sized i8 type, so we can use
3948 // that.
3949 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
3950 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
3951
3952 SDValue WideVec;
3953 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
3954 // For a splat, perform a scalar truncate before creating the wider
3955 // vector.
3956 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
3957 DAG.getConstant(1, DL, Splat.getValueType()));
3958 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
3959 } else {
3960 SmallVector<SDValue, 8> Ops(Op->op_values());
3961 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
3962 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
3963 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
3964 }
3965
3966 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
3967 }
3968
3969 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
3970 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
3971 return Gather;
3972 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
3974 if (!VT.isFloatingPoint())
3975 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
3976 Splat =
3977 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
3978 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
3979 }
3980
3981 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
3982 return Res;
3983
3984 // If we're compiling for an exact VLEN value, we can split our work per
3985 // register in the register group.
3986 if (const auto VLen = Subtarget.getRealVLen();
3987 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
3988 MVT ElemVT = VT.getVectorElementType();
3989 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
3990 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3991 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
3992 MVT M1VT = getContainerForFixedLengthVector(DAG, OneRegVT, Subtarget);
3993 assert(M1VT == getLMUL1VT(M1VT));
3994
3995 // The following semantically builds up a fixed length concat_vector
3996 // of the component build_vectors. We eagerly lower to scalable and
3997 // insert_subvector here to avoid DAG combining it back to a large
3998 // build_vector.
3999 SmallVector<SDValue> BuildVectorOps(Op->ops());
4000 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4001 SDValue Vec = DAG.getUNDEF(ContainerVT);
4002 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4003 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
4004 SDValue SubBV =
4005 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4006 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
4007 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4008 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT, Vec, SubBV,
4009 DAG.getVectorIdxConstant(InsertIdx, DL));
4010 }
4011 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4012 }
4013
4014 // If we're about to resort to vslide1down (or stack usage), pack our
4015 // elements into the widest scalar type we can. This will force a VL/VTYPE
4016 // toggle, but reduces the critical path, the number of vslide1down ops
4017 // required, and possibly enables scalar folds of the values.
4018 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4019 return Res;
4020
4021 // For m1 vectors, if we have non-undef values in both halves of our vector,
4022 // split the vector into low and high halves, build them separately, then
4023 // use a vselect to combine them. For long vectors, this cuts the critical
4024 // path of the vslide1down sequence in half, and gives us an opportunity
4025 // to special case each half independently. Note that we don't change the
4026 // length of the sub-vectors here, so if both fallback to the generic
4027 // vslide1down path, we should be able to fold the vselect into the final
4028 // vslidedown (for the undef tail) for the first half w/ masking.
4029 unsigned NumElts = VT.getVectorNumElements();
4030 unsigned NumUndefElts =
4031 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4032 unsigned NumDefElts = NumElts - NumUndefElts;
4033 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4034 ContainerVT.bitsLE(getLMUL1VT(ContainerVT))) {
4035 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4036 SmallVector<SDValue> MaskVals;
4037 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4038 SubVecAOps.reserve(NumElts);
4039 SubVecBOps.reserve(NumElts);
4040 for (unsigned i = 0; i < NumElts; i++) {
4041 SDValue Elem = Op->getOperand(i);
4042 if (i < NumElts / 2) {
4043 SubVecAOps.push_back(Elem);
4044 SubVecBOps.push_back(UndefElem);
4045 } else {
4046 SubVecAOps.push_back(UndefElem);
4047 SubVecBOps.push_back(Elem);
4048 }
4049 bool SelectMaskVal = (i < NumElts / 2);
4050 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4051 }
4052 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
4053 MaskVals.size() == NumElts);
4054
4055 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
4056 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
4057 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
4058 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
4059 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4060 }
4061
4062 // Cap the cost at a value linear to the number of elements in the vector.
4063 // The default lowering is to use the stack. The vector store + scalar loads
4064 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
4065 // being (at least) linear in LMUL. As a result, using the vslidedown
4066 // lowering for every element ends up being VL*LMUL..
4067 // TODO: Should we be directly costing the stack alternative? Doing so might
4068 // give us a more accurate upper bound.
4069 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
4070
4071 // TODO: unify with TTI getSlideCost.
4072 InstructionCost PerSlideCost = 1;
4073 switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
4074 default: break;
4076 PerSlideCost = 2;
4077 break;
4079 PerSlideCost = 4;
4080 break;
4082 PerSlideCost = 8;
4083 break;
4084 }
4085
4086 // TODO: Should we be using the build instseq then cost + evaluate scheme
4087 // we use for integer constants here?
4088 unsigned UndefCount = 0;
4089 for (const SDValue &V : Op->ops()) {
4090 if (V.isUndef()) {
4091 UndefCount++;
4092 continue;
4093 }
4094 if (UndefCount) {
4095 LinearBudget -= PerSlideCost;
4096 UndefCount = 0;
4097 }
4098 LinearBudget -= PerSlideCost;
4099 }
4100 if (UndefCount) {
4101 LinearBudget -= PerSlideCost;
4102 }
4103
4104 if (LinearBudget < 0)
4105 return SDValue();
4106
4107 assert((!VT.isFloatingPoint() ||
4108 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
4109 "Illegal type which will result in reserved encoding");
4110
4111 const unsigned Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
4112
4113 SDValue Vec;
4114 UndefCount = 0;
4115 for (SDValue V : Op->ops()) {
4116 if (V.isUndef()) {
4117 UndefCount++;
4118 continue;
4119 }
4120
4121 // Start our sequence with a TA splat in the hopes that hardware is able to
4122 // recognize there's no dependency on the prior value of our temporary
4123 // register.
4124 if (!Vec) {
4125 Vec = DAG.getSplatVector(VT, DL, V);
4126 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4127 UndefCount = 0;
4128 continue;
4129 }
4130
4131 if (UndefCount) {
4132 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4133 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4134 Vec, Offset, Mask, VL, Policy);
4135 UndefCount = 0;
4136 }
4137 auto OpCode =
4139 if (!VT.isFloatingPoint())
4140 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
4141 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4142 V, Mask, VL);
4143 }
4144 if (UndefCount) {
4145 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
4146 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
4147 Vec, Offset, Mask, VL, Policy);
4148 }
4149 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4150}
4151
4152static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
4154 SelectionDAG &DAG) {
4155 if (!Passthru)
4156 Passthru = DAG.getUNDEF(VT);
4157 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
4158 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
4159 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
4160 // If Hi constant is all the same sign bit as Lo, lower this as a custom
4161 // node in order to try and match RVV vector/scalar instructions.
4162 if ((LoC >> 31) == HiC)
4163 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4164
4165 // If vl is equal to VLMAX or fits in 4 bits and Hi constant is equal to Lo,
4166 // we could use vmv.v.x whose EEW = 32 to lower it. This allows us to use
4167 // vlmax vsetvli or vsetivli to change the VL.
4168 // FIXME: Support larger constants?
4169 // FIXME: Support non-constant VLs by saturating?
4170 if (LoC == HiC) {
4171 SDValue NewVL;
4172 if (isAllOnesConstant(VL) ||
4173 (isa<RegisterSDNode>(VL) &&
4174 cast<RegisterSDNode>(VL)->getReg() == RISCV::X0))
4175 NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
4176 else if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
4177 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
4178
4179 if (NewVL) {
4180 MVT InterVT =
4181 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
4182 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
4183 DAG.getUNDEF(InterVT), Lo, NewVL);
4184 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
4185 }
4186 }
4187 }
4188
4189 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
4190 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
4191 isa<ConstantSDNode>(Hi.getOperand(1)) &&
4192 Hi.getConstantOperandVal(1) == 31)
4193 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4194
4195 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
4196 // even if it might be sign extended.
4197 if (Hi.isUndef())
4198 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
4199
4200 // Fall back to a stack store and stride x0 vector load.
4201 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
4202 Hi, VL);
4203}
4204