LLVM 17.0.0git
RISCVISelLowering.cpp
Go to the documentation of this file.
1//===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISCV uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVMachineFunctionInfo.h"
18#include "RISCVRegisterInfo.h"
19#include "RISCVSubtarget.h"
20#include "RISCVTargetMachine.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/Analysis/MemoryLocation.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineJumpTableInfo.h"
28#include "llvm/CodeGen/MachineRegisterInfo.h"
29#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30#include "llvm/CodeGen/ValueTypes.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/DiagnosticPrinter.h"
33#include "llvm/IR/IRBuilder.h"
34#include "llvm/IR/Instructions.h"
35#include "llvm/IR/IntrinsicsRISCV.h"
36#include "llvm/IR/PatternMatch.h"
37#include "llvm/Support/CommandLine.h"
38#include "llvm/Support/Debug.h"
39#include "llvm/Support/ErrorHandling.h"
40#include "llvm/Support/KnownBits.h"
41#include "llvm/Support/MathExtras.h"
42#include "llvm/Support/raw_ostream.h"
43#include <optional>
44
45using namespace llvm;
46
47#define DEBUG_TYPE "riscv-lower"
48
49STATISTIC(NumTailCalls, "Number of tail calls");
50
51static cl::opt<unsigned> ExtensionMaxWebSize(
52 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
53 cl::desc("Give the maximum size (in number of nodes) of the web of "
54 "instructions that we will consider for VW expansion"),
55 cl::init(18));
56
57static cl::opt<bool>
58 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
59 cl::desc("Allow the formation of VW_W operations (e.g., "
60 "VWADD_W) with splat constants"),
61 cl::init(false));
62
63static cl::opt<unsigned> NumRepeatedDivisors(
64 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
65 cl::desc("Set the minimum number of repetitions of a divisor to allow "
66 "transformation to multiplications by the reciprocal"),
67 cl::init(2));
68
69static cl::opt<int>
70 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
71 cl::desc("Give the maximum number of instructions that we will "
72 "use for creating a floating-point immediate value"),
73 cl::init(2));
74
75RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
76 const RISCVSubtarget &STI)
77 : TargetLowering(TM), Subtarget(STI) {
78
79 if (Subtarget.isRV32E())
80 report_fatal_error("Codegen not yet implemented for RV32E");
81
82 RISCVABI::ABI ABI = Subtarget.getTargetABI();
83 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
84
85 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
86 !Subtarget.hasStdExtF()) {
87 errs() << "Hard-float 'f' ABI can't be used for a target that "
88 "doesn't support the F instruction set extension (ignoring "
89 "target-abi)\n";
90 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
91 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
92 !Subtarget.hasStdExtD()) {
93 errs() << "Hard-float 'd' ABI can't be used for a target that "
94 "doesn't support the D instruction set extension (ignoring "
95 "target-abi)\n";
96 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
97 }
98
99 switch (ABI) {
100 default:
101 report_fatal_error("Don't know how to lower this ABI");
102 case RISCVABI::ABI_ILP32:
103 case RISCVABI::ABI_ILP32F:
104 case RISCVABI::ABI_ILP32D:
105 case RISCVABI::ABI_LP64:
106 case RISCVABI::ABI_LP64F:
107 case RISCVABI::ABI_LP64D:
108 break;
109 }
110
111 MVT XLenVT = Subtarget.getXLenVT();
112
113 // Set up the register classes.
114 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
115
116 if (Subtarget.hasStdExtZfhOrZfhmin())
117 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
118 if (Subtarget.hasStdExtF())
119 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
120 if (Subtarget.hasStdExtD())
121 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
122
123 static const MVT::SimpleValueType BoolVecVTs[] = {
124 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
125 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
126 static const MVT::SimpleValueType IntVecVTs[] = {
127 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
128 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
129 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
130 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
131 MVT::nxv4i64, MVT::nxv8i64};
132 static const MVT::SimpleValueType F16VecVTs[] = {
133 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
134 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
135 static const MVT::SimpleValueType F32VecVTs[] = {
136 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
137 static const MVT::SimpleValueType F64VecVTs[] = {
138 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
139
140 if (Subtarget.hasVInstructions()) {
141 auto addRegClassForRVV = [this](MVT VT) {
142 // Disable the smallest fractional LMUL types if ELEN is less than
143 // RVVBitsPerBlock.
144 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELEN();
145 if (VT.getVectorMinNumElements() < MinElts)
146 return;
147
148 unsigned Size = VT.getSizeInBits().getKnownMinValue();
149 const TargetRegisterClass *RC;
150 if (Size <= RISCV::RVVBitsPerBlock)
151 RC = &RISCV::VRRegClass;
152 else if (Size == 2 * RISCV::RVVBitsPerBlock)
153 RC = &RISCV::VRM2RegClass;
154 else if (Size == 4 * RISCV::RVVBitsPerBlock)
155 RC = &RISCV::VRM4RegClass;
156 else if (Size == 8 * RISCV::RVVBitsPerBlock)
157 RC = &RISCV::VRM8RegClass;
158 else
159 llvm_unreachable("Unexpected size");
160
161 addRegisterClass(VT, RC);
162 };
163
164 for (MVT VT : BoolVecVTs)
165 addRegClassForRVV(VT);
166 for (MVT VT : IntVecVTs) {
167 if (VT.getVectorElementType() == MVT::i64 &&
168 !Subtarget.hasVInstructionsI64())
169 continue;
170 addRegClassForRVV(VT);
171 }
172
173 if (Subtarget.hasVInstructionsF16())
174 for (MVT VT : F16VecVTs)
175 addRegClassForRVV(VT);
176
177 if (Subtarget.hasVInstructionsF32())
178 for (MVT VT : F32VecVTs)
179 addRegClassForRVV(VT);
180
181 if (Subtarget.hasVInstructionsF64())
182 for (MVT VT : F64VecVTs)
183 addRegClassForRVV(VT);
184
185 if (Subtarget.useRVVForFixedLengthVectors()) {
186 auto addRegClassForFixedVectors = [this](MVT VT) {
187 MVT ContainerVT = getContainerForFixedLengthVector(VT);
188 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
189 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
190 addRegisterClass(VT, TRI.getRegClass(RCID));
191 };
192 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
193 if (useRVVForFixedLengthVectorVT(VT))
194 addRegClassForFixedVectors(VT);
195
196 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
197 if (useRVVForFixedLengthVectorVT(VT))
198 addRegClassForFixedVectors(VT);
199 }
200 }
201
202 // Compute derived properties from the register classes.
203 computeRegisterProperties(STI.getRegisterInfo());
204
205 setStackPointerRegisterToSaveRestore(RISCV::X2);
206
207 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
208 MVT::i1, Promote);
209 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
210 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
211 MVT::i1, Promote);
212
213 // TODO: add all necessary setOperationAction calls.
214 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
215
216 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
217 setOperationAction(ISD::BR_CC, XLenVT, Expand);
218 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
219 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
220
221 setCondCodeAction(ISD::SETLE, XLenVT, Expand);
222 setCondCodeAction(ISD::SETGT, XLenVT, Custom);
223 setCondCodeAction(ISD::SETGE, XLenVT, Expand);
224 setCondCodeAction(ISD::SETULE, XLenVT, Expand);
225 setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
226 setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
227
228 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
229
230 setOperationAction(ISD::VASTART, MVT::Other, Custom);
231 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
232
233 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
234
235 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
236
237 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb())
238 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
239
240 if (Subtarget.is64Bit()) {
241 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
242
243 setOperationAction(ISD::LOAD, MVT::i32, Custom);
244
245 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
246 MVT::i32, Custom);
247
248 setOperationAction({ISD::UADDO, ISD::USUBO, ISD::UADDSAT, ISD::USUBSAT},
249 MVT::i32, Custom);
250 } else {
251 setLibcallName(
252 {RTLIB::SHL_I128, RTLIB::SRL_I128, RTLIB::SRA_I128, RTLIB::MUL_I128},
253 nullptr);
254 setLibcallName(RTLIB::MULO_I64, nullptr);
255 }
256
257 if (!Subtarget.hasStdExtM() && !Subtarget.hasStdExtZmmul()) {
258 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
259 } else {
260 if (Subtarget.is64Bit()) {
261 setOperationAction(ISD::MUL, {MVT::i32, MVT::i128}, Custom);
262 } else {
263 setOperationAction(ISD::MUL, MVT::i64, Custom);
264 }
265 }
266
267 if (!Subtarget.hasStdExtM()) {
268 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM},
269 XLenVT, Expand);
270 } else {
271 if (Subtarget.is64Bit()) {
272 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
273 {MVT::i8, MVT::i16, MVT::i32}, Custom);
274 }
275 }
276
277 setOperationAction(
278 {ISD::SDIVREM, ISD::UDIVREM, ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT,
279 Expand);
280
281 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
282 Custom);
283
284 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
285 Subtarget.hasVendorXTHeadBb()) {
286 if (Subtarget.is64Bit())
287 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
288 } else {
289 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
290 }
291
292 // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
293 // pattern match it directly in isel.
294 setOperationAction(ISD::BSWAP, XLenVT,
295 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
296 Subtarget.hasVendorXTHeadBb())
297 ? Legal
298 : Expand);
299 // Zbkb can use rev8+brev8 to implement bitreverse.
300 setOperationAction(ISD::BITREVERSE, XLenVT,
301 Subtarget.hasStdExtZbkb() ? Custom : Expand);
302
303 if (Subtarget.hasStdExtZbb()) {
304 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
305 Legal);
306
307 if (Subtarget.is64Bit())
308 setOperationAction(
309 {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF},
310 MVT::i32, Custom);
311 } else {
312 setOperationAction({ISD::CTTZ, ISD::CTLZ, ISD::CTPOP}, XLenVT, Expand);
313 }
314
315 if (Subtarget.hasVendorXTHeadBb()) {
316 setOperationAction(ISD::CTLZ, XLenVT, Legal);
317
318 // We need the custom lowering to make sure that the resulting sequence
319 // for the 32bit case is efficient on 64bit targets.
320 if (Subtarget.is64Bit())
321 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, MVT::i32, Custom);
322 }
323
324 if (Subtarget.is64Bit())
325 setOperationAction(ISD::ABS, MVT::i32, Custom);
326
327 if (!Subtarget.hasVendorXVentanaCondOps() &&
328 !Subtarget.hasVendorXTHeadCondMov())
329 setOperationAction(ISD::SELECT, XLenVT, Custom);
330
331 static const unsigned FPLegalNodeTypes[] = {
332 ISD::FMINNUM, ISD::FMAXNUM, ISD::LRINT,
333 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
334 ISD::STRICT_LRINT, ISD::STRICT_LLRINT, ISD::STRICT_LROUND,
335 ISD::STRICT_LLROUND, ISD::STRICT_FMA, ISD::STRICT_FADD,
336 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV,
337 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS};
338
339 static const ISD::CondCode FPCCToExpand[] = {
340 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
341 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
342 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
343
344 static const unsigned FPOpToExpand[] = {
345 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW,
346 ISD::FREM, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
347
348 static const unsigned FPRndMode[] = {
349 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
350 ISD::FROUNDEVEN};
351
352 if (Subtarget.hasStdExtZfhOrZfhmin())
353 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
354
355 if (Subtarget.hasStdExtZfhOrZfhmin()) {
356 if (Subtarget.hasStdExtZfh()) {
357 setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
358 setOperationAction(FPRndMode, MVT::f16,
359 Subtarget.hasStdExtZfa() ? Legal : Custom);
360 setOperationAction(ISD::SELECT, MVT::f16, Custom);
361 } else {
362 static const unsigned ZfhminPromoteOps[] = {
363 ISD::FMINNUM, ISD::FMAXNUM, ISD::FADD,
364 ISD::FSUB, ISD::FMUL, ISD::FMA,
365 ISD::FDIV, ISD::FSQRT, ISD::FABS,
366 ISD::FNEG, ISD::STRICT_FMA, ISD::STRICT_FADD,
367 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV,
368 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
369 ISD::SETCC, ISD::FCEIL, ISD::FFLOOR,
370 ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
371 ISD::FROUNDEVEN, ISD::SELECT};
372
373 setOperationAction(ZfhminPromoteOps, MVT::f16, Promote);
374 setOperationAction({ISD::STRICT_LRINT, ISD::STRICT_LLRINT,
375 ISD::STRICT_LROUND, ISD::STRICT_LLROUND},
376 MVT::f16, Legal);
377 // FIXME: Need to promote f16 FCOPYSIGN to f32, but the
378 // DAGCombiner::visitFP_ROUND probably needs improvements first.
379 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Expand);
380 }
381
382 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
383 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
384 setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
385 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
386 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
387
388 setOperationAction(ISD::FNEARBYINT, MVT::f16,
389 Subtarget.hasStdExtZfa() ? Legal : Promote);
390 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
391 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
392 ISD::FEXP2, ISD::FLOG, ISD::FLOG2, ISD::FLOG10},
393 MVT::f16, Promote);
394
395 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
396 // complete support for all operations in LegalizeDAG.
397 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
398 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
399 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
400 ISD::STRICT_FTRUNC},
401 MVT::f16, Promote);
402
403 // We need to custom promote this.
404 if (Subtarget.is64Bit())
405 setOperationAction(ISD::FPOWI, MVT::i32, Custom);
406 }
407
408 if (Subtarget.hasStdExtF()) {
409 setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
410 setOperationAction(FPRndMode, MVT::f32,
411 Subtarget.hasStdExtZfa() ? Legal : Custom);
412 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
413 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
414 setOperationAction(ISD::SELECT, MVT::f32, Custom);
415 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
416 setOperationAction(FPOpToExpand, MVT::f32, Expand);
417 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
418 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
419
420 if (Subtarget.hasStdExtZfa())
421 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
422 }
423
424 if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
425 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
426
427 if (Subtarget.hasStdExtD()) {
428 setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
429
430 if (Subtarget.hasStdExtZfa()) {
431 setOperationAction(FPRndMode, MVT::f64, Legal);
432 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
433 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
434 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
435 }
436
437 if (Subtarget.is64Bit())
438 setOperationAction(FPRndMode, MVT::f64,
439 Subtarget.hasStdExtZfa() ? Legal : Custom);
440
441 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
442 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
443 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
444 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
445 setOperationAction(ISD::SELECT, MVT::f64, Custom);
446 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
447 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
448 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
449 setOperationAction(FPOpToExpand, MVT::f64, Expand);
450 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
451 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
452 }
453
454 if (Subtarget.is64Bit())
455 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
456 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
457 MVT::i32, Custom);
458
459 if (Subtarget.hasStdExtF()) {
460 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
461 Custom);
462
463 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT,
464 ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP},
465 XLenVT, Legal);
466
467 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
468 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
469 }
470
471 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
472 ISD::JumpTable},
473 XLenVT, Custom);
474
475 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
476
477 if (Subtarget.is64Bit())
478 setOperationAction(ISD::Constant, MVT::i64, Custom);
479
480 // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
481 // Unfortunately this can't be determined just from the ISA naming string.
482 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
483 Subtarget.is64Bit() ? Legal : Custom);
484
485 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
486 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
487 if (Subtarget.is64Bit())
488 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
489
490 if (Subtarget.hasStdExtA()) {
491 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
492 setMinCmpXchgSizeInBits(32);
493 } else if (Subtarget.hasForcedAtomics()) {
494 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
495 } else {
496 setMaxAtomicSizeInBitsSupported(0);
497 }
498
499 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
500
501 setBooleanContents(ZeroOrOneBooleanContent);
502
503 if (Subtarget.hasVInstructions()) {
504 setBooleanVectorContents(ZeroOrOneBooleanContent);
505
506 setOperationAction(ISD::VSCALE, XLenVT, Custom);
507
508 // RVV intrinsics may have illegal operands.
509 // We also need to custom legalize vmv.x.s.
510 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
511 {MVT::i8, MVT::i16}, Custom);
512 if (Subtarget.is64Bit())
513 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
514 else
515 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
516 MVT::i64, Custom);
517
518 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
519 MVT::Other, Custom);
520
521 static const unsigned IntegerVPOps[] = {
522 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
523 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
524 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
525 ISD::VP_XOR, ISD::VP_ASHR, ISD::VP_LSHR,
526 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
527 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
528 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
529 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
530 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
531 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
532 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
533 ISD::VP_ABS};
534
535 static const unsigned FloatingPointVPOps[] = {
536 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
537 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
538 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
539 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
540 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
541 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
542 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
543 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
544 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
545 ISD::VP_FRINT, ISD::VP_FNEARBYINT};
546
547 static const unsigned IntegerVecReduceOps[] = {
548 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
549 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
550 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
551
552 static const unsigned FloatingPointVecReduceOps[] = {
553 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
554 ISD::VECREDUCE_FMAX};
555
556 if (!Subtarget.is64Bit()) {
557 // We must custom-lower certain vXi64 operations on RV32 due to the vector
558 // element type being illegal.
559 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
560 MVT::i64, Custom);
561
562 setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
563
564 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
565 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
566 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
567 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
568 MVT::i64, Custom);
569 }
570
571 for (MVT VT : BoolVecVTs) {
572 if (!isTypeLegal(VT))
573 continue;
574
575 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
576
577 // Mask VTs are custom-expanded into a series of standard nodes
578 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
579 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
580 VT, Custom);
581
582 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
583 Custom);
584
585 setOperationAction(ISD::SELECT, VT, Custom);
586 setOperationAction(
587 {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_MERGE, ISD::VP_SELECT}, VT,
588 Expand);
589
590 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Custom);
591
592 setOperationAction(
593 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
594 Custom);
595
596 setOperationAction(
597 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
598 Custom);
599
600 // RVV has native int->float & float->int conversions where the
601 // element type sizes are within one power-of-two of each other. Any
602 // wider distances between type sizes have to be lowered as sequences
603 // which progressively narrow the gap in stages.
604 setOperationAction(
605 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
606 VT, Custom);
607 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
608 Custom);
609
610 // Expand all extending loads to types larger than this, and truncating
611 // stores from types larger than this.
612 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
613 setTruncStoreAction(OtherVT, VT, Expand);
614 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, OtherVT,
615 VT, Expand);
616 }
617
618 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
619 ISD::VP_TRUNCATE, ISD::VP_SETCC},
620 VT, Custom);
621
622 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
623 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
624
625 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
626
627 setOperationPromotedToType(
628 ISD::VECTOR_SPLICE, VT,
629 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
630 }
631
632 for (MVT VT : IntVecVTs) {
633 if (!isTypeLegal(VT))
634 continue;
635
636 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
637 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
638
639 // Vectors implement MULHS/MULHU.
640 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
641
642 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
643 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
644 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
645
646 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
647 Legal);
648
649 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
650
651 setOperationAction({ISD::CTTZ, ISD::CTLZ, ISD::CTPOP}, VT, Expand);
652
653 setOperationAction(ISD::BSWAP, VT, Expand);
654 setOperationAction({ISD::VP_BSWAP, ISD::VP_BITREVERSE}, VT, Expand);
655 setOperationAction({ISD::VP_FSHL, ISD::VP_FSHR}, VT, Expand);
656 setOperationAction({ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
657 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
658 VT, Expand);
659
660 // Custom-lower extensions and truncations from/to mask types.
661 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
662 VT, Custom);
663
664 // RVV has native int->float & float->int conversions where the
665 // element type sizes are within one power-of-two of each other. Any
666 // wider distances between type sizes have to be lowered as sequences
667 // which progressively narrow the gap in stages.
668 setOperationAction(
669 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT},
670 VT, Custom);
671 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
672 Custom);
673
674 setOperationAction(
675 {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT, Legal);
676
677 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
678 // nodes which truncate by one power of two at a time.
679 setOperationAction(ISD::TRUNCATE, VT, Custom);
680
681 // Custom-lower insert/extract operations to simplify patterns.
682 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
683 Custom);
684
685 // Custom-lower reduction operations to set up the corresponding custom
686 // nodes' operands.
687 setOperationAction(IntegerVecReduceOps, VT, Custom);
688
689 setOperationAction(IntegerVPOps, VT, Custom);
690
691 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
692
693 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
694 VT, Custom);
695
696 setOperationAction(
697 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
698 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
699 VT, Custom);
700
701 setOperationAction(
702 {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
703 VT, Custom);
704
705 setOperationAction(ISD::SELECT, VT, Custom);
706 setOperationAction(ISD::SELECT_CC, VT, Expand);
707
708 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
709
710 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
711 setTruncStoreAction(VT, OtherVT, Expand);
712 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, OtherVT,
713 VT, Expand);
714 }
715
716 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
717 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
718
719 // Splice
720 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
721
722 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the range
723 // of f32.
724 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
725 if (isTypeLegal(FloatVT)) {
726 setOperationAction(
727 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
728 Custom);
729 }
730 }
731
732 // Expand various CCs to best match the RVV ISA, which natively supports UNE
733 // but no other unordered comparisons, and supports all ordered comparisons
734 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
735 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
736 // and we pattern-match those back to the "original", swapping operands once
737 // more. This way we catch both operations and both "vf" and "fv" forms with
738 // fewer patterns.
739 static const ISD::CondCode VFPCCToExpand[] = {
740 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
741 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
742 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
743 };
744
745 // Sets common operation actions on RVV floating-point vector types.
746 const auto SetCommonVFPActions = [&](MVT VT) {
747 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
748 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
749 // sizes are within one power-of-two of each other. Therefore conversions
750 // between vXf16 and vXf64 must be lowered as sequences which convert via
751 // vXf32.
752 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
753 // Custom-lower insert/extract operations to simplify patterns.
754 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
755 Custom);
756 // Expand various condition codes (explained above).
757 setCondCodeAction(VFPCCToExpand, VT, Expand);
758
759 setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, VT, Legal);
760
761 setOperationAction(
762 {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND, ISD::FROUNDEVEN},
763 VT, Custom);
764
765 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
766
767 // Expand FP operations that need libcalls.
768 setOperationAction(ISD::FREM, VT, Expand);
769 setOperationAction(ISD::FPOW, VT, Expand);
770 setOperationAction(ISD::FCOS, VT, Expand);
771 setOperationAction(ISD::FSIN, VT, Expand);
772 setOperationAction(ISD::FSINCOS, VT, Expand);
773 setOperationAction(ISD::FEXP, VT, Expand);
774 setOperationAction(ISD::FEXP2, VT, Expand);
775 setOperationAction(ISD::FLOG, VT, Expand);
776 setOperationAction(ISD::FLOG2, VT, Expand);
777 setOperationAction(ISD::FLOG10, VT, Expand);
778 setOperationAction(ISD::FRINT, VT, Expand);
779 setOperationAction(ISD::FNEARBYINT, VT, Expand);
780
781 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
782
783 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
784
785 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
786 VT, Custom);
787
788 setOperationAction(
789 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
790 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
791 VT, Custom);
792
793 setOperationAction(ISD::SELECT, VT, Custom);
794 setOperationAction(ISD::SELECT_CC, VT, Expand);
795
796 setOperationAction(
797 {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR},
798 VT, Custom);
799
800 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
801 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
802
803 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE}, VT, Custom);
804
805 setOperationAction(FloatingPointVPOps, VT, Custom);
806
807 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
808 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
809 ISD::STRICT_FDIV},
810 VT, Legal);
811 };
812
813 // Sets common extload/truncstore actions on RVV floating-point vector
814 // types.
815 const auto SetCommonVFPExtLoadTruncStoreActions =
816 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
817 for (auto SmallVT : SmallerVTs) {
818 setTruncStoreAction(VT, SmallVT, Expand);
819 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
820 }
821 };
822
823 if (Subtarget.hasVInstructionsF16()) {
824 for (MVT VT : F16VecVTs) {
825 if (!isTypeLegal(VT))
826 continue;
827 SetCommonVFPActions(VT);
828 }
829 }
830
831 if (Subtarget.hasVInstructionsF32()) {
832 for (MVT VT : F32VecVTs) {
833 if (!isTypeLegal(VT))
834 continue;
835 SetCommonVFPActions(VT);
836 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
837 }
838 }
839
840 if (Subtarget.hasVInstructionsF64()) {
841 for (MVT VT : F64VecVTs) {
842 if (!isTypeLegal(VT))
843 continue;
844 SetCommonVFPActions(VT);
845 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
846 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
847 }
848 }
849
850 if (Subtarget.useRVVForFixedLengthVectors()) {
851 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
852 if (!useRVVForFixedLengthVectorVT(VT))
853 continue;
854
855 // By default everything must be expanded.
856 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
857 setOperationAction(Op, VT, Expand);
858 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
859 setTruncStoreAction(VT, OtherVT, Expand);
860 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD},
861 OtherVT, VT, Expand);
862 }
863
864 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
865 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
866 Custom);
867
868 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS}, VT,
869 Custom);
870
871 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
872 VT, Custom);
873
874 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
875
876 setOperationAction(ISD::SETCC, VT, Custom);
877
878 setOperationAction(ISD::SELECT, VT, Custom);
879
880 setOperationAction(ISD::TRUNCATE, VT, Custom);
881
882 setOperationAction(ISD::BITCAST, VT, Custom);
883
884 setOperationAction(
885 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
886 Custom);
887
888 setOperationAction(
889 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
890 Custom);
891
892 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
893 ISD::FP_TO_UINT},
894 VT, Custom);
895 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
896 Custom);
897
898 // Operations below are different for between masks and other vectors.
899 if (VT.getVectorElementType() == MVT::i1) {
900 setOperationAction({ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND,
901 ISD::OR, ISD::XOR},
902 VT, Custom);
903
904 setOperationAction({ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
905 ISD::VP_SETCC, ISD::VP_TRUNCATE},
906 VT, Custom);
907 continue;
908 }
909
910 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
911 // it before type legalization for i64 vectors on RV32. It will then be
912 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
913 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
914 // improvements first.
915 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
916 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
917 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
918 }
919
920 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
921
922 setOperationAction(
923 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
924
925 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
926 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
927 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
928 ISD::VP_SCATTER},
929 VT, Custom);
930
931 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
932 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
933 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
934 VT, Custom);
935
936 setOperationAction(
937 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
938
939 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
940 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
941 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
942
943 setOperationAction(
944 {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT,
945 Custom);
946
947 setOperationAction(ISD::VSELECT, VT, Custom);
948 setOperationAction(ISD::SELECT_CC, VT, Expand);
949
950 setOperationAction(
951 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
952
953 // Custom-lower reduction operations to set up the corresponding custom
954 // nodes' operands.
955 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
956 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
957 ISD::VECREDUCE_UMIN},
958 VT, Custom);
959
960 setOperationAction(IntegerVPOps, VT, Custom);
961
962 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
963 // range of f32.
964 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
965 if (isTypeLegal(FloatVT))
966 setOperationAction(
967 {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
968 Custom);
969 }
970
971 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
972 if (!useRVVForFixedLengthVectorVT(VT))
973 continue;
974
975 // By default everything must be expanded.
976 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
977 setOperationAction(Op, VT, Expand);
978 for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
979 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
980 setTruncStoreAction(VT, OtherVT, Expand);
981 }
982
983 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
984 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
985 Custom);
986
987 setOperationAction({ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS,
988 ISD::VECTOR_SHUFFLE, ISD::INSERT_VECTOR_ELT,
989 ISD::EXTRACT_VECTOR_ELT},
990 VT, Custom);
991
992 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
993 ISD::MGATHER, ISD::MSCATTER},
994 VT, Custom);
995
996 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
997 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
998 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
999 ISD::VP_SCATTER},
1000 VT, Custom);
1001
1002 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1003 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1004 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM},
1005 VT, Custom);
1006
1007 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1008
1009 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1010 ISD::FROUNDEVEN},
1011 VT, Custom);
1012
1013 setCondCodeAction(VFPCCToExpand, VT, Expand);
1014
1015 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1016 setOperationAction(ISD::SELECT_CC, VT, Expand);
1017
1018 setOperationAction(ISD::BITCAST, VT, Custom);
1019
1020 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1021
1022 setOperationAction(FloatingPointVPOps, VT, Custom);
1023
1024 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
1025 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB,
1026 ISD::STRICT_FMUL, ISD::STRICT_FDIV},
1027 VT, Custom);
1028 }
1029
1030 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1031 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32, MVT::i64},
1032 Custom);
1033 if (Subtarget.hasStdExtZfhOrZfhmin())
1034 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1035 if (Subtarget.hasStdExtF())
1036 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1037 if (Subtarget.hasStdExtD())
1038 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1039 }
1040 }
1041
1042 if (Subtarget.hasForcedAtomics()) {
1043 // Set atomic rmw/cas operations to expand to force __sync libcalls.
1044 setOperationAction(
1045 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1046 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1047 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1048 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1049 XLenVT, Expand);
1050 }
1051
1052 if (Subtarget.hasVendorXTHeadMemIdx()) {
1053 for (unsigned im = (unsigned)ISD::PRE_INC; im != (unsigned)ISD::POST_DEC;
1054 ++im) {
1055 setIndexedLoadAction(im, MVT::i8, Legal);
1056 setIndexedStoreAction(im, MVT::i8, Legal);
1057 setIndexedLoadAction(im, MVT::i16, Legal);
1058 setIndexedStoreAction(im, MVT::i16, Legal);
1059 setIndexedLoadAction(im, MVT::i32, Legal);
1060 setIndexedStoreAction(im, MVT::i32, Legal);
1061
1062 if (Subtarget.is64Bit()) {
1063 setIndexedLoadAction(im, MVT::i64, Legal);
1064 setIndexedStoreAction(im, MVT::i64, Legal);
1065 }
1066 }
1067 }
1068
1069 // Function alignments.
1070 const Align FunctionAlignment(Subtarget.hasStdExtCOrZca() ? 2 : 4);
1071 setMinFunctionAlignment(FunctionAlignment);
1072 // Set preferred alignments.
1073 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
1074 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
1075
1076 setMinimumJumpTableEntries(5);
1077
1078 // Jumps are expensive, compared to logic
1079 setJumpIsExpensive();
1080
1081 setTargetDAGCombine({ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::AND,
1082 ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT});
1083 if (Subtarget.is64Bit())
1084 setTargetDAGCombine(ISD::SRA);
1085
1086 if (Subtarget.hasStdExtF())
1087 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM});
1088
1089 if (Subtarget.hasStdExtZbb())
1090 setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN});
1091
1092 if (Subtarget.hasStdExtZbs() && Subtarget.is64Bit())
1093 setTargetDAGCombine(ISD::TRUNCATE);
1094
1095 if (Subtarget.hasStdExtZbkb())
1096 setTargetDAGCombine(ISD::BITREVERSE);
1097 if (Subtarget.hasStdExtZfhOrZfhmin())
1098 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1099 if (Subtarget.hasStdExtF())
1100 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1101 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
1102 if (Subtarget.hasVInstructions())
1103 setTargetDAGCombine({ISD::FCOPYSIGN, ISD::MGATHER, ISD::MSCATTER,
1104 ISD::VP_GATHER, ISD::VP_SCATTER, ISD::SRA, ISD::SRL,
1105 ISD::SHL, ISD::STORE, ISD::SPLAT_VECTOR});
1106 if (Subtarget.hasVendorXTHeadMemPair())
1107 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
1108 if (Subtarget.useRVVForFixedLengthVectors())
1109 setTargetDAGCombine(ISD::BITCAST);
1110
1111 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
1112 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
1113}
1114
1115EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1116 LLVMContext &Context,
1117 EVT VT) const {
1118 if (!VT.isVector())
1119 return getPointerTy(DL);
1120 if (Subtarget.hasVInstructions() &&
1121 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1122 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
1123 return VT.changeVectorElementTypeToInteger();
1124}
1125
1126MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1127 return Subtarget.getXLenVT();
1128}
1129
1130bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1131 const CallInst &I,
1132 MachineFunction &MF,
1133 unsigned Intrinsic) const {
1134 auto &DL = I.getModule()->getDataLayout();
1135 switch (Intrinsic) {
1136 default:
1137 return false;
1138 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
1139 case Intrinsic::riscv_masked_atomicrmw_add_i32:
1140 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
1141 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
1142 case Intrinsic::riscv_masked_atomicrmw_max_i32:
1143 case Intrinsic::riscv_masked_atomicrmw_min_i32:
1144 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
1145 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
1146 case Intrinsic::riscv_masked_cmpxchg_i32:
1147 Info.opc = ISD::INTRINSIC_W_CHAIN;
1148 Info.memVT = MVT::i32;
1149 Info.ptrVal = I.getArgOperand(0);
1150 Info.offset = 0;
1151 Info.align = Align(4);
1152 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
1153 MachineMemOperand::MOVolatile;
1154 return true;
1155 case Intrinsic::riscv_masked_strided_load:
1156 Info.opc = ISD::INTRINSIC_W_CHAIN;
1157 Info.ptrVal = I.getArgOperand(1);
1158 Info.memVT = getValueType(DL, I.getType()->getScalarType());
1159 Info.align = Align(DL.getTypeSizeInBits(I.getType()->getScalarType()) / 8);
1160 Info.size = MemoryLocation::UnknownSize;
1161 Info.flags |= MachineMemOperand::MOLoad;
1162 return true;
1163 case Intrinsic::riscv_masked_strided_store:
1164 Info.opc = ISD::INTRINSIC_VOID;
1165 Info.ptrVal = I.getArgOperand(1);
1166 Info.memVT =
1167 getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1168 Info.align = Align(
1169 DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1170 8);
1171 Info.size = MemoryLocation::UnknownSize;
1172 Info.flags |= MachineMemOperand::MOStore;
1173 return true;
1174 case Intrinsic::riscv_seg2_load:
1175 case Intrinsic::riscv_seg3_load:
1176 case Intrinsic::riscv_seg4_load:
1177 case Intrinsic::riscv_seg5_load:
1178 case Intrinsic::riscv_seg6_load:
1179 case Intrinsic::riscv_seg7_load:
1180 case Intrinsic::riscv_seg8_load:
1181 Info.opc = ISD::INTRINSIC_W_CHAIN;
1182 Info.ptrVal = I.getArgOperand(0);
1183 Info.memVT =
1184 getValueType(DL, I.getType()->getStructElementType(0)->getScalarType());
1185 Info.align =
1186 Align(DL.getTypeSizeInBits(
1187 I.getType()->getStructElementType(0)->getScalarType()) /
1188 8);
1189 Info.size = MemoryLocation::UnknownSize;
1190 Info.flags |= MachineMemOperand::MOLoad;
1191 return true;
1192 case Intrinsic::riscv_seg2_store:
1193 case Intrinsic::riscv_seg3_store:
1194 case Intrinsic::riscv_seg4_store:
1195 case Intrinsic::riscv_seg5_store:
1196 case Intrinsic::riscv_seg6_store:
1197 case Intrinsic::riscv_seg7_store:
1198 case Intrinsic::riscv_seg8_store:
1199 Info.opc = ISD::INTRINSIC_VOID;
1200 // Operands are (vec, ..., vec, ptr, vl, int_id)
1201 Info.ptrVal = I.getArgOperand(I.getNumOperands() - 3);
1202 Info.memVT =
1203 getValueType(DL, I.getArgOperand(0)->getType()->getScalarType());
1204 Info.align = Align(
1205 DL.getTypeSizeInBits(I.getArgOperand(0)->getType()->getScalarType()) /
1206 8);
1207 Info.size = MemoryLocation::UnknownSize;
1208 Info.flags |= MachineMemOperand::MOStore;
1209 return true;
1210 }
1211}
1212
1213bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
1214 const AddrMode &AM, Type *Ty,
1215 unsigned AS,
1216 Instruction *I) const {
1217 // No global is ever allowed as a base.
1218 if (AM.BaseGV)
1219 return false;
1220
1221 // RVV instructions only support register addressing.
1222 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
1223 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
1224
1225 // Require a 12-bit signed offset.
1226 if (!isInt<12>(AM.BaseOffs))
1227 return false;
1228
1229 switch (AM.Scale) {
1230 case 0: // "r+i" or just "i", depending on HasBaseReg.
1231 break;
1232 case 1:
1233 if (!AM.HasBaseReg) // allow "r+i".
1234 break;
1235 return false; // disallow "r+r" or "r+r+i".
1236 default:
1237 return false;
1238 }
1239
1240 return true;
1241}
1242
1243bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
1244 return isInt<12>(Imm);
1245}
1246
1247bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
1248 return isInt<12>(Imm);
1249}
1250
1251// On RV32, 64-bit integers are split into their high and low parts and held
1252// in two different registers, so the trunc is free since the low register can
1253// just be used.
1254// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
1255// isTruncateFree?
1256bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
1257 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
1258 return false;
1259 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
1260 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
1261 return (SrcBits == 64 && DestBits == 32);
1262}
1263
1264bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
1265 // We consider i64->i32 free on RV64 since we have good selection of W
1266 // instructions that make promoting operations back to i64 free in many cases.
1267 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
1268 !DstVT.isInteger())
1269 return false;
1270 unsigned SrcBits = SrcVT.getSizeInBits();
1271 unsigned DestBits = DstVT.getSizeInBits();
1272 return (SrcBits == 64 && DestBits == 32);
1273}
1274
1275bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
1276 // Zexts are free if they can be combined with a load.
1277 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
1278 // poorly with type legalization of compares preferring sext.
1279 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1280 EVT MemVT = LD->getMemoryVT();
1281 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
1282 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
1283 LD->getExtensionType() == ISD::ZEXTLOAD))
1284 return true;
1285 }
1286
1287 return TargetLowering::isZExtFree(Val, VT2);
1288}
1289
1290bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
1291 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
1292}
1293
1294bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
1295 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
1296}
1297
1298bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
1299 return Subtarget.hasStdExtZbb();
1300}
1301
1302bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
1303 return Subtarget.hasStdExtZbb() || Subtarget.hasVendorXTHeadBb();
1304}
1305
1306bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
1307 const Instruction &AndI) const {
1308 // We expect to be able to match a bit extraction instruction if the Zbs
1309 // extension is supported and the mask is a power of two. However, we
1310 // conservatively return false if the mask would fit in an ANDI instruction,
1311 // on the basis that it's possible the sinking+duplication of the AND in
1312 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
1313 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
1314 if (!Subtarget.hasStdExtZbs() && !Subtarget.hasVendorXTHeadBs())
1315 return false;
1316 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
1317 if (!Mask)
1318 return false;
1319 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
1320}
1321
1322bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
1323 EVT VT = Y.getValueType();
1324
1325 // FIXME: Support vectors once we have tests.
1326 if (VT.isVector())
1327 return false;
1328
1329 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
1330 !isa<ConstantSDNode>(Y);
1331}
1332
1333bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
1334 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
1335 if (Subtarget.hasStdExtZbs())
1336 return X.getValueType().isScalarInteger();
1337 auto *C = dyn_cast<ConstantSDNode>(Y);
1338 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
1339 if (Subtarget.hasVendorXTHeadBs())
1340 return C != nullptr;
1341 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
1342 return C && C->getAPIntValue().ule(10);
1343}
1344
1345bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(unsigned Opcode,
1346 EVT VT) const {
1347 // Only enable for rvv.
1348 if (!VT.isVector() || !Subtarget.hasVInstructions())
1349 return false;
1350
1351 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
1352 return false;
1353
1354 return true;
1355}
1356
1357bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
1358 Type *Ty) const {
1359 assert(Ty->isIntegerTy());
1360
1361 unsigned BitSize = Ty->getIntegerBitWidth();
1362 if (BitSize > Subtarget.getXLen())
1363 return false;
1364
1365 // Fast path, assume 32-bit immediates are cheap.
1366 int64_t Val = Imm.getSExtValue();
1367 if (isInt<32>(Val))
1368 return true;
1369
1370 // A constant pool entry may be more aligned thant he load we're trying to
1371 // replace. If we don't support unaligned scalar mem, prefer the constant
1372 // pool.
1373 // TODO: Can the caller pass down the alignment?
1374 if (!Subtarget.enableUnalignedScalarMem())
1375 return true;
1376
1377 // Prefer to keep the load if it would require many instructions.
1378 // This uses the same threshold we use for constant pools but doesn't
1379 // check useConstantPoolForLargeInts.
1380 // TODO: Should we keep the load only when we're definitely going to emit a
1381 // constant pool?
1382
1383 RISCVMatInt::InstSeq Seq =
1384 RISCVMatInt::generateInstSeq(Val, Subtarget.getFeatureBits());
1385 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
1386}
1387
1388bool RISCVTargetLowering::
1389 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
1390 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
1391 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
1392 SelectionDAG &DAG) const {
1393 // One interesting pattern that we'd want to form is 'bit extract':
1394 // ((1 >> Y) & 1) ==/!= 0
1395 // But we also need to be careful not to try to reverse that fold.
1396
1397 // Is this '((1 >> Y) & 1)'?
1398 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
1399 return false; // Keep the 'bit extract' pattern.
1400
1401 // Will this be '((1 >> Y) & 1)' after the transform?
1402 if (NewShiftOpcode == ISD::SRL && CC->isOne())
1403 return true; // Do form the 'bit extract' pattern.
1404
1405 // If 'X' is a constant, and we transform, then we will immediately
1406 // try to undo the fold, thus causing endless combine loop.
1407 // So only do the transform if X is not a constant. This matches the default
1408 // implementation of this function.
1409 return !XC;
1410}
1411
1412bool RISCVTargetLowering::canSplatOperand(unsigned Opcode, int Operand) const {
1413 switch (Opcode) {
1414 case Instruction::Add:
1415 case Instruction::Sub:
1416 case Instruction::Mul:
1417 case Instruction::And:
1418 case Instruction::Or:
1419 case Instruction::Xor:
1420 case Instruction::FAdd:
1421 case Instruction::FSub:
1422 case Instruction::FMul:
1423 case Instruction::FDiv:
1424 case Instruction::ICmp:
1425 case Instruction::FCmp:
1426 return true;
1427 case Instruction::Shl:
1428 case Instruction::LShr:
1429 case Instruction::AShr:
1430 case Instruction::UDiv:
1431 case Instruction::SDiv:
1432 case Instruction::URem:
1433 case Instruction::SRem:
1434 return Operand == 1;
1435 default:
1436 return false;
1437 }
1438}
1439
1440
1441bool RISCVTargetLowering::canSplatOperand(Instruction *I, int Operand) const {
1442 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1443 return false;
1444
1445 if (canSplatOperand(I->getOpcode(), Operand))
1446 return true;
1447
1448 auto *II = dyn_cast<IntrinsicInst>(I);
1449 if (!II)
1450 return false;
1451
1452 switch (II->getIntrinsicID()) {
1453 case Intrinsic::fma:
1454 case Intrinsic::vp_fma:
1455 return Operand == 0 || Operand == 1;
1456 case Intrinsic::vp_shl:
1457 case Intrinsic::vp_lshr:
1458 case Intrinsic::vp_ashr:
1459 case Intrinsic::vp_udiv:
1460 case Intrinsic::vp_sdiv:
1461 case Intrinsic::vp_urem:
1462 case Intrinsic::vp_srem:
1463 return Operand == 1;
1464 // These intrinsics are commutative.
1465 case Intrinsic::vp_add:
1466 case Intrinsic::vp_mul:
1467 case Intrinsic::vp_and:
1468 case Intrinsic::vp_or:
1469 case Intrinsic::vp_xor:
1470 case Intrinsic::vp_fadd:
1471 case Intrinsic::vp_fmul:
1472 // These intrinsics have 'vr' versions.
1473 case Intrinsic::vp_sub:
1474 case Intrinsic::vp_fsub:
1475 case Intrinsic::vp_fdiv:
1476 return Operand == 0 || Operand == 1;
1477 default:
1478 return false;
1479 }
1480}
1481
1482/// Check if sinking \p I's operands to I's basic block is profitable, because
1483/// the operands can be folded into a target instruction, e.g.
1484/// splats of scalars can fold into vector instructions.
1485bool RISCVTargetLowering::shouldSinkOperands(
1486 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
1487 using namespace llvm::PatternMatch;
1488
1489 if (!I->getType()->isVectorTy() || !Subtarget.hasVInstructions())
1490 return false;
1491
1492 for (auto OpIdx : enumerate(I->operands())) {
1493 if (!canSplatOperand(I, OpIdx.index()))
1494 continue;
1495
1496 Instruction *Op = dyn_cast<Instruction>(OpIdx.value().get());
1497 // Make sure we are not already sinking this operand
1498 if (!Op || any_of(Ops, [&](Use *U) { return U->get() == Op; }))
1499 continue;
1500
1501 // We are looking for a splat that can be sunk.
1502 if (!match(Op, m_Shuffle(m_InsertElt(m_Undef(), m_Value(), m_ZeroInt()),
1503 m_Undef(), m_ZeroMask())))
1504 continue;
1505
1506 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
1507 // and vector registers
1508 for (Use &U : Op->uses()) {
1509 Instruction *Insn = cast<Instruction>(U.getUser());
1510 if (!canSplatOperand(Insn, U.getOperandNo()))
1511 return false;
1512 }
1513
1514 Ops.push_back(&Op->getOperandUse(0));
1515 Ops.push_back(&OpIdx.value());
1516 }
1517 return true;
1518}
1519
1520bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
1521 unsigned Opc = VecOp.getOpcode();
1522
1523 // Assume target opcodes can't be scalarized.
1524 // TODO - do we have any exceptions?
1525 if (Opc >= ISD::BUILTIN_OP_END)
1526 return false;
1527
1528 // If the vector op is not supported, try to convert to scalar.
1529 EVT VecVT = VecOp.getValueType();
1530 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
1531 return true;
1532
1533 // If the vector op is supported, but the scalar op is not, the transform may
1534 // not be worthwhile.
1535 EVT ScalarVT = VecVT.getScalarType();
1536 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
1537}
1538
1539bool RISCVTargetLowering::isOffsetFoldingLegal(
1540 const GlobalAddressSDNode *GA) const {
1541 // In order to maximise the opportunity for common subexpression elimination,
1542 // keep a separate ADD node for the global address offset instead of folding
1543 // it in the global address node. Later peephole optimisations may choose to
1544 // fold it back in when profitable.
1545 return false;
1546}
1547
1548bool RISCVTargetLowering::isLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
1549 if (!Subtarget.hasStdExtZfa())
1550 return false;
1551
1552 bool IsSupportedVT = false;
1553 if (VT == MVT::f16) {
1554 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
1555 } else if (VT == MVT::f32) {
1556 IsSupportedVT = true;
1557 } else if (VT == MVT::f64) {
1558 assert(Subtarget.hasStdExtD() && "Expect D extension");
1559 IsSupportedVT = true;
1560 }
1561
1562 return IsSupportedVT && RISCVLoadFPImm::getLoadFPImm(Imm) != -1;
1563}
1564
1565bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1566 bool ForCodeSize) const {
1567 bool IsLegalVT = false;
1568 if (VT == MVT::f16)
1569 IsLegalVT = Subtarget.hasStdExtZfhOrZfhmin();
1570 else if (VT == MVT::f32)
1571 IsLegalVT = Subtarget.hasStdExtF();
1572 else if (VT == MVT::f64)
1573 IsLegalVT = Subtarget.hasStdExtD();
1574
1575 if (!IsLegalVT)
1576 return false;
1577
1578 if (isLegalZfaFPImm(Imm, VT))
1579 return true;
1580
1581 // Cannot create a 64 bit floating-point immediate value for rv32.
1582 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
1583 // td can handle +0.0 or -0.0 already.
1584 // -0.0 can be created by fmv + fneg.
1585 return Imm.isZero();
1586 }
1587 // Special case: the cost for -0.0 is 1.
1588 int Cost = Imm.isNegZero()
1589 ? 1
1590 : RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
1591 Subtarget.getXLen(),
1592 Subtarget.getFeatureBits());
1593 // If the constantpool data is already in cache, only Cost 1 is cheaper.
1594 return Cost < FPImmCost;
1595}
1596
1597// TODO: This is very conservative.
1598bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1599 unsigned Index) const {
1600 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
1601 return false;
1602
1603 // Only support extracting a fixed from a fixed vector for now.
1604 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
1605 return false;
1606
1607 unsigned ResElts = ResVT.getVectorNumElements();
1608 unsigned SrcElts = SrcVT.getVectorNumElements();
1609
1610 // Convervatively only handle extracting half of a vector.
1611 // TODO: Relax this.
1612 if ((ResElts * 2) != SrcElts)
1613 return false;
1614
1615 // The smallest type we can slide is i8.
1616 // TODO: We can extract index 0 from a mask vector without a slide.
1617 if (ResVT.getVectorElementType() == MVT::i1)
1618 return false;
1619
1620 // Slide can support arbitrary index, but we only treat vslidedown.vi as
1621 // cheap.
1622 if (Index >= 32)
1623 return false;
1624
1625 // TODO: We can do arbitrary slidedowns, but for now only support extracting
1626 // the upper half of a vector until we have more test coverage.
1627 return Index == 0 || Index == ResElts;
1628}
1629
1630MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1631 CallingConv::ID CC,
1632 EVT VT) const {
1633 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
1634 // We might still end up using a GPR but that will be decided based on ABI.
1635 if (VT == MVT::f16 && Subtarget.hasStdExtF() &&
1636 !Subtarget.hasStdExtZfhOrZfhmin())
1637 return MVT::f32;
1638
1639 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1640}
1641
1642unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1643 CallingConv::ID CC,
1644 EVT VT) const {
1645 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
1646 // We might still end up using a GPR but that will be decided based on ABI.
1647 if (VT == MVT::f16 && Subtarget.hasStdExtF() &&
1648 !Subtarget.hasStdExtZfhOrZfhmin())
1649 return 1;
1650
1651 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1652}
1653
1654// Changes the condition code and swaps operands if necessary, so the SetCC
1655// operation matches one of the comparisons supported directly by branches
1656// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1657// with 1/-1.
1658static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1659 ISD::CondCode &CC, SelectionDAG &DAG) {
1660 // If this is a single bit test that can't be handled by ANDI, shift the
1661 // bit to be tested to the MSB and perform a signed compare with 0.
1662 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
1663 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1664 isa<ConstantSDNode>(LHS.getOperand(1))) {
1665 uint64_t Mask = LHS.getConstantOperandVal(1);
1666 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
1667 unsigned ShAmt = 0;
1668 if (isPowerOf2_64(Mask)) {
1669 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
1670 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
1671 } else {
1672 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
1673 }
1674
1675 LHS = LHS.getOperand(0);
1676 if (ShAmt != 0)
1677 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
1678 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
1679 return;
1680 }
1681 }
1682
1683 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
1684 int64_t C = RHSC->getSExtValue();
1685 switch (CC) {
1686 default: break;
1687 case ISD::SETGT:
1688 // Convert X > -1 to X >= 0.
1689 if (C == -1) {
1690 RHS = DAG.getConstant(0, DL, RHS.getValueType());
1691 CC = ISD::SETGE;
1692 return;
1693 }
1694 break;
1695 case ISD::SETLT:
1696 // Convert X < 1 to 0 <= X.
1697 if (C == 1) {
1698 RHS = LHS;
1699 LHS = DAG.getConstant(0, DL, RHS.getValueType());
1700 CC = ISD::SETGE;
1701 return;
1702 }
1703 break;
1704 }
1705 }
1706
1707 switch (CC) {
1708 default:
1709 break;
1710 case ISD::SETGT:
1711 case ISD::SETLE:
1712 case ISD::SETUGT:
1713 case ISD::SETULE:
1714 CC = ISD::getSetCCSwappedOperands(CC);
1715 std::swap(LHS, RHS);
1716 break;
1717 }
1718}
1719
1720RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1721 assert(VT.isScalableVector() && "Expecting a scalable vector type");
1722 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1723 if (VT.getVectorElementType() == MVT::i1)
1724 KnownSize *= 8;
1725
1726 switch (KnownSize) {
1727 default:
1728 llvm_unreachable("Invalid LMUL.");
1729 case 8:
1730 return RISCVII::VLMUL::LMUL_F8;
1731 case 16:
1732 return RISCVII::VLMUL::LMUL_F4;
1733 case 32:
1734 return RISCVII::VLMUL::LMUL_F2;
1735 case 64:
1736 return RISCVII::VLMUL::LMUL_1;
1737 case 128:
1738 return RISCVII::VLMUL::LMUL_2;
1739 case 256:
1740 return RISCVII::VLMUL::LMUL_4;
1741 case 512:
1742 return RISCVII::VLMUL::LMUL_8;
1743 }
1744}
1745
1746unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1747 switch (LMul) {
1748 default:
1749 llvm_unreachable("Invalid LMUL.");
1750 case RISCVII::VLMUL::LMUL_F8:
1751 case RISCVII::VLMUL::LMUL_F4:
1752 case RISCVII::VLMUL::LMUL_F2:
1753 case RISCVII::VLMUL::LMUL_1:
1754 return RISCV::VRRegClassID;
1755 case RISCVII::VLMUL::LMUL_2:
1756 return RISCV::VRM2RegClassID;
1757 case RISCVII::VLMUL::LMUL_4:
1758 return RISCV::VRM4RegClassID;
1759 case RISCVII::VLMUL::LMUL_8:
1760 return RISCV::VRM8RegClassID;
1761 }
1762}
1763
1764unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1765 RISCVII::VLMUL LMUL = getLMUL(VT);
1766 if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1767 LMUL == RISCVII::VLMUL::LMUL_F4 ||
1768 LMUL == RISCVII::VLMUL::LMUL_F2 ||
1769 LMUL == RISCVII::VLMUL::LMUL_1) {
1770 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1771 "Unexpected subreg numbering");
1772 return RISCV::sub_vrm1_0 + Index;
1773 }
1774 if (LMUL == RISCVII::VLMUL::LMUL_2) {
1775 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1776 "Unexpected subreg numbering");
1777 return RISCV::sub_vrm2_0 + Index;
1778 }
1779 if (LMUL == RISCVII::VLMUL::LMUL_4) {
1780 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1781 "Unexpected subreg numbering");
1782 return RISCV::sub_vrm4_0 + Index;
1783 }
1784 llvm_unreachable("Invalid vector type.");
1785}
1786
1787unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1788 if (VT.getVectorElementType() == MVT::i1)
1789 return RISCV::VRRegClassID;
1790 return getRegClassIDForLMUL(getLMUL(VT));
1791}
1792
1793// Attempt to decompose a subvector insert/extract between VecVT and
1794// SubVecVT via subregister indices. Returns the subregister index that
1795// can perform the subvector insert/extract with the given element index, as
1796// well as the index corresponding to any leftover subvectors that must be
1797// further inserted/extracted within the register class for SubVecVT.
1798std::pair<unsigned, unsigned>
1799RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1800 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1801 const RISCVRegisterInfo *TRI) {
1802 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1803 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1804 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1805 "Register classes not ordered");
1806 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1807 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1808 // Try to compose a subregister index that takes us from the incoming
1809 // LMUL>1 register class down to the outgoing one. At each step we half
1810 // the LMUL:
1811 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1812 // Note that this is not guaranteed to find a subregister index, such as
1813 // when we are extracting from one VR type to another.
1814 unsigned SubRegIdx = RISCV::NoSubRegister;
1815 for (const unsigned RCID :
1816 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1817 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1818 VecVT = VecVT.getHalfNumVectorElementsVT();
1819 bool IsHi =
1820 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1821 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1822 getSubregIndexByMVT(VecVT, IsHi));
1823 if (IsHi)
1824 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1825 }
1826 return {SubRegIdx, InsertExtractIdx};
1827}
1828
1829// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1830// stores for those types.
1831bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1832 return !Subtarget.useRVVForFixedLengthVectors() ||
1833 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1834}
1835
1836bool RISCVTargetLowering::isLegalElementTypeForRVV(Type *ScalarTy) const {
1837 if (ScalarTy->isPointerTy())
1838 return true;
1839
1840 if (ScalarTy->isIntegerTy(8) || ScalarTy->isIntegerTy(16) ||
1841 ScalarTy->isIntegerTy(32))
1842 return true;
1843
1844 if (ScalarTy->isIntegerTy(64))
1845 return Subtarget.hasVInstructionsI64();
1846
1847 if (ScalarTy->isHalfTy())
1848 return Subtarget.hasVInstructionsF16();
1849 if (ScalarTy->isFloatTy())
1850 return Subtarget.hasVInstructionsF32();
1851 if (ScalarTy->isDoubleTy())
1852 return Subtarget.hasVInstructionsF64();
1853
1854 return false;
1855}
1856
1857unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
1858 return NumRepeatedDivisors;
1859}
1860
1861static SDValue getVLOperand(SDValue Op) {
1862 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
1863 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
1864 "Unexpected opcode");
1865 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
1866 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
1867 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
1868 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
1869 if (!II)
1870 return SDValue();
1871 return Op.getOperand(II->VLOperand + 1 + HasChain);
1872}
1873
1874static bool useRVVForFixedLengthVectorVT(MVT VT,
1875 const RISCVSubtarget &Subtarget) {
1876 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1877 if (!Subtarget.useRVVForFixedLengthVectors())
1878 return false;
1879
1880 // We only support a set of vector types with a consistent maximum fixed size
1881 // across all supported vector element types to avoid legalization issues.
1882 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1883 // fixed-length vector type we support is 1024 bytes.
1884 if (VT.getFixedSizeInBits() > 1024 * 8)
1885 return false;
1886
1887 unsigned MinVLen = Subtarget.getRealMinVLen();
1888
1889 MVT EltVT = VT.getVectorElementType();
1890
1891 // Don't use RVV for vectors we cannot scalarize if required.
1892 switch (EltVT.SimpleTy) {
1893 // i1 is supported but has different rules.
1894 default:
1895 return false;
1896 case MVT::i1:
1897 // Masks can only use a single register.
1898 if (VT.getVectorNumElements() > MinVLen)
1899 return false;
1900 MinVLen /= 8;
1901 break;
1902 case MVT::i8:
1903 case MVT::i16:
1904 case MVT::i32:
1905 break;
1906 case MVT::i64:
1907 if (!Subtarget.hasVInstructionsI64())
1908 return false;
1909 break;
1910 case MVT::f16:
1911 if (!Subtarget.hasVInstructionsF16())
1912 return false;
1913 break;
1914 case MVT::f32:
1915 if (!Subtarget.hasVInstructionsF32())
1916 return false;
1917 break;
1918 case MVT::f64:
1919 if (!Subtarget.hasVInstructionsF64())
1920 return false;
1921 break;
1922 }
1923
1924 // Reject elements larger than ELEN.
1925 if (EltVT.getSizeInBits() > Subtarget.getELEN())
1926 return false;
1927
1928 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1929 // Don't use RVV for types that don't fit.
1930 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1931 return false;
1932
1933 // TODO: Perhaps an artificial restriction, but worth having whilst getting
1934 // the base fixed length RVV support in place.
1935 if (!VT.isPow2VectorType())
1936 return false;
1937
1938 return true;
1939}
1940
1941bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1942 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1943}
1944
1945// Return the largest legal scalable vector type that matches VT's element type.
1946static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1947 const RISCVSubtarget &Subtarget) {
1948 // This may be called before legal types are setup.
1949 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1950 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1951 "Expected legal fixed length vector!");
1952
1953 unsigned MinVLen = Subtarget.getRealMinVLen();
1954 unsigned MaxELen = Subtarget.getELEN();
1955
1956 MVT EltVT = VT.getVectorElementType();
1957 switch (EltVT.SimpleTy) {
1958 default:
1959 llvm_unreachable("unexpected element type for RVV container");
1960 case MVT::i1:
1961 case MVT::i8:
1962 case MVT::i16:
1963 case MVT::i32:
1964 case MVT::i64:
1965 case MVT::f16:
1966 case MVT::f32:
1967 case MVT::f64: {
1968 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1969 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
1970 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
1971 unsigned NumElts =
1972 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
1973 NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
1974 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
1975 return MVT::getScalableVectorVT(EltVT, NumElts);
1976 }
1977 }
1978}
1979
1980static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1981 const RISCVSubtarget &Subtarget) {
1982 return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1983 Subtarget);
1984}
1985
1986MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1987 return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1988}
1989
1990// Grow V to consume an entire RVV register.
1991static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1992 const RISCVSubtarget &Subtarget) {
1993 assert(VT.isScalableVector() &&
1994 "Expected to convert into a scalable vector!");
1995 assert(V.getValueType().isFixedLengthVector() &&
1996 "Expected a fixed length vector operand!");
1997 SDLoc DL(V);
1998 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1999 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
2000}
2001
2002// Shrink V so it's just big enough to maintain a VT's worth of data.
2003static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
2004 const RISCVSubtarget &Subtarget) {
2005 assert(VT.isFixedLengthVector() &&
2006 "Expected to convert into a fixed length vector!");
2007 assert(V.getValueType().isScalableVector() &&
2008 "Expected a scalable vector operand!");
2009 SDLoc DL(V);
2010 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
2011 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
2012}
2013
2014/// Return the type of the mask type suitable for masking the provided
2015/// vector type. This is simply an i1 element type vector of the same
2016/// (possibly scalable) length.
2017static MVT getMaskTypeFor(MVT VecVT) {
2018 assert(VecVT.isVector());
2019 ElementCount EC = VecVT.getVectorElementCount();
2020 return MVT::getVectorVT(MVT::i1, EC);
2021}
2022
2023/// Creates an all ones mask suitable for masking a vector of type VecTy with
2024/// vector length VL. .
2025static SDValue getAllOnesMask(MVT VecVT, SDValue VL, SDLoc DL,
2026 SelectionDAG &DAG) {
2027 MVT MaskVT = getMaskTypeFor(VecVT);
2028 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
2029}
2030
2031static SDValue getVLOp(uint64_t NumElts, SDLoc DL, SelectionDAG &DAG,
2032 const RISCVSubtarget &Subtarget) {
2033 return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
2034}
2035
2036static std::pair<SDValue, SDValue>
2037getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
2038 const RISCVSubtarget &Subtarget) {
2039 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2040 SDValue VL = getVLOp(NumElts, DL, DAG, Subtarget);
2041 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2042 return {Mask, VL};
2043}
2044
2045// Gets the two common "VL" operands: an all-ones mask and the vector length.
2046// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
2047// the vector type that the fixed-length vector is contained in. Otherwise if
2048// VecVT is scalable, then ContainerVT should be the same as VecVT.
2049static std::pair<SDValue, SDValue>
2050getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
2051 const RISCVSubtarget &Subtarget) {
2052 if (VecVT.isFixedLengthVector())
2053 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
2054 Subtarget);
2055 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
2056 MVT XLenVT = Subtarget.getXLenVT();
2057 SDValue VL = DAG.getRegister(RISCV::X0, XLenVT);
2058 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
2059 return {Mask, VL};
2060}
2061
2062// As above but assuming the given type is a scalable vector type.
2063static std::pair<SDValue, SDValue>
2064getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
2065 const RISCVSubtarget &Subtarget) {
2066 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
2067 return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
2068}
2069
2070SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, SDLoc DL,
2071 SelectionDAG &DAG) const {
2072 assert(VecVT.isScalableVector() && "Expected scalable vector");
2073 unsigned MinElts = VecVT.getVectorMinNumElements();
2074 return DAG.getNode(ISD::VSCALE, DL, Subtarget.getXLenVT(),
2075 getVLOp(MinElts, DL, DAG, Subtarget));
2076}
2077
2078// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
2079// of either is (currently) supported. This can get us into an infinite loop
2080// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
2081// as a ..., etc.
2082// Until either (or both) of these can reliably lower any node, reporting that
2083// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
2084// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
2085// which is not desirable.
2086bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
2087 EVT VT, unsigned DefinedValues) const {
2088 return false;
2089}
2090
2091static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
2092 const RISCVSubtarget &Subtarget) {
2093 // RISCV FP-to-int conversions saturate to the destination register size, but
2094 // don't produce 0 for nan. We can use a conversion instruction and fix the
2095 // nan case with a compare and a select.
2096 SDValue Src = Op.getOperand(0);
2097
2098 MVT DstVT = Op.getSimpleValueType();
2099 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2100
2101 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
2102
2103 if (!DstVT.isVector()) {
2104 // In absense of Zfh, promote f16 to f32, then saturate the result.
2105 if (Src.getSimpleValueType() == MVT::f16 && !Subtarget.hasStdExtZfh()) {
2106 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
2107 }
2108
2109 unsigned Opc;
2110 if (SatVT == DstVT)
2111 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
2112 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
2113 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
2114 else
2115 return SDValue();
2116 // FIXME: Support other SatVTs by clamping before or after the conversion.
2117
2118 SDLoc DL(Op);
2119 SDValue FpToInt = DAG.getNode(
2120 Opc, DL, DstVT, Src,
2121 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
2122
2123 if (Opc == RISCVISD::FCVT_WU_RV64)
2124 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
2125
2126 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
2127 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
2128 ISD::CondCode::SETUO);
2129 }
2130
2131 // Vectors.
2132
2133 MVT DstEltVT = DstVT.getVectorElementType();
2134 MVT SrcVT = Src.getSimpleValueType();
2135 MVT SrcEltVT = SrcVT.getVectorElementType();
2136 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
2137 unsigned DstEltSize = DstEltVT.getSizeInBits();
2138
2139 // Only handle saturating to the destination type.
2140 if (SatVT != DstEltVT)
2141 return SDValue();
2142
2143 // FIXME: Don't support narrowing by more than 1 steps for now.
2144 if (SrcEltSize > (2 * DstEltSize))
2145 return SDValue();
2146
2147 MVT DstContainerVT = DstVT;
2148 MVT SrcContainerVT = SrcVT;
2149 if (DstVT.isFixedLengthVector()) {
2150 DstContainerVT = getContainerForFixedLengthVector(DAG, DstVT, Subtarget);
2151 SrcContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
2152 assert(DstContainerVT.getVectorElementCount() ==
2153 SrcContainerVT.getVectorElementCount() &&
2154 "Expected same element count");
2155 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2156 }
2157
2158 SDLoc DL(Op);
2159
2160 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
2161
2162 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
2163 {Src, Src, DAG.getCondCode(ISD::SETNE),
2164 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
2165
2166 // Need to widen by more than 1 step, promote the FP type, then do a widening
2167 // convert.
2168 if (DstEltSize > (2 * SrcEltSize)) {
2169 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
2170 MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
2171 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
2172 }
2173
2174 unsigned RVVOpc =
2175 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
2176 SDValue Res = DAG.getNode(RVVOpc, DL, DstContainerVT, Src, Mask, VL);
2177
2178 SDValue SplatZero = DAG.getNode(
2179 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
2180 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
2181 Res = DAG.getNode(RISCVISD::VSELECT_VL, DL, DstContainerVT, IsNan, SplatZero,
2182 Res, VL);
2183
2184 if (DstVT.isFixedLengthVector())
2185 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
2186
2187 return Res;
2188}
2189
2190static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
2191 switch (Opc) {
2192 case ISD::FROUNDEVEN:
2193 case ISD::VP_FROUNDEVEN:
2194 return RISCVFPRndMode::RNE;
2195 case ISD::FTRUNC:
2196 case ISD::VP_FROUNDTOZERO:
2197 return RISCVFPRndMode::RTZ;
2198 case ISD::FFLOOR:
2199 case ISD::VP_FFLOOR:
2200 return RISCVFPRndMode::RDN;
2201 case ISD::FCEIL:
2202 case ISD::VP_FCEIL:
2203 return RISCVFPRndMode::RUP;
2204 case ISD::FROUND:
2205 case ISD::VP_FROUND:
2206 return RISCVFPRndMode::RMM;
2207 case ISD::FRINT:
2208 return RISCVFPRndMode::DYN;
2209 }
2210
2211 return RISCVFPRndMode::Invalid;
2212}
2213
2214// Expand vector FTRUNC, FCEIL, FFLOOR, FROUND, VP_FCEIL, VP_FFLOOR, VP_FROUND
2215// VP_FROUNDEVEN, VP_FROUNDTOZERO, VP_FRINT and VP_FNEARBYINT by converting to
2216// the integer domain and back. Taking care to avoid converting values that are
2217// nan or already correct.
2218static SDValue
2219lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
2220 const RISCVSubtarget &Subtarget) {
2221 MVT VT = Op.getSimpleValueType();
2222 assert(VT.isVector() && "Unexpected type");
2223
2224 SDLoc DL(Op);
2225
2226 SDValue Src = Op.getOperand(0);
2227
2228 MVT ContainerVT = VT;
2229 if (VT.isFixedLengthVector()) {
2230 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2231 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
2232 }
2233
2234 SDValue Mask, VL;
2235 if (Op->isVPOpcode()) {
2236 Mask = Op.getOperand(1);
2237 VL = Op.getOperand(2);
2238 } else {
2239 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2240 }
2241
2242 // Freeze the source since we are increasing the number of uses.
2243 Src = DAG.getFreeze(Src);
2244
2245 // We do the conversion on the absolute value and fix the sign at the end.
2246 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
2247
2248 // Determine the largest integer that can be represented exactly. This and
2249 // values larger than it don't have any fractional bits so don't need to
2250 // be converted.
2251 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(ContainerVT);
2252 unsigned Precision = APFloat::semanticsPrecision(FltSem);
2253 APFloat MaxVal = APFloat(FltSem);
2254 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
2255 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
2256 SDValue MaxValNode =
2257 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
2258 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
2259 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
2260
2261 // If abs(Src) was larger than MaxVal or nan, keep it.
2262 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
2263 Mask =
2264 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
2265 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
2266 Mask, Mask, VL});
2267
2268 // Truncate to integer and convert back to FP.
2269 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
2270 MVT XLenVT = Subtarget.getXLenVT();
2271 SDValue Truncated;
2272
2273 switch (Op.getOpcode()) {
2274 default:
2275 llvm_unreachable("Unexpected opcode");
2276 case ISD::FCEIL:
2277 case ISD::VP_FCEIL:
2278 case ISD::FFLOOR:
2279 case ISD::VP_FFLOOR:
2280 case ISD::FROUND:
2281 case ISD::FROUNDEVEN:
2282 case ISD::VP_FROUND:
2283 case ISD::VP_FROUNDEVEN:
2284 case ISD::VP_FROUNDTOZERO: {
2285 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
2286 assert(FRM != RISCVFPRndMode::Invalid);
2287 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
2288 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
2289 break;
2290 }
2291 case ISD::FTRUNC:
2292 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
2293 Mask, VL);
2294 break;
2295 case ISD::VP_FRINT:
2296 Truncated = DAG.getNode(RISCVISD::VFCVT_X_F_VL, DL, IntVT, Src, Mask, VL);
2297 break;
2298 case ISD::VP_FNEARBYINT:
2299 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
2300 Mask, VL);
2301 break;
2302 }
2303
2304 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
2305 if (Op.getOpcode() != ISD::VP_FNEARBYINT)
2306 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
2307 Mask, VL);
2308
2309 // Restore the original sign so that -0.0 is preserved.
2310 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
2311 Src, Src, Mask, VL);
2312
2313 if (!VT.isFixedLengthVector())
2314 return Truncated;
2315
2316 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
2317}
2318
2319static SDValue
2320lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
2321 const RISCVSubtarget &Subtarget) {
2322 MVT VT = Op.getSimpleValueType();
2323 if (VT.isVector())
2324 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
2325
2326 if (DAG.shouldOptForSize())
2327 return SDValue();
2328
2329 SDLoc DL(Op);
2330 SDValue Src = Op.getOperand(0);
2331
2332 // Create an integer the size of the mantissa with the MSB set. This and all
2333 // values larger than it don't have any fractional bits so don't need to be
2334 // converted.
2335 const fltSemantics &FltSem = DAG.EVTToAPFloatSemantics(VT);
2336 unsigned Precision = APFloat::semanticsPrecision(FltSem);
2337 APFloat MaxVal = APFloat(FltSem);
2338 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
2339 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
2340 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
2341
2342 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
2343 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
2344 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
2345}
2346
2347struct VIDSequence {
2348 int64_t StepNumerator;
2349 unsigned StepDenominator;
2350 int64_t Addend;
2351};
2352
2353static std::optional<uint64_t> getExactInteger(const APFloat &APF,
2354 uint32_t BitWidth) {
2355 APSInt ValInt(BitWidth, !APF.isNegative());
2356 // We use an arbitrary rounding mode here. If a floating-point is an exact
2357 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
2358 // the rounding mode changes the output value, then it is not an exact
2359 // integer.
2360 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
2361 bool IsExact;
2362 // If it is out of signed integer range, it will return an invalid operation.
2363 // If it is not an exact integer, IsExact is false.
2364 if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
2365 APFloatBase::opInvalidOp) ||
2366 !IsExact)
2367 return std::nullopt;
2368 return ValInt.extractBitsAsZExtValue(BitWidth, 0);
2369}
2370
2371// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
2372// to the (non-zero) step S and start value X. This can be then lowered as the
2373// RVV sequence (VID * S) + X, for example.
2374// The step S is represented as an integer numerator divided by a positive
2375// denominator. Note that the implementation currently only identifies
2376// sequences in which either the numerator is +/- 1 or the denominator is 1. It
2377// cannot detect 2/3, for example.
2378// Note that this method will also match potentially unappealing index
2379// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
2380// determine whether this is worth generating code for.
2381static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
2382 unsigned NumElts = Op.getNumOperands();
2383 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
2384 bool IsInteger = Op.getValueType().isInteger();
2385
2386 std::optional<unsigned> SeqStepDenom;
2387 std::optional<int64_t> SeqStepNum, SeqAddend;
2388 std::optional<std::pair<uint64_t, unsigned>> PrevElt;
2389 unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
2390 for (unsigned Idx = 0; Idx < NumElts; Idx++) {
2391 // Assume undef elements match the sequence; we just have to be careful
2392 // when interpolating across them.
2393 if (Op.getOperand(Idx).isUndef())
2394 continue;
2395
2396 uint64_t Val;
2397 if (IsInteger) {
2398 // The BUILD_VECTOR must be all constants.
2399 if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
2400 return std::nullopt;
2401 Val = Op.getConstantOperandVal(Idx) &
2402 maskTrailingOnes<uint64_t>(EltSizeInBits);
2403 } else {
2404 // The BUILD_VECTOR must be all constants.
2405 if (!isa<ConstantFPSDNode>(Op.getOperand(Idx)))
2406 return std::nullopt;
2407 if (auto ExactInteger = getExactInteger(
2408 cast<ConstantFPSDNode>(Op.getOperand(Idx))->getValueAPF(),
2409 EltSizeInBits))
2410 Val = *ExactInteger;
2411 else
2412 return std::nullopt;
2413 }
2414
2415 if (PrevElt) {
2416 // Calculate the step since the last non-undef element, and ensure
2417 // it's consistent across the entire sequence.
2418 unsigned IdxDiff = Idx - PrevElt->second;
2419 int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
2420
2421 // A zero-value value difference means that we're somewhere in the middle
2422 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
2423 // step change before evaluating the sequence.
2424 if (ValDiff == 0)
2425 continue;
2426
2427 int64_t Remainder = ValDiff % IdxDiff;
2428 // Normalize the step if it's greater than 1.
2429 if (Remainder != ValDiff) {
2430 // The difference must cleanly divide the element span.
2431 if (Remainder != 0)
2432 return std::nullopt;
2433 ValDiff /= IdxDiff;
2434 IdxDiff = 1;
2435 }
2436
2437 if (!SeqStepNum)
2438 SeqStepNum = ValDiff;
2439 else if (ValDiff != SeqStepNum)
2440 return std::nullopt;
2441
2442 if (!SeqStepDenom)
2443 SeqStepDenom = IdxDiff;
2444 else if (IdxDiff != *SeqStepDenom)
2445 return std::nullopt;
2446 }
2447
2448 // Record this non-undef element for later.
2449 if (!PrevElt || PrevElt->first != Val)
2450 PrevElt = std::make_pair(Val, Idx);
2451 }
2452
2453 // We need to have logged a step for this to count as a legal index sequence.
2454 if (!SeqStepNum || !SeqStepDenom)
2455 return std::nullopt;
2456
2457 // Loop back through the sequence and validate elements we might have skipped
2458 // while waiting for a valid step. While doing this, log any sequence addend.
2459 for (unsigned Idx = 0; Idx < NumElts; Idx++) {
2460 if (Op.getOperand(Idx).isUndef())
2461 continue;
2462 uint64_t Val;
2463 if (IsInteger) {
2464 Val = Op.getConstantOperandVal(Idx) &
2465 maskTrailingOnes<uint64_t>(EltSizeInBits);
2466 } else {
2467 Val = *getExactInteger(
2468 cast<ConstantFPSDNode>(Op.getOperand(Idx))->getValueAPF(),
2469 EltSizeInBits);
2470 }
2471 uint64_t ExpectedVal =
2472 (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
2473 int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
2474 if (!SeqAddend)
2475 SeqAddend = Addend;
2476 else if (Addend != SeqAddend)
2477 return std::nullopt;
2478 }
2479
2480 assert(SeqAddend && "Must have an addend if we have a step");
2481
2482 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
2483}
2484
2485// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
2486// and lower it as a VRGATHER_VX_VL from the source vector.
2487static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
2488 SelectionDAG &DAG,
2489 const RISCVSubtarget &Subtarget) {
2490 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2491 return SDValue();
2492 SDValue Vec = SplatVal.getOperand(0);
2493 // Only perform this optimization on vectors of the same size for simplicity.
2494 // Don't perform this optimization for i1 vectors.
2495 // FIXME: Support i1 vectors, maybe by promoting to i8?
2496 if (Vec.getValueType() != VT || VT.getVectorElementType() == MVT::i1)
2497 return SDValue();
2498 SDValue Idx = SplatVal.getOperand(1);
2499 // The index must be a legal type.
2500 if (Idx.getValueType() != Subtarget.getXLenVT())
2501 return SDValue();
2502
2503 MVT ContainerVT = VT;
2504 if (VT.isFixedLengthVector()) {
2505 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2506 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
2507 }
2508
2509 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2510
2511 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Vec,
2512 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
2513
2514 if (!VT.isFixedLengthVector())
2515 return Gather;
2516
2517 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2518}
2519
2520static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
2521 const RISCVSubtarget &Subtarget) {
2522 MVT VT = Op.getSimpleValueType();
2523 assert(VT.isFixedLengthVector() && "Unexpected vector!");
2524
2525 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
2526
2527 SDLoc DL(Op);
2528 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2529
2530 MVT XLenVT = Subtarget.getXLenVT();
2531 unsigned NumElts = Op.getNumOperands();
2532
2533 if (VT.getVectorElementType() == MVT::i1) {
2534 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
2535 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
2536 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
2537 }
2538
2539 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
2540 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
2541 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
2542 }
2543
2544 // Lower constant mask BUILD_VECTORs via an integer vector type, in
2545 // scalar integer chunks whose bit-width depends on the number of mask
2546 // bits and XLEN.
2547 // First, determine the most appropriate scalar integer type to use. This
2548 // is at most XLenVT, but may be shrunk to a smaller vector element type
2549 // according to the size of the final vector - use i8 chunks rather than
2550 // XLenVT if we're producing a v8i1. This results in more consistent
2551 // codegen across RV32 and RV64.
2552 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
2553 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELEN());
2554 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
2555 // If we have to use more than one INSERT_VECTOR_ELT then this
2556 // optimization is likely to increase code size; avoid peforming it in
2557 // such a case. We can use a load from a constant pool in this case.
2558 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
2559 return SDValue();
2560 // Now we can create our integer vector type. Note that it may be larger
2561 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
2562 MVT IntegerViaVecVT =
2563 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
2564 divideCeil(NumElts, NumViaIntegerBits));
2565
2566 uint64_t Bits = 0;
2567 unsigned BitPos = 0, IntegerEltIdx = 0;
2568 SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
2569
2570 for (unsigned I = 0; I < NumElts; I++, BitPos++) {
2571 // Once we accumulate enough bits to fill our scalar type, insert into
2572 // our vector and clear our accumulated data.
2573 if (I != 0 && I % NumViaIntegerBits == 0) {
2574 if (NumViaIntegerBits <= 32)
2575 Bits = SignExtend64<32>(Bits);
2576 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2577 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
2578 Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2579 Bits = 0;
2580 BitPos = 0;
2581 IntegerEltIdx++;
2582 }
2583 SDValue V = Op.getOperand(I);
2584 bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
2585 Bits |= ((uint64_t)BitValue << BitPos);
2586 }
2587
2588 // Insert the (remaining) scalar value into position in our integer
2589 // vector type.
2590 if (NumViaIntegerBits <= 32)
2591 Bits = SignExtend64<32>(Bits);
2592 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
2593 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
2594 DAG.getConstant(IntegerEltIdx, DL, XLenVT));
2595
2596 if (NumElts < NumViaIntegerBits) {
2597 // If we're producing a smaller vector than our minimum legal integer
2598 // type, bitcast to the equivalent (known-legal) mask type, and extract
2599 // our final mask.
2600 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
2601 Vec = DAG.getBitcast(MVT::v8i1, Vec);
2602 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
2603 DAG.getConstant(0, DL, XLenVT));
2604 } else {
2605 // Else we must have produced an integer type with the same size as the
2606 // mask type; bitcast for the final result.
2607 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
2608 Vec = DAG.getBitcast(VT, Vec);
2609 }
2610
2611 return Vec;
2612 }
2613
2614 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
2615 // vector type, we have a legal equivalently-sized i8 type, so we can use
2616 // that.
2617 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
2618 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
2619
2620 SDValue WideVec;
2621 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2622 // For a splat, perform a scalar truncate before creating the wider
2623 // vector.
2624 assert(Splat.getValueType() == XLenVT &&
2625 "Unexpected type for i1 splat value");
2626 Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
2627 DAG.getConstant(1, DL, XLenVT));
2628 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
2629 } else {
2630 SmallVector<SDValue, 8> Ops(Op->op_values());
2631 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
2632 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
2633 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
2634 }
2635
2636 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
2637 }
2638
2639 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
2640 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
2641 return Gather;
2642 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
2643 : RISCVISD::VMV_V_X_VL;
2644 Splat =
2645 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
2646 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
2647 }
2648
2649 // Try and match index sequences, which we can lower to the vid instruction
2650 // with optional modifications. An all-undef vector is matched by
2651 // getSplatValue, above.
2652 if (auto SimpleVID = isSimpleVIDSequence(Op)) {
2653 int64_t StepNumerator = SimpleVID->StepNumerator;
2654 unsigned StepDenominator = SimpleVID->StepDenominator;
2655 int64_t Addend = SimpleVID->Addend;
2656
2657 assert(StepNumerator != 0 && "Invalid step");
2658 bool Negate = false;
2659 int64_t SplatStepVal = StepNumerator;
2660 unsigned StepOpcode = ISD::MUL;
2661 if (StepNumerator != 1) {
2662 if (isPowerOf2_64(std::abs(StepNumerator))) {
2663 Negate = StepNumerator < 0;
2664 StepOpcode = ISD::SHL;
2665 SplatStepVal = Log2_64(std::abs(StepNumerator));
2666 }
2667 }
2668
2669 // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
2670 // threshold since it's the immediate value many RVV instructions accept.
2671 // There is no vmul.vi instruction so ensure multiply constant can fit in
2672 // a single addi instruction.
2673 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
2674 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
2675 isPowerOf2_32(StepDenominator) &&
2676 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<5>(Addend)) {
2677 MVT VIDVT =
2678 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
2679 MVT VIDContainerVT =
2680 getContainerForFixedLengthVector(DAG, VIDVT, Subtarget);
2681 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
2682 // Convert right out of the scalable type so we can use standard ISD
2683 // nodes for the rest of the computation. If we used scalable types with
2684 // these, we'd lose the fixed-length vector info and generate worse
2685 // vsetvli code.
2686 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
2687 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
2688 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
2689 SDValue SplatStep = DAG.getSplatBuildVector(
2690 VIDVT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
2691 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
2692 }
2693 if (StepDenominator != 1) {
2694 SDValue SplatStep = DAG.getSplatBuildVector(
2695 VIDVT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
2696 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
2697 }
2698 if (Addend != 0 || Negate) {
2699 SDValue SplatAddend = DAG.getSplatBuildVector(
2700 VIDVT, DL, DAG.getConstant(Addend, DL, XLenVT));
2701 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
2702 VID);
2703 }
2704 if (VT.isFloatingPoint()) {
2705 // TODO: Use vfwcvt to reduce register pressure.
2706 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
2707 }
2708 return VID;
2709 }
2710 }
2711
2712 // Attempt to detect "hidden" splats, which only reveal themselves as splats
2713 // when re-interpreted as a vector with a larger element type. For example,
2714 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
2715 // could be instead splat as
2716 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
2717 // TODO: This optimization could also work on non-constant splats, but it
2718 // would require bit-manipulation instructions to construct the splat value.
2719 SmallVector<SDValue> Sequence;
2720 unsigned EltBitSize = VT.getScalarSizeInBits();
2721 const auto *BV = cast<BuildVectorSDNode>(Op);
2722 if (VT.isInteger() && EltBitSize < 64 &&
2723 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
2724 BV->getRepeatedSequence(Sequence) &&
2725 (Sequence.size() * EltBitSize) <= 64) {
2726 unsigned SeqLen = Sequence.size();
2727 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
2728 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
2729 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
2730 ViaIntVT == MVT::i64) &&
2731 "Unexpected sequence type");
2732
2733 unsigned EltIdx = 0;
2734 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
2735 uint64_t SplatValue = 0;
2736 // Construct the amalgamated value which can be splatted as this larger
2737 // vector type.
2738 for (const auto &SeqV : Sequence) {
2739 if (!SeqV.isUndef())
2740 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
2741 << (EltIdx * EltBitSize));
2742 EltIdx++;
2743 }
2744
2745 // On RV64, sign-extend from 32 to 64 bits where possible in order to
2746 // achieve better constant materializion.
2747 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
2748 SplatValue = SignExtend64<32>(SplatValue);
2749
2750 // Since we can't introduce illegal i64 types at this stage, we can only
2751 // perform an i64 splat on RV32 if it is its own sign-extended value. That
2752 // way we can use RVV instructions to splat.
2753 assert((ViaIntVT.bitsLE(XLenVT) ||
2754 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
2755 "Unexpected bitcast sequence");
2756 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
2757 SDValue ViaVL =
2758 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
2759 MVT ViaContainerVT =
2760 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
2761 SDValue Splat =
2762 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
2763 DAG.getUNDEF(ViaContainerVT),
2764 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
2765 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
2766 return DAG.getBitcast(VT, Splat);
2767 }
2768 }
2769
2770 // Try and optimize BUILD_VECTORs with "dominant values" - these are values
2771 // which constitute a large proportion of the elements. In such cases we can
2772 // splat a vector with the dominant element and make up the shortfall with
2773 // INSERT_VECTOR_ELTs.
2774 // Note that this includes vectors of 2 elements by association. The
2775 // upper-most element is the "dominant" one, allowing us to use a splat to
2776 // "insert" the upper element, and an insert of the lower element at position
2777 // 0, which improves codegen.
2778 SDValue DominantValue;
2779 unsigned MostCommonCount = 0;
2780 DenseMap<SDValue, unsigned> ValueCounts;
2781 unsigned NumUndefElts =
2782 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
2783
2784 // Track the number of scalar loads we know we'd be inserting, estimated as
2785 // any non-zero floating-point constant. Other kinds of element are either
2786 // already in registers or are materialized on demand. The threshold at which
2787 // a vector load is more desirable than several scalar materializion and
2788 // vector-insertion instructions is not known.
2789 unsigned NumScalarLoads = 0;
2790
2791 for (SDValue V : Op->op_values()) {
2792 if (V.isUndef())
2793 continue;
2794
2795 ValueCounts.insert(std::make_pair(V, 0));
2796 unsigned &Count = ValueCounts[V];
2797
2798 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
2799 NumScalarLoads += !CFP->isExactlyValue(+0.0);
2800
2801 // Is this value dominant? In case of a tie, prefer the highest element as
2802 // it's cheaper to insert near the beginning of a vector than it is at the
2803 // end.
2804 if (++Count >= MostCommonCount) {
2805 DominantValue = V;
2806 MostCommonCount = Count;
2807 }
2808 }
2809
2810 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
2811 unsigned NumDefElts = NumElts - NumUndefElts;
2812 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
2813
2814 // Don't perform this optimization when optimizing for size, since
2815 // materializing elements and inserting them tends to cause code bloat.
2816 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
2817 ((MostCommonCount > DominantValueCountThreshold) ||
2818 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
2819 // Start by splatting the most common element.
2820 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
2821
2822 DenseSet<SDValue> Processed{DominantValue};
2823 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
2824 for (const auto &OpIdx : enumerate(Op->ops())) {
2825 const SDValue &V = OpIdx.value();
2826 if (V.isUndef() || !Processed.insert(V).second)
2827 continue;
2828 if (ValueCounts[V] == 1) {
2829 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
2830 DAG.getConstant(OpIdx.index(), DL, XLenVT));
2831 } else {
2832 // Blend in all instances of this value using a VSELECT, using a
2833 // mask where each bit signals whether that element is the one
2834 // we're after.
2835 SmallVector<SDValue> Ops;
2836 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
2837 return DAG.getConstant(V == V1, DL, XLenVT);
2838 });
2839 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
2840 DAG.getBuildVector(SelMaskTy, DL, Ops),
2841 DAG.getSplatBuildVector(VT, DL, V), Vec);
2842 }
2843 }
2844
2845 return Vec;
2846 }
2847
2848 return SDValue();
2849}
2850
2851static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
2852 SDValue Lo, SDValue Hi, SDValue VL,
2853 SelectionDAG &DAG) {
2854 if (!Passthru)
2855 Passthru = DAG.getUNDEF(VT);
2856 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
2857 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
2858 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
2859 // If Hi constant is all the same sign bit as Lo, lower this as a custom
2860 // node in order to try and match RVV vector/scalar instructions.
2861 if ((LoC >> 31) == HiC)
2862 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
2863
2864 // If vl is equal to XLEN_MAX and Hi constant is equal to Lo, we could use
2865 // vmv.v.x whose EEW = 32 to lower it.
2866 auto *Const = dyn_cast<ConstantSDNode>(VL);
2867 if (LoC == HiC && Const && Const->isAllOnes()) {
2868 MVT InterVT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
2869 // TODO: if vl <= min(VLMAX), we can also do this. But we could not
2870 // access the subtarget here now.
2871 auto InterVec = DAG.getNode(
2872 RISCVISD::VMV_V_X_VL, DL, InterVT, DAG.getUNDEF(InterVT), Lo,
2873 DAG.getRegister(RISCV::X0, MVT::i32));
2874 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
2875 }
2876 }
2877
2878 // Fall back to a stack store and stride x0 vector load.
2879 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
2880 Hi, VL);
2881}
2882
2883// Called by type legalization to handle splat of i64 on RV32.
2884// FIXME: We can optimize this when the type has sign or zero bits in one
2885// of the halves.
2886static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
2887 SDValue Scalar, SDValue VL,
2888 SelectionDAG &DAG) {
2889 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
2890 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2891 DAG.getConstant(0, DL, MVT::i32));
2892 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
2893 DAG.getConstant(1, DL, MVT::i32));
2894 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
2895}
2896
2897// This function lowers a splat of a scalar operand Splat with the vector
2898// length VL. It ensures the final sequence is type legal, which is useful when
2899// lowering a splat after type legalization.
2900static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
2901 MVT VT, SDLoc DL, SelectionDAG &DAG,
2902 const RISCVSubtarget &Subtarget) {
2903 bool HasPassthru = Passthru && !Passthru.isUndef();
2904 if (!HasPassthru && !Passthru)
2905 Passthru = DAG.getUNDEF(VT);
2906 if (VT.isFloatingPoint()) {
2907 // If VL is 1, we could use vfmv.s.f.
2908 if (isOneConstant(VL))
2909 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, Passthru, Scalar, VL);
2910 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
2911 }
2912
2913 MVT XLenVT = Subtarget.getXLenVT();
2914
2915 // Simplest case is that the operand needs to be promoted to XLenVT.
2916 if (Scalar.getValueType().bitsLE(XLenVT)) {
2917 // If the operand is a constant, sign extend to increase our chances
2918 // of being able to use a .vi instruction. ANY_EXTEND would become a
2919 // a zero extend and the simm5 check in isel would fail.
2920 // FIXME: Should we ignore the upper bits in isel instead?
2921 unsigned ExtOpc =
2922 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2923 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2924 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar);
2925 // If VL is 1 and the scalar value won't benefit from immediate, we could
2926 // use vmv.s.x.
2927 if (isOneConstant(VL) &&
2928 (!Const || isNullConstant(Scalar) || !isInt<5>(Const->getSExtValue())))
2929 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru, Scalar, VL);
2930 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
2931 }
2932
2933 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
2934 "Unexpected scalar for splat lowering!");
2935
2936 if (isOneConstant(VL) && isNullConstant(Scalar))
2937 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
2938 DAG.getConstant(0, DL, XLenVT), VL);
2939
2940 // Otherwise use the more complicated splatting algorithm.
2941 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
2942}
2943
2944static MVT getLMUL1VT(MVT VT) {
2945 assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
2946 "Unexpected vector MVT");
2947 return MVT::getScalableVectorVT(
2948 VT.getVectorElementType(),
2949 RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
2950}
2951
2952// This function lowers an insert of a scalar operand Scalar into lane
2953// 0 of the vector regardless of the value of VL. The contents of the
2954// remaining lanes of the result vector are unspecified. VL is assumed
2955// to be non-zero.
2956static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL,
2957 MVT VT, SDLoc DL, SelectionDAG &DAG,
2958 const RISCVSubtarget &Subtarget) {
2959 const MVT XLenVT = Subtarget.getXLenVT();
2960
2961 SDValue Passthru = DAG.getUNDEF(VT);
2962 if (VT.isFloatingPoint()) {
2963 // TODO: Use vmv.v.i for appropriate constants
2964 // Use M1 or smaller to avoid over constraining register allocation
2965 const MVT M1VT = getLMUL1VT(VT);
2966 auto InnerVT = VT.bitsLE(M1VT) ? VT : M1VT;
2967 SDValue Result = DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, InnerVT,
2968 DAG.getUNDEF(InnerVT), Scalar, VL);
2969 if (VT != InnerVT)
2970 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
2971 DAG.getUNDEF(VT),
2972 Result, DAG.getConstant(0, DL, XLenVT));
2973 return Result;
2974 }
2975
2976
2977 // Avoid the tricky legalization cases by falling back to using the
2978 // splat code which already handles it gracefully.
2979 if (!Scalar.getValueType().bitsLE(XLenVT))
2980 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
2981 DAG.getConstant(1, DL, XLenVT),
2982 VT, DL, DAG, Subtarget);
2983
2984 // If the operand is a constant, sign extend to increase our chances
2985 // of being able to use a .vi instruction. ANY_EXTEND would become a
2986 // a zero extend and the simm5 check in isel would fail.
2987 // FIXME: Should we ignore the upper bits in isel instead?
2988 unsigned ExtOpc =
2989 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
2990 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
2991 // We use a vmv.v.i if possible. We limit this to LMUL1. LMUL2 or
2992 // higher would involve overly constraining the register allocator for
2993 // no purpose.
2994 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Scalar)) {
2995 if (!isNullConstant(Scalar) && isInt<5>(Const->getSExtValue()) &&
2996 VT.bitsLE(getLMUL1VT(VT)))
2997 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
2998 }
2999 // Use M1 or smaller to avoid over constraining register allocation
3000 const MVT M1VT = getLMUL1VT(VT);
3001 auto InnerVT = VT.bitsLE(M1VT) ? VT : M1VT;
3002 SDValue Result = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, InnerVT,
3003 DAG.getUNDEF(InnerVT), Scalar, VL);
3004 if (VT != InnerVT)
3005 Result = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
3006 DAG.getUNDEF(VT),
3007 Result, DAG.getConstant(0, DL, XLenVT));
3008 return Result;
3009
3010}
3011
3012// Is this a shuffle extracts either the even or odd elements of a vector?
3013// That is, specifically, either (a) or (b) below.
3014// t34: v8i8 = extract_subvector t11, Constant:i64<0>
3015// t33: v8i8 = extract_subvector t11, Constant:i64<8>
3016// a) t35: v8i8 = vector_shuffle<0,2,4,6,8,10,12,14> t34, t33
3017// b) t35: v8i8 = vector_shuffle<1,3,5,7,9,11,13,15> t34, t33
3018// Returns {Src Vector, Even Elements} om success
3019static bool isDeinterleaveShuffle(MVT VT, MVT ContainerVT, SDValue V1,
3020 SDValue V2, ArrayRef<int> Mask,
3021 const RISCVSubtarget &Subtarget) {
3022 // Need to be able to widen the vector.
3023 if (VT.getScalarSizeInBits() >= Subtarget.getELEN())
3024 return false;
3025
3026 // Both input must be extracts.
3027 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
3028 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
3029 return false;
3030
3031 // Extracting from the same source.
3032 SDValue Src = V1.getOperand(0);
3033 if (Src != V2.getOperand(0))
3034 return false;
3035
3036 // Src needs to have twice the number of elements.
3037 if (Src.getValueType().getVectorNumElements() != (Mask.size() * 2))
3038 return false;
3039
3040 // The extracts must extract the two halves of the source.
3041 if (V1.getConstantOperandVal(1) != 0 ||
3042 V2.getConstantOperandVal(1) != Mask.size())
3043 return false;
3044
3045 // First index must be the first even or odd element from V1.
3046 if (Mask[0] != 0 && Mask[0] != 1)
3047 return false;
3048
3049 // The others must increase by 2 each time.
3050 // TODO: Support undef elements?
3051 for (unsigned i = 1; i != Mask.size(); ++i)
3052 if (Mask[i] != Mask[i - 1] + 2)
3053 return false;
3054
3055 return true;
3056}
3057
3058/// Is this shuffle interleaving contiguous elements from one vector into the
3059/// even elements and contiguous elements from another vector into the odd
3060/// elements. \p Src1 will contain the element that should be in the first even
3061/// element. \p Src2 will contain the element that should be in the first odd
3062/// element. These can be the first element in a source or the element half
3063/// way through the source.
3064static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
3065 int &OddSrc, const RISCVSubtarget &Subtarget) {
3066 // We need to be able to widen elements to the next larger integer type.
3067 if (VT.getScalarSizeInBits() >= Subtarget.getELEN())
3068 return false;
3069
3070 int Size = Mask.size();
3071 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
3072
3073 SmallVector<unsigned, 2> StartIndexes;
3074 if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
3075 return false;
3076
3077 EvenSrc = StartIndexes[0] % 2 ? StartIndexes[1] : StartIndexes[0];
3078 OddSrc = StartIndexes[0] % 2 ? StartIndexes[0] : StartIndexes[1];
3079
3080 // One source should be low half of first vector.
3081 if (EvenSrc != 0 && OddSrc != 0)
3082 return false;
3083
3084 return true;
3085}
3086
3087/// Match shuffles that concatenate two vectors, rotate the concatenation,
3088/// and then extract the original number of elements from the rotated result.
3089/// This is equivalent to vector.splice or X86's PALIGNR instruction. The
3090/// returned rotation amount is for a rotate right, where elements move from
3091/// higher elements to lower elements. \p LoSrc indicates the first source
3092/// vector of the rotate or -1 for undef. \p HiSrc indicates the second vector
3093/// of the rotate or -1 for undef. At least one of \p LoSrc and \p HiSrc will be
3094/// 0 or 1 if a rotation is found.
3095///
3096/// NOTE: We talk about rotate to the right which matches how bit shift and
3097/// rotate instructions are described where LSBs are on the right, but LLVM IR
3098/// and the table below write vectors with the lowest elements on the left.
3099static int isElementRotate(int &LoSrc, int &HiSrc, ArrayRef<int> Mask) {
3100 int Size = Mask.size();
3101
3102 // We need to detect various ways of spelling a rotation:
3103 // [11, 12, 13, 14, 15, 0, 1, 2]
3104 // [-1, 12, 13, 14, -1, -1, 1, -1]
3105 // [-1, -1, -1, -1, -1, -1, 1, 2]
3106 // [ 3, 4, 5, 6, 7, 8, 9, 10]
3107 // [-1, 4, 5, 6, -1, -1, 9, -1]
3108 // [-1, 4, 5, 6, -1, -1, -1, -1]
3109 int Rotation = 0;
3110 LoSrc = -1;
3111 HiSrc = -1;
3112 for (int i = 0; i != Size; ++i) {
3113 int M = Mask[i];
3114 if (M < 0)
3115 continue;
3116
3117 // Determine where a rotate vector would have started.
3118 int StartIdx = i - (M % Size);
3119 // The identity rotation isn't interesting, stop.
3120 if (StartIdx == 0)
3121 return -1;
3122
3123 // If we found the tail of a vector the rotation must be the missing
3124 // front. If we found the head of a vector, it must be how much of the
3125 // head.
3126 int CandidateRotation = StartIdx < 0 ? -StartIdx : Size - StartIdx;
3127
3128 if (Rotation == 0)
3129 Rotation = CandidateRotation;
3130 else if (Rotation != CandidateRotation)
3131 // The rotations don't match, so we can't match this mask.
3132 return -1;
3133
3134 // Compute which value this mask is pointing at.
3135 int MaskSrc = M < Size ? 0 : 1;
3136
3137 // Compute which of the two target values this index should be assigned to.
3138 // This reflects whether the high elements are remaining or the low elemnts
3139 // are remaining.
3140 int &TargetSrc = StartIdx < 0 ? HiSrc : LoSrc;
3141
3142 // Either set up this value if we've not encountered it before, or check
3143 // that it remains consistent.
3144 if (TargetSrc < 0)
3145 TargetSrc = MaskSrc;
3146 else if (TargetSrc != MaskSrc)
3147 // This may be a rotation, but it pulls from the inputs in some
3148 // unsupported interleaving.
3149 return -1;
3150 }
3151
3152 // Check that we successfully analyzed the mask, and normalize the results.
3153 assert(Rotation != 0 && "Failed to locate a viable rotation!");
3154 assert((LoSrc >= 0 || HiSrc >= 0) &&
3155 "Failed to find a rotated input vector!");
3156
3157 return Rotation;
3158}
3159
3160// Lower a deinterleave shuffle to vnsrl.
3161// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (EvenElts == true)
3162// -> [p, q, r, s] (EvenElts == false)
3163// VT is the type of the vector to return, <[vscale x ]n x ty>
3164// Src is the vector to deinterleave of type <[vscale x ]n*2 x ty>
3165static SDValue getDeinterleaveViaVNSRL(const SDLoc &DL, MVT VT, SDValue Src,
3166 bool EvenElts,
3167 const RISCVSubtarget &Subtarget,
3168 SelectionDAG &DAG) {
3169 // The result is a vector of type <m x n x ty>
3170 MVT ContainerVT = VT;
3171 // Convert fixed vectors to scalable if needed
3172 if (ContainerVT.isFixedLengthVector()) {
3173 assert(Src.getSimpleValueType().isFixedLengthVector());
3174 ContainerVT = getContainerForFixedLengthVector(DAG, ContainerVT, Subtarget);
3175
3176 // The source is a vector of type <m x n*2 x ty>
3177 MVT SrcContainerVT =
3178 MVT::getVectorVT(ContainerVT.getVectorElementType(),
3179 ContainerVT.getVectorElementCount() * 2);
3180 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3181 }
3182
3183 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3184
3185 // Bitcast the source vector from <m x n*2 x ty> -> <m x n x ty*2>
3186 // This also converts FP to int.
3187 unsigned EltBits = ContainerVT.getScalarSizeInBits();
3188 MVT WideSrcContainerVT = MVT::getVectorVT(
3189 MVT::getIntegerVT(EltBits * 2), ContainerVT.getVectorElementCount());
3190 Src = DAG.getBitcast(WideSrcContainerVT, Src);
3191
3192 // The integer version of the container type.
3193 MVT IntContainerVT = ContainerVT.changeVectorElementTypeToInteger();
3194
3195 // If we want even elements, then the shift amount is 0. Otherwise, shift by
3196 // the original element size.
3197 unsigned Shift = EvenElts ? 0 : EltBits;
3198 SDValue SplatShift = DAG.getNode(
3199 RISCVISD::VMV_V_X_VL, DL, IntContainerVT, DAG.getUNDEF(ContainerVT),
3200 DAG.getConstant(Shift, DL, Subtarget.getXLenVT()), VL);
3201 SDValue Res =
3202 DAG.getNode(RISCVISD::VNSRL_VL, DL, IntContainerVT, Src, SplatShift,
3203 DAG.getUNDEF(IntContainerVT), TrueMask, VL);
3204 // Cast back to FP if needed.
3205 Res = DAG.getBitcast(ContainerVT, Res);
3206
3207 if (VT.isFixedLengthVector())
3208 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
3209 return Res;
3210}
3211
3212static SDValue
3213getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
3214 EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
3215 SDValue VL,
3216 unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3217 if (Merge.isUndef())
3218 Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
3219 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3220 SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
3221 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3222}
3223
3224static SDValue
3225getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, SDLoc DL,
3226 EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask,
3227 SDValue VL,
3228 unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3229 if (Merge.isUndef())
3230 Policy = RISCVII::TAIL_AGNOSTIC | RISCVII::MASK_AGNOSTIC;
3231 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3232 SDValue Ops[] = {Merge, Op, Offset, Mask, VL, PolicyOp};
3233 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3234}
3235
3236// Lower the following shuffle to vslidedown.
3237// a)
3238// t49: v8i8 = extract_subvector t13, Constant:i64<0>
3239// t109: v8i8 = extract_subvector t13, Constant:i64<8>
3240// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
3241// b)
3242// t69: v16i16 = extract_subvector t68, Constant:i64<0>
3243// t23: v8i16 = extract_subvector t69, Constant:i64<0>
3244// t29: v4i16 = extract_subvector t23, Constant:i64<4>
3245// t26: v8i16 = extract_subvector t69, Constant:i64<8>
3246// t30: v4i16 = extract_subvector t26, Constant:i64<0>
3247// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
3248static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
3249 SDValue V1, SDValue V2,
3250 ArrayRef<int> Mask,
3251 const RISCVSubtarget &Subtarget,
3252 SelectionDAG &DAG) {
3253 auto findNonEXTRACT_SUBVECTORParent =
3254 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
3255 uint64_t Offset = 0;
3256 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
3257 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
3258 // a scalable vector. But we don't want to match the case.
3259 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
3260 Offset += Parent.getConstantOperandVal(1);
3261 Parent = Parent.getOperand(0);
3262 }
3263 return std::make_pair(Parent, Offset);
3264 };
3265
3266 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
3267 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
3268
3269 // Extracting from the same source.
3270 SDValue Src = V1Src;
3271 if (Src != V2Src)
3272 return SDValue();
3273
3274 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
3275 SmallVector<int, 16> NewMask(Mask);
3276 for (size_t i = 0; i != NewMask.size(); ++i) {
3277 if (NewMask[i] == -1)
3278 continue;
3279
3280 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
3281 NewMask[i] = NewMask[i] + V1IndexOffset;
3282 } else {
3283 // Minus NewMask.size() is needed. Otherwise, the b case would be
3284 // <5,6,7,12> instead of <5,6,7,8>.
3285 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
3286 }
3287 }
3288
3289 // First index must be known and non-zero. It will be used as the slidedown
3290 // amount.
3291 if (NewMask[0] <= 0)
3292 return SDValue();
3293
3294 // NewMask is also continuous.
3295 for (unsigned i = 1; i != NewMask.size(); ++i)
3296 if (NewMask[i - 1] + 1 != NewMask[i])
3297 return SDValue();
3298
3299 MVT XLenVT = Subtarget.getXLenVT();
3300 MVT SrcVT = Src.getSimpleValueType();
3301 MVT ContainerVT = getContainerForFixedLengthVector(DAG, SrcVT, Subtarget);
3302 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
3303 SDValue Slidedown =
3304 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
3305 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
3306 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
3307 return DAG.getNode(
3308 ISD::EXTRACT_SUBVECTOR, DL, VT,
3309 convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget),
3310 DAG.getConstant(0, DL, XLenVT));
3311}
3312
3313// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
3314// to create an interleaved vector of <[vscale x] n*2 x ty>.
3315// This requires that the size of ty is less than the subtarget's maximum ELEN.
3316static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV, SDLoc &DL,
3317 SelectionDAG &DAG,
3318 const RISCVSubtarget &Subtarget) {
3319 MVT VecVT = EvenV.getSimpleValueType();
3320 MVT VecContainerVT = VecVT; // <vscale x n x ty>
3321 // Convert fixed vectors to scalable if needed
3322 if (VecContainerVT.isFixedLengthVector()) {
3323 VecContainerVT = getContainerForFixedLengthVector(DAG, VecVT, Subtarget);
3324 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
3325 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
3326 }
3327
3328 assert(VecVT.getScalarSizeInBits() < Subtarget.getELEN());
3329
3330 // We're working with a vector of the same size as the resulting
3331 // interleaved vector, but with half the number of elements and
3332 // twice the SEW (Hence the restriction on not using the maximum
3333 // ELEN)
3334 MVT WideVT =
3335 MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
3336 VecVT.getVectorElementCount());
3337 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
3338 if (WideContainerVT.isFixedLengthVector())
3339 WideContainerVT = getContainerForFixedLengthVector(DAG, WideVT, Subtarget);
3340
3341 // Bitcast the input vectors to integers in case they are FP
3342 VecContainerVT = VecContainerVT.changeTypeToInteger();
3343 EvenV = DAG.getBitcast(VecContainerVT, EvenV);
3344 OddV = DAG.getBitcast(VecContainerVT, OddV);
3345
3346 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
3347 SDValue Passthru = DAG.getUNDEF(WideContainerVT);
3348
3349 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
3350 // vwaddu.vv
3351 SDValue Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT,
3352 EvenV, OddV, Passthru, Mask, VL);
3353
3354 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
3355 SDValue AllOnesVec = DAG.getSplatVector(
3356 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
3357 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT, OddV,
3358 AllOnesVec, Passthru, Mask, VL);
3359
3360 // Add the two together so we get
3361 // (OddV * 0xff...ff) + (OddV + EvenV)
3362 // = (OddV * 0x100...00) + EvenV
3363 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
3364 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
3365 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT, Interleaved,
3366 OddsMul, Passthru, Mask, VL);
3367
3368 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
3369 MVT ResultContainerVT = MVT::getVectorVT(
3370 VecVT.getVectorElementType(), // Make sure to use original type
3371 VecContainerVT.getVectorElementCount().multiplyCoefficientBy(2));
3372 Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
3373
3374 // Convert back to a fixed vector if needed
3375 MVT ResultVT =
3376 MVT::getVectorVT(VecVT.getVectorElementType(),
3377 VecVT.getVectorElementCount().multiplyCoefficientBy(2));
3378 if (ResultVT.isFixedLengthVector())
3379 Interleaved =
3380 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
3381
3382 return Interleaved;
3383}
3384
3385static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
3386 const RISCVSubtarget &Subtarget) {
3387 SDValue V1 = Op.getOperand(0);
3388 SDValue V2 = Op.getOperand(1);
3389 SDLoc DL(Op);
3390 MVT XLenVT = Subtarget.getXLenVT();
3391 MVT VT = Op.getSimpleValueType();
3392 unsigned NumElts = VT.getVectorNumElements();
3393 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
3394
3395 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3396
3397 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3398
3399 if (SVN->isSplat()) {
3400 const int Lane = SVN->getSplatIndex();
3401 if (Lane >= 0) {
3402 MVT SVT = VT.getVectorElementType();
3403
3404 // Turn splatted vector load into a strided load with an X0 stride.
3405 SDValue V = V1;
3406 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
3407 // with undef.
3408 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
3409 int Offset = Lane;
3410 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
3411 int OpElements =
3412 V.getOperand(0).getSimpleValueType().getVectorNumElements();
3413 V = V.getOperand(Offset / OpElements);
3414 Offset %= OpElements;
3415 }
3416
3417 // We need to ensure the load isn't atomic or volatile.
3418 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
3419 auto *Ld = cast<LoadSDNode>(V);
3420 Offset *= SVT.getStoreSize();
3421 SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
3422 TypeSize::Fixed(Offset), DL);
3423
3424 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
3425 if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
3426 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
3427 SDValue IntID =
3428 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
3429 SDValue Ops[] = {Ld->getChain(),
3430 IntID,
3431 DAG.getUNDEF(ContainerVT),
3432 NewAddr,
3433 DAG.getRegister(RISCV::X0, XLenVT),
3434 VL};
3435 SDValue NewLoad = DAG.getMemIntrinsicNode(
3436 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
3437 DAG.getMachineFunction().getMachineMemOperand(
3438 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
3439 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
3440 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
3441 }
3442
3443 // Otherwise use a scalar load and splat. This will give the best
3444 // opportunity to fold a splat into the operation. ISel can turn it into
3445 // the x0 strided load if we aren't able to fold away the select.
3446 if (SVT.isFloatingPoint())
3447 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
3448 Ld->getPointerInfo().getWithOffset(Offset),
3449 Ld->getOriginalAlign(),
3450 Ld->getMemOperand()->getFlags());
3451 else
3452 V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
3453 Ld->getPointerInfo().getWithOffset(Offset), SVT,
3454 Ld->getOriginalAlign(),
3455 Ld->getMemOperand()->getFlags());
3456 DAG.makeEquivalentMemoryOrdering(Ld, V);
3457
3458 unsigned Opc =
3459 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
3460 SDValue Splat =
3461 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), V, VL);
3462 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
3463 }
3464
3465 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
3466 assert(Lane < (int)NumElts && "Unexpected lane!");
3467 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
3468 V1, DAG.getConstant(Lane, DL, XLenVT),
3469 DAG.getUNDEF(ContainerVT), TrueMask, VL);
3470 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
3471 }
3472 }
3473
3474 ArrayRef<int> Mask = SVN->getMask();
3475
3476 if (SDValue V =
3477 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
3478 return V;
3479
3480 // Lower rotations to a SLIDEDOWN and a SLIDEUP. One of the source vectors may
3481 // be undef which can be handled with a single SLIDEDOWN/UP.
3482 int LoSrc, HiSrc;
3483 int Rotation = isElementRotate(LoSrc, HiSrc, Mask);
3484 if (Rotation > 0) {
3485 SDValue LoV, HiV;
3486 if (LoSrc >= 0) {
3487 LoV = LoSrc == 0 ? V1 : V2;
3488 LoV = convertToScalableVector(ContainerVT, LoV, DAG, Subtarget);
3489 }
3490 if (HiSrc >= 0) {
3491 HiV = HiSrc == 0 ? V1 : V2;
3492 HiV = convertToScalableVector(ContainerVT, HiV, DAG, Subtarget);
3493 }
3494
3495 // We found a rotation. We need to slide HiV down by Rotation. Then we need
3496 // to slide LoV up by (NumElts - Rotation).
3497 unsigned InvRotate = NumElts - Rotation;
3498
3499 SDValue Res = DAG.getUNDEF(ContainerVT);
3500 if (HiV) {
3501 // If we are doing a SLIDEDOWN+SLIDEUP, reduce the VL for the SLIDEDOWN.
3502 // FIXME: If we are only doing a SLIDEDOWN, don't reduce the VL as it
3503 // causes multiple vsetvlis in some test cases such as lowering
3504 // reduce.mul
3505 SDValue DownVL = VL;
3506 if (LoV)
3507 DownVL = DAG.getConstant(InvRotate, DL, XLenVT);
3508 Res = getVSlidedown(DAG, Subtarget, DL, ContainerVT, Res, HiV,
3509 DAG.getConstant(Rotation, DL, XLenVT), TrueMask,
3510 DownVL);
3511 }
3512 if (LoV)
3513 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, Res, LoV,
3514 DAG.getConstant(InvRotate, DL, XLenVT), TrueMask, VL,
3515 RISCVII::TAIL_AGNOSTIC);
3516
3517 return convertFromScalableVector(VT, Res, DAG, Subtarget);
3518 }
3519
3520 // If this is a deinterleave and we can widen the vector, then we can use
3521 // vnsrl to deinterleave.
3522 if (isDeinterleaveShuffle(VT, ContainerVT, V1, V2, Mask, Subtarget)) {
3523 return getDeinterleaveViaVNSRL(DL, VT, V1.getOperand(0), Mask[0] == 0,
3524 Subtarget, DAG);
3525 }
3526
3527 // Detect an interleave shuffle and lower to
3528 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
3529 int EvenSrc, OddSrc;
3530 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget)) {
3531 // Extract the halves of the vectors.
3532 MVT HalfVT = VT.getHalfNumVectorElementsVT();
3533
3534 int Size = Mask.size();
3535 SDValue EvenV, OddV;
3536 assert(EvenSrc >= 0 && "Undef source?");
3537 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
3538 EvenV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, EvenV,
3539 DAG.getConstant(EvenSrc % Size, DL, XLenVT));
3540
3541 assert(OddSrc >= 0 && "Undef source?");
3542 OddV = (OddSrc / Size) == 0 ? V1 : V2;
3543 OddV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, OddV,
3544 DAG.getConstant(OddSrc % Size, DL, XLenVT));
3545
3546 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
3547 }
3548
3549 // Detect shuffles which can be re-expressed as vector selects; these are
3550 // shuffles in which each element in the destination is taken from an element
3551 // at the corresponding index in either source vectors.
3552 bool IsSelect = all_of(enumerate(Mask), [&](const auto &MaskIdx) {
3553 int MaskIndex = MaskIdx.value();
3554 return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
3555 });
3556
3557 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
3558
3559 SmallVector<SDValue> MaskVals;
3560 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
3561 // merged with a second vrgather.
3562 SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
3563
3564 // By default we preserve the original operand order, and use a mask to
3565 // select LHS as true and RHS as false. However, since RVV vector selects may
3566 // feature splats but only on the LHS, we may choose to invert our mask and
3567 // instead select between RHS and LHS.
3568 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
3569 bool InvertMask = IsSelect == SwapOps;
3570
3571 // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
3572 // half.
3573 DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
3574
3575 // Now construct the mask that will be used by the vselect or blended
3576 // vrgather operation. For vrgathers, construct the appropriate indices into
3577 // each vector.
3578 for (int MaskIndex : Mask) {
3579 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
3580 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
3581 if (!IsSelect) {
3582 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
3583 GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
3584 ? DAG.getConstant(MaskIndex, DL, XLenVT)
3585 : DAG.getUNDEF(XLenVT));
3586 GatherIndicesRHS.push_back(
3587 IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
3588 : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
3589 if (IsLHSOrUndefIndex && MaskIndex >= 0)
3590 ++LHSIndexCounts[MaskIndex];
3591 if (!IsLHSOrUndefIndex)
3592 ++RHSIndexCounts[MaskIndex - NumElts];
3593 }
3594 }
3595
3596 if (SwapOps) {
3597 std::swap(V1, V2);
3598 std::swap(GatherIndicesLHS, GatherIndicesRHS);
3599 }
3600
3601 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
3602 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
3603 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
3604
3605 if (IsSelect)
3606 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
3607
3608 if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
3609 // On such a large vector we're unable to use i8 as the index type.
3610 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
3611 // may involve vector splitting if we're already at LMUL=8, or our
3612 // user-supplied maximum fixed-length LMUL.
3613 return SDValue();
3614 }
3615
3616 unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
3617 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
3618 MVT IndexVT = VT.changeTypeToInteger();
3619 // Since we can't introduce illegal index types at this stage, use i16 and
3620 // vrgatherei16 if the corresponding index type for plain vrgather is greater
3621 // than XLenVT.
3622 if (IndexVT.getScalarType().bitsGT(XLenVT)) {
3623 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
3624 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
3625 }
3626
3627 MVT IndexContainerVT =
3628 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
3629
3630 SDValue Gather;
3631 // TODO: This doesn't trigger for i64 vectors on RV32, since there we
3632 // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
3633 if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
3634 Gather = lowerScalarSplat(SDValue(), SplatValue, VL, ContainerVT, DL, DAG,
3635 Subtarget);
3636 } else {
3637 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
3638 // If only one index is used, we can use a "splat" vrgather.
3639 // TODO: We can splat the most-common index and fix-up any stragglers, if
3640 // that's beneficial.
3641 if (LHSIndexCounts.size() == 1) {
3642 int SplatIndex = LHSIndexCounts.begin()->getFirst();
3643 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
3644 DAG.getConstant(SplatIndex, DL, XLenVT),
3645 DAG.getUNDEF(ContainerVT), TrueMask, VL);
3646 } else {
3647 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
3648 LHSIndices =
3649 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
3650
3651 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
3652 DAG.getUNDEF(ContainerVT), TrueMask, VL);
3653 }
3654 }
3655
3656 // If a second vector operand is used by this shuffle, blend it in with an
3657 // additional vrgather.
3658 if (!V2.isUndef()) {
3659 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
3660
3661 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
3662 SelectMask =
3663 convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
3664
3665 // If only one index is used, we can use a "splat" vrgather.
3666 // TODO: We can splat the most-common index and fix-up any stragglers, if
3667 // that's beneficial.
3668 if (RHSIndexCounts.size() == 1) {
3669 int SplatIndex = RHSIndexCounts.begin()->getFirst();
3670 Gather = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
3671 DAG.getConstant(SplatIndex, DL, XLenVT), Gather,
3672 SelectMask, VL);
3673 } else {
3674 SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
3675 RHSIndices =
3676 convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
3677 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, Gather,
3678 SelectMask, VL);
3679 }
3680 }
3681
3682 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
3683}
3684
3685bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
3686 // Support splats for any type. These should type legalize well.
3687 if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
3688 return true;
3689
3690 // Only support legal VTs for other shuffles for now.
3691 if (!isTypeLegal(VT))
3692 return false;
3693
3694 MVT SVT = VT.getSimpleVT();
3695
3696 int Dummy1, Dummy2;
3697 return (isElementRotate(Dummy1, Dummy2, M) > 0) ||
3698 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
3699}
3700
3701// Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
3702// the exponent.
3703SDValue
3704RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op,
3705 SelectionDAG &DAG) const {
3706 MVT VT = Op.getSimpleValueType();
3707 unsigned EltSize = VT.getScalarSizeInBits();
3708 SDValue Src = Op.getOperand(0);
3709 SDLoc DL(Op);
3710
3711 // We choose FP type that can represent the value if possible. Otherwise, we
3712 // use rounding to zero conversion for correct exponent of the result.
3713 // TODO: Use f16 for i8 when possible?
3714 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
3715 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
3716 FloatEltVT = MVT::f32;
3717 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
3718
3719 // Legal types should have been checked in the RISCVTargetLowering
3720 // constructor.
3721 // TODO: Splitting may make sense in some cases.
3722 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
3723 "Expected legal float type!");
3724
3725 // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
3726 // The trailing zero count is equal to log2 of this single bit value.
3727 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
3728 SDValue Neg = DAG.getNegative(Src, DL, VT);
3729 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
3730 }
3731
3732 // We have a legal FP type, convert to it.
3733 SDValue FloatVal;
3734 if (FloatVT.bitsGT(VT)) {
3735 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
3736 } else {
3737 // Use RTZ to avoid rounding influencing exponent of FloatVal.
3738 MVT ContainerVT = VT;
3739 if (VT.isFixedLengthVector()) {
3740 ContainerVT = getContainerForFixedLengthVector(VT);
3741 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3742 }
3743
3744 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3745 SDValue RTZRM =
3746 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
3747 MVT ContainerFloatVT =
3748 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
3749 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
3750 Src, Mask, RTZRM, VL);
3751 if (VT.isFixedLengthVector())
3752 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
3753 }
3754 // Bitcast to integer and shift the exponent to the LSB.
3755 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
3756 SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
3757 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
3758 SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
3759 DAG.getConstant(ShiftAmt, DL, IntVT));
3760 // Restore back to original type. Truncation after SRL is to generate vnsrl.
3761 if (IntVT.bitsLT(VT))
3762 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
3763 else if (IntVT.bitsGT(VT))
3764 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
3765 // The exponent contains log2 of the value in biased form.
3766 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
3767
3768 // For trailing zeros, we just need to subtract the bias.
3769 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
3770 return DAG.getNode(ISD::SUB, DL, VT, Exp,
3771 DAG.getConstant(ExponentBias, DL, VT));
3772
3773 // For leading zeros, we need to remove the bias and convert from log2 to
3774 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
3775 unsigned Adjust = ExponentBias + (EltSize - 1);
3776 SDValue Res =
3777 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
3778 // The above result with zero input equals to Adjust which is greater than
3779 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
3780 if (Op.getOpcode() == ISD::CTLZ)
3781 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
3782 return Res;
3783}
3784
3785// While RVV has alignment restrictions, we should always be able to load as a
3786// legal equivalently-sized byte-typed vector instead. This method is
3787// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
3788// the load is already correctly-aligned, it returns SDValue().
3789SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
3790 SelectionDAG &DAG) const {
3791 auto *Load = cast<LoadSDNode>(Op);
3792 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
3793
3794 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
3795 Load->getMemoryVT(),
3796 *Load->getMemOperand()))
3797 return SDValue();
3798
3799 SDLoc DL(Op);
3800 MVT VT = Op.getSimpleValueType();
3801 unsigned EltSizeBits = VT.getScalarSizeInBits();
3802 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
3803 "Unexpected unaligned RVV load type");
3804 MVT NewVT =
3805 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
3806 assert(NewVT.isValid() &&
3807 "Expecting equally-sized RVV vector types to be legal");
3808 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
3809 Load->getPointerInfo(), Load->getOriginalAlign(),
3810 Load->getMemOperand()->getFlags());
3811 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
3812}
3813
3814// While RVV has alignment restrictions, we should always be able to store as a
3815// legal equivalently-sized byte-typed vector instead. This method is
3816// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
3817// returns SDValue() if the store is already correctly aligned.
3818SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
3819 SelectionDAG &DAG) const {
3820 auto *Store = cast<StoreSDNode>(Op);
3821 assert(Store && Store->getValue().getValueType().isVector() &&
3822 "Expected vector store");
3823
3824 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
3825 Store->getMemoryVT(),
3826 *Store->getMemOperand()))
3827 return SDValue();
3828
3829 SDLoc DL(Op);
3830 SDValue StoredVal = Store->getValue();
3831 MVT VT = StoredVal.getSimpleValueType();
3832 unsigned EltSizeBits = VT.getScalarSizeInBits();
3833 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
3834 "Unexpected unaligned RVV store type");
3835 MVT NewVT =
3836 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
3837 assert(NewVT.isValid() &&
3838 "Expecting equally-sized RVV vector types to be legal");
3839 StoredVal = DAG.getBitcast(NewVT, StoredVal);
3840 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
3841 Store->getPointerInfo(), Store->getOriginalAlign(),
3842 Store->getMemOperand()->getFlags());
3843}
3844
3845static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
3846 const RISCVSubtarget &Subtarget) {
3847 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
3848
3849 int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
3850
3851 // All simm32 constants should be handled by isel.
3852 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
3853 // this check redundant, but small immediates are common so this check
3854 // should have better compile time.
3855 if (isInt<32>(Imm))
3856 return Op;
3857
3858 // We only need to cost the immediate, if constant pool lowering is enabled.
3859 if (!Subtarget.useConstantPoolForLargeInts())
3860 return Op;
3861
3862 RISCVMatInt::InstSeq Seq =
3863 RISCVMatInt::generateInstSeq(Imm, Subtarget.getFeatureBits());
3864 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
3865 return Op;
3866
3867 // Expand to a constant pool using the default expansion code.
3868 return SDValue();
3869}
3870
3871static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
3872 const RISCVSubtarget &Subtarget) {
3873 SDLoc dl(Op);
3874 AtomicOrdering FenceOrdering =
3875 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
3876 SyncScope::ID FenceSSID =
3877 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
3878
3879 if (Subtarget.hasStdExtZtso()) {
3880 // The only fence that needs an instruction is a sequentially-consistent
3881 // cross-thread fence.
3882 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
3883 FenceSSID == SyncScope::System)
3884 return Op;
3885
3886 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
3887 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
3888 }
3889
3890 // singlethread fences only synchronize with signal handlers on the same
3891 // thread and thus only need to preserve instruction order, not actually
3892 // enforce memory ordering.
3893 if (FenceSSID == SyncScope::SingleThread)
3894 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
3895 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
3896
3897 return Op;
3898}
3899
3900SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
3901 SelectionDAG &DAG) const {
3902 switch (Op.getOpcode()) {
3903 default:
3904 report_fatal_error("unimplemented operand");
3905 case ISD::ATOMIC_FENCE:
3906 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
3907 case ISD::GlobalAddress:
3908 return lowerGlobalAddress(Op, DAG);
3909 case ISD::BlockAddress:
3910 return lowerBlockAddress(Op, DAG);
3911 case ISD::ConstantPool:
3912 return lowerConstantPool(Op, DAG);
3913 case ISD::JumpTable:
3914 return lowerJumpTable(Op, DAG);
3915 case ISD::GlobalTLSAddress:
3916 return lowerGlobalTLSAddress(Op, DAG);
3917 case ISD::Constant:
3918 return lowerConstant(Op, DAG, Subtarget);
3919 case ISD::SELECT:
3920 return lowerSELECT(Op, DAG);
3921 case ISD::BRCOND:
3922 return lowerBRCOND(Op, DAG);
3923 case ISD::VASTART:
3924 return lowerVASTART(Op, DAG);
3925 case ISD::FRAMEADDR:
3926 return lowerFRAMEADDR(Op, DAG);
3927 case ISD::RETURNADDR:
3928 return lowerRETURNADDR(Op, DAG);
3929 case ISD::SHL_PARTS:
3930 return lowerShiftLeftParts(Op, DAG);
3931 case ISD::SRA_PARTS:
3932 return lowerShiftRightParts(Op, DAG, true);
3933 case ISD::SRL_PARTS:
3934 return lowerShiftRightParts(Op, DAG, false);
3935 case ISD::BITCAST: {
3936 SDLoc DL(Op);
3937 EVT VT = Op.getValueType();
3938 SDValue Op0 = Op.getOperand(0);
3939 EVT Op0VT = Op0.getValueType();
3940 MVT XLenVT = Subtarget.getXLenVT();
3941 if (VT == MVT::f16 && Op0VT == MVT::i16 &&
3942 Subtarget.hasStdExtZfhOrZfhmin()) {
3943 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
3944 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
3945 return FPConv;
3946 }
3947 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
3948 Subtarget.hasStdExtF()) {
3949 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
3950 SDValue FPConv =
3951 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
3952 return FPConv;
3953 }
3954 if (VT == MVT::f64 && Op0VT == MVT::i64 && XLenVT == MVT::i32 &&
3955 Subtarget.hasStdExtZfa()) {
3956 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op0,
3957 DAG.getConstant(0, DL, MVT::i32));
3958 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Op0,
3959 DAG.getConstant(1, DL, MVT::i32));
3960 SDValue RetReg =
3961 DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
3962 return RetReg;
3963 }
3964
3965 // Consider other scalar<->scalar casts as legal if the types are legal.
3966 // Otherwise expand them.
3967 if (!VT.isVector() && !Op0VT.isVector()) {
3968 if (isTypeLegal(VT) && isTypeLegal(Op0VT))
3969 return Op;
3970 return SDValue();
3971 }
3972
3973 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
3974 "Unexpected types");
3975
3976 if (VT.isFixedLengthVector()) {
3977 // We can handle fixed length vector bitcasts with a simple replacement
3978 // in isel.
3979 if (Op0VT.isFixedLengthVector())
3980 return Op;
3981 // When bitcasting from scalar to fixed-length vector, insert the scalar
3982 // into a one-element vector of the result type, and perform a vector
3983 // bitcast.
3984 if (!Op0VT.isVector()) {
3985 EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
3986 if (!isTypeLegal(BVT))
3987 return SDValue();
3988 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
3989 DAG.getUNDEF(BVT), Op0,
3990 DAG.getConstant(0, DL, XLenVT)));
3991 }
3992 return SDValue();
3993 }
3994 // Custom-legalize bitcasts from fixed-length vector types to scalar types
3995 // thus: bitcast the vector to a one-element vector type whose element type
3996 // is the same as the result type, and extract the first element.
3997 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
3998 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
3999 if (!isTypeLegal(BVT))
4000 return SDValue();
4001 SDValue BVec = DAG.getBitcast(BVT, Op0);
4002 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
4003 DAG.getConstant(0, DL, XLenVT));
4004 }
4005 return SDValue();
4006 }
4007 case ISD::INTRINSIC_WO_CHAIN:
4008 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
4009 case ISD::INTRINSIC_W_CHAIN:
4010 return LowerINTRINSIC_W_CHAIN(Op, DAG);
4011 case ISD::INTRINSIC_VOID:
4012 return LowerINTRINSIC_VOID(Op, DAG);
4013 case ISD::BITREVERSE: {
4014 MVT VT = Op.getSimpleValueType();
4015 SDLoc DL(Op);
4016 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
4017 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
4018 // Expand bitreverse to a bswap(rev8) followed by brev8.
4019 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
4020 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
4021 }
4022 case ISD::TRUNCATE:
4023 // Only custom-lower vector truncates
4024 if (!Op.getSimpleValueType().isVector())
4025 return Op;
4026 return lowerVectorTruncLike(Op, DAG);
4027 case ISD::ANY_EXTEND:
4028 case ISD::ZERO_EXTEND:
4029 if (Op.getOperand(0).getValueType().isVector() &&
4030 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
4031 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
4032 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
4033 case ISD::SIGN_EXTEND:
4034 if (Op.getOperand(0).getValueType().isVector() &&
4035 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
4036 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
4037 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
4038 case ISD::SPLAT_VECTOR_PARTS:
4039 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
4040 case ISD::INSERT_VECTOR_ELT:
4041 return lowerINSERT_VECTOR_ELT(Op, DAG);
4042 case ISD::EXTRACT_VECTOR_ELT:
4043 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
4044 case ISD::VSCALE: {
4045 MVT VT = Op.getSimpleValueType();
4046 SDLoc DL(Op);
4047 SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
4048 // We define our scalable vector types for lmul=1 to use a 64 bit known
4049 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
4050 // vscale as VLENB / 8.
4051 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
4052 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
4053 report_fatal_error("Support for VLEN==32 is incomplete.");
4054 // We assume VLENB is a multiple of 8. We manually choose the best shift
4055 // here because SimplifyDemandedBits isn't always able to simplify it.
4056 uint64_t Val = Op.getConstantOperandVal(0);
4057 if (isPowerOf2_64(Val)) {
4058 uint64_t Log2 = Log2_64(Val);
4059 if (Log2 < 3)
4060 return DAG.getNode(ISD::SRL, DL, VT, VLENB,
4061 DAG.getConstant(3 - Log2, DL, VT));
4062 if (Log2 > 3)
4063 return DAG.getNode(ISD::SHL, DL, VT, VLENB,
4064 DAG.getConstant(Log2 - 3, DL, VT));
4065 return VLENB;
4066 }
4067 // If the multiplier is a multiple of 8, scale it down to avoid needing
4068 // to shift the VLENB value.
4069 if ((Val % 8) == 0)
4070 return DAG.getNode(ISD::MUL, DL, VT, VLENB,
4071 DAG.getConstant(Val / 8, DL, VT));
4072
4073 SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
4074 DAG.getConstant(3, DL, VT));
4075 return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
4076 }
4077 case ISD::FPOWI: {
4078 // Custom promote f16 powi with illegal i32 integer type on RV64. Once
4079 // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
4080 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
4081 Op.getOperand(1).getValueType() == MVT::i32) {
4082 SDLoc DL(Op);
4083 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
4084 SDValue Powi =
4085 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
4086 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
4087 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
4088 }
4089 return SDValue();
4090 }
4091 case ISD::FP_EXTEND:
4092 case ISD::FP_ROUND:
4093 if (!Op.getValueType().isVector())
4094 return Op;
4095 return lowerVectorFPExtendOrRoundLike(Op, DAG);
4096 case ISD::STRICT_FP_EXTEND:
4097 return lowerStrictFPExtend(Op, DAG);
4098 case ISD::FP_TO_SINT:
4099 case ISD::FP_TO_UINT:
4100 case ISD::SINT_TO_FP:
4101 case ISD::UINT_TO_FP: {
4102 // RVV can only do fp<->int conversions to types half/double the size as
4103 // the source. We custom-lower any conversions that do two hops into
4104 // sequences.
4105 MVT VT = Op.getSimpleValueType();
4106 if (!VT.isVector())
4107 return Op;
4108 SDLoc DL(Op);
4109 SDValue Src = Op.getOperand(0);
4110 MVT EltVT = VT.getVectorElementType();
4111 MVT SrcVT = Src.getSimpleValueType();
4112 MVT SrcEltVT = SrcVT.getVectorElementType();
4113 unsigned EltSize = EltVT.getSizeInBits();
4114 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
4115 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
4116 "Unexpected vector element types");
4117
4118 bool IsInt2FP = SrcEltVT.isInteger();
4119 // Widening conversions
4120 if (EltSize > (2 * SrcEltSize)) {
4121 if (IsInt2FP) {
4122 // Do a regular integer sign/zero extension then convert to float.
4123 MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize / 2),
4124 VT.getVectorElementCount());
4125 unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
4126 ? ISD::ZERO_EXTEND
4127 : ISD::SIGN_EXTEND;
4128 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
4129 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
4130 }
4131 // FP2Int
4132 assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
4133 // Do one doubling fp_extend then complete the operation by converting
4134 // to int.
4135 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
4136 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
4137 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
4138 }
4139
4140 // Narrowing conversions
4141 if (SrcEltSize > (2 * EltSize)) {
4142 if (IsInt2FP) {
4143 // One narrowing int_to_fp, then an fp_round.
4144 assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
4145 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
4146 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
4147 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
4148 }
4149 // FP2Int
4150 // One narrowing fp_to_int, then truncate the integer. If the float isn't
4151 // representable by the integer, the result is poison.
4152 MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize / 2),
4153 VT.getVectorElementCount());
4154 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
4155 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
4156 }
4157
4158 // Scalable vectors can exit here. Patterns will handle equally-sized
4159 // conversions halving/doubling ones.
4160 if (!VT.isFixedLengthVector())
4161 return Op;
4162
4163 // For fixed-length vectors we lower to a custom "VL" node.
4164 unsigned RVVOpc = 0;
4165 switch (Op.getOpcode()) {
4166 default:
4167 llvm_unreachable("Impossible opcode");
4168 case ISD::FP_TO_SINT:
4169 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
4170 break;
4