LLVM 23.0.0git
RISCVISelLowering.cpp
Go to the documentation of this file.
1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineJumpTableInfo.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SDPatternMatch.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsRISCV.h"
44#include "llvm/MC/MCCodeEmitter.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/InstructionCost.h"
50#include "llvm/Support/KnownBits.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "riscv-lower"
58
59STATISTIC(NumTailCalls, "Number of tail calls");
60
61static cl::opt<unsigned> ExtensionMaxWebSize(
62 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
63 cl::desc("Give the maximum size (in number of nodes) of the web of "
64 "instructions that we will consider for VW expansion"),
65 cl::init(18));
66
67static cl::opt<bool>
68 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
69 cl::desc("Allow the formation of VW_W operations (e.g., "
70 "VWADD_W) with splat constants"),
71 cl::init(false));
72
73static cl::opt<unsigned> NumRepeatedDivisors(
74 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
75 cl::desc("Set the minimum number of repetitions of a divisor to allow "
76 "transformation to multiplications by the reciprocal"),
77 cl::init(2));
78
79static cl::opt<int>
80 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
81 cl::desc("Give the maximum number of instructions that we will "
82 "use for creating a floating-point immediate value"),
83 cl::init(3));
84
85static cl::opt<bool>
86 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
87 cl::desc("Swap add and addi in cases where the add may "
88 "be combined with a shift"),
89 cl::init(true));
90
91// TODO: Support more ops
98
99RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
100 const RISCVSubtarget &STI)
101 : TargetLowering(TM, STI), Subtarget(STI) {
102
103 RISCVABI::ABI ABI = Subtarget.getTargetABI();
104 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
105
106 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
107 !Subtarget.hasStdExtF()) {
108 errs() << "Hard-float 'f' ABI can't be used for a target that "
109 "doesn't support the F instruction set extension (ignoring "
110 "target-abi)\n";
111 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
112 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
113 !Subtarget.hasStdExtD()) {
114 errs() << "Hard-float 'd' ABI can't be used for a target that "
115 "doesn't support the D instruction set extension (ignoring "
116 "target-abi)\n";
117 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
118 }
119
120 switch (ABI) {
121 default:
122 reportFatalUsageError("Don't know how to lower this ABI");
123 case RISCVABI::ABI_ILP32:
124 case RISCVABI::ABI_ILP32E:
125 case RISCVABI::ABI_LP64E:
126 case RISCVABI::ABI_ILP32F:
127 case RISCVABI::ABI_ILP32D:
128 case RISCVABI::ABI_LP64:
129 case RISCVABI::ABI_LP64F:
130 case RISCVABI::ABI_LP64D:
131 break;
132 }
133
134 MVT XLenVT = Subtarget.getXLenVT();
135
136 // Set up the register classes.
137 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
138
139 if (Subtarget.hasStdExtZfhmin())
140 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
141 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
142 addRegisterClass(MVT::bf16, &RISCV::FPR16RegClass);
143 if (Subtarget.hasStdExtF())
144 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
145 if (Subtarget.hasStdExtD())
146 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
147 if (Subtarget.hasStdExtZhinxmin())
148 addRegisterClass(MVT::f16, &RISCV::GPRF16RegClass);
149 if (Subtarget.hasStdExtZfinx())
150 addRegisterClass(MVT::f32, &RISCV::GPRF32RegClass);
151 if (Subtarget.hasStdExtZdinx()) {
152 if (Subtarget.is64Bit())
153 addRegisterClass(MVT::f64, &RISCV::GPRRegClass);
154 else
155 addRegisterClass(MVT::f64, &RISCV::GPRPairRegClass);
156 }
157
158 static const MVT::SimpleValueType BoolVecVTs[] = {
159 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
160 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
161 static const MVT::SimpleValueType IntVecVTs[] = {
162 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
163 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
164 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
165 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
166 MVT::nxv4i64, MVT::nxv8i64};
167 static const MVT::SimpleValueType F16VecVTs[] = {
168 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
169 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
170 static const MVT::SimpleValueType BF16VecVTs[] = {
171 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
172 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
173 static const MVT::SimpleValueType F32VecVTs[] = {
174 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
175 static const MVT::SimpleValueType F64VecVTs[] = {
176 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
177 static const MVT::SimpleValueType VecTupleVTs[] = {
178 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
179 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
180 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
181 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
182 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
183 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
184 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
185 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
186 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
187 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
188 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
189
190 if (Subtarget.hasVInstructions()) {
191 auto addRegClassForRVV = [this](MVT VT) {
192 // Disable the smallest fractional LMUL types if ELEN is less than
193 // RVVBitsPerBlock.
194 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
195 if (VT.getVectorMinNumElements() < MinElts)
196 return;
197
198 unsigned Size = VT.getSizeInBits().getKnownMinValue();
199 const TargetRegisterClass *RC;
200 if (Size <= RISCV::RVVBitsPerBlock)
201 RC = &RISCV::VRRegClass;
202 else if (Size == 2 * RISCV::RVVBitsPerBlock)
203 RC = &RISCV::VRM2RegClass;
204 else if (Size == 4 * RISCV::RVVBitsPerBlock)
205 RC = &RISCV::VRM4RegClass;
206 else if (Size == 8 * RISCV::RVVBitsPerBlock)
207 RC = &RISCV::VRM8RegClass;
208 else
209 llvm_unreachable("Unexpected size");
210
211 addRegisterClass(VT, RC);
212 };
213
214 for (MVT VT : BoolVecVTs)
215 addRegClassForRVV(VT);
216 for (MVT VT : IntVecVTs) {
217 if (VT.getVectorElementType() == MVT::i64 &&
218 !Subtarget.hasVInstructionsI64())
219 continue;
220 addRegClassForRVV(VT);
221 }
222
223 if (Subtarget.hasVInstructionsF16Minimal() ||
224 Subtarget.hasVendorXAndesVPackFPH())
225 for (MVT VT : F16VecVTs)
226 addRegClassForRVV(VT);
227
228 if (Subtarget.hasVInstructionsBF16Minimal() ||
229 Subtarget.hasVendorXAndesVBFHCvt())
230 for (MVT VT : BF16VecVTs)
231 addRegClassForRVV(VT);
232
233 if (Subtarget.hasVInstructionsF32())
234 for (MVT VT : F32VecVTs)
235 addRegClassForRVV(VT);
236
237 if (Subtarget.hasVInstructionsF64())
238 for (MVT VT : F64VecVTs)
239 addRegClassForRVV(VT);
240
241 if (Subtarget.useRVVForFixedLengthVectors()) {
242 auto addRegClassForFixedVectors = [this](MVT VT) {
243 MVT ContainerVT = getContainerForFixedLengthVector(VT);
244 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
245 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
246 addRegisterClass(VT, TRI.getRegClass(RCID));
247 };
248 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
249 if (useRVVForFixedLengthVectorVT(VT))
250 addRegClassForFixedVectors(VT);
251
252 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
253 if (useRVVForFixedLengthVectorVT(VT))
254 addRegClassForFixedVectors(VT);
255 }
256
257 addRegisterClass(MVT::riscv_nxv1i8x2, &RISCV::VRN2M1RegClass);
258 addRegisterClass(MVT::riscv_nxv1i8x3, &RISCV::VRN3M1RegClass);
259 addRegisterClass(MVT::riscv_nxv1i8x4, &RISCV::VRN4M1RegClass);
260 addRegisterClass(MVT::riscv_nxv1i8x5, &RISCV::VRN5M1RegClass);
261 addRegisterClass(MVT::riscv_nxv1i8x6, &RISCV::VRN6M1RegClass);
262 addRegisterClass(MVT::riscv_nxv1i8x7, &RISCV::VRN7M1RegClass);
263 addRegisterClass(MVT::riscv_nxv1i8x8, &RISCV::VRN8M1RegClass);
264 addRegisterClass(MVT::riscv_nxv2i8x2, &RISCV::VRN2M1RegClass);
265 addRegisterClass(MVT::riscv_nxv2i8x3, &RISCV::VRN3M1RegClass);
266 addRegisterClass(MVT::riscv_nxv2i8x4, &RISCV::VRN4M1RegClass);
267 addRegisterClass(MVT::riscv_nxv2i8x5, &RISCV::VRN5M1RegClass);
268 addRegisterClass(MVT::riscv_nxv2i8x6, &RISCV::VRN6M1RegClass);
269 addRegisterClass(MVT::riscv_nxv2i8x7, &RISCV::VRN7M1RegClass);
270 addRegisterClass(MVT::riscv_nxv2i8x8, &RISCV::VRN8M1RegClass);
271 addRegisterClass(MVT::riscv_nxv4i8x2, &RISCV::VRN2M1RegClass);
272 addRegisterClass(MVT::riscv_nxv4i8x3, &RISCV::VRN3M1RegClass);
273 addRegisterClass(MVT::riscv_nxv4i8x4, &RISCV::VRN4M1RegClass);
274 addRegisterClass(MVT::riscv_nxv4i8x5, &RISCV::VRN5M1RegClass);
275 addRegisterClass(MVT::riscv_nxv4i8x6, &RISCV::VRN6M1RegClass);
276 addRegisterClass(MVT::riscv_nxv4i8x7, &RISCV::VRN7M1RegClass);
277 addRegisterClass(MVT::riscv_nxv4i8x8, &RISCV::VRN8M1RegClass);
278 addRegisterClass(MVT::riscv_nxv8i8x2, &RISCV::VRN2M1RegClass);
279 addRegisterClass(MVT::riscv_nxv8i8x3, &RISCV::VRN3M1RegClass);
280 addRegisterClass(MVT::riscv_nxv8i8x4, &RISCV::VRN4M1RegClass);
281 addRegisterClass(MVT::riscv_nxv8i8x5, &RISCV::VRN5M1RegClass);
282 addRegisterClass(MVT::riscv_nxv8i8x6, &RISCV::VRN6M1RegClass);
283 addRegisterClass(MVT::riscv_nxv8i8x7, &RISCV::VRN7M1RegClass);
284 addRegisterClass(MVT::riscv_nxv8i8x8, &RISCV::VRN8M1RegClass);
285 addRegisterClass(MVT::riscv_nxv16i8x2, &RISCV::VRN2M2RegClass);
286 addRegisterClass(MVT::riscv_nxv16i8x3, &RISCV::VRN3M2RegClass);
287 addRegisterClass(MVT::riscv_nxv16i8x4, &RISCV::VRN4M2RegClass);
288 addRegisterClass(MVT::riscv_nxv32i8x2, &RISCV::VRN2M4RegClass);
289 }
290
291 // fixed vector is stored in GPRs for P extension packed operations
292 if (Subtarget.hasStdExtP()) {
293 if (Subtarget.is64Bit()) {
294 addRegisterClass(MVT::v2i32, &RISCV::GPRRegClass);
295 addRegisterClass(MVT::v4i16, &RISCV::GPRRegClass);
296 addRegisterClass(MVT::v8i8, &RISCV::GPRRegClass);
297 } else {
298 addRegisterClass(MVT::v2i16, &RISCV::GPRRegClass);
299 addRegisterClass(MVT::v4i8, &RISCV::GPRRegClass);
300
301 addRegisterClass(MVT::v2i32, &RISCV::GPRPairRegClass);
302 addRegisterClass(MVT::v4i16, &RISCV::GPRPairRegClass);
303 addRegisterClass(MVT::v8i8, &RISCV::GPRPairRegClass);
304 }
305 }
306
307 // Compute derived properties from the register classes.
308 computeRegisterProperties(STI.getRegisterInfo());
309
310 setStackPointerRegisterToSaveRestore(RISCV::X2);
311
312 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, XLenVT,
313 MVT::i1, Promote);
314 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
315 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, MVT::i32,
316 MVT::i1, Promote);
317
318 // TODO: add all necessary setOperationAction calls.
319 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Custom);
320
321 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
322 setOperationAction(ISD::BR_CC, XLenVT, Expand);
323 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
324 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
325
326 setCondCodeAction(ISD::SETGT, XLenVT, Custom);
327 setCondCodeAction(ISD::SETGE, XLenVT, Expand);
328 setCondCodeAction(ISD::SETUGT, XLenVT, Custom);
329 setCondCodeAction(ISD::SETUGE, XLenVT, Expand);
330 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
331 setCondCodeAction(ISD::SETULE, XLenVT, Expand);
332 setCondCodeAction(ISD::SETLE, XLenVT, Expand);
333 }
334
335 setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand);
336
337 setOperationAction(ISD::VASTART, MVT::Other, Custom);
338 setOperationAction({ISD::VAARG, ISD::VACOPY, ISD::VAEND}, MVT::Other, Expand);
339
340 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
341 !Subtarget.hasVendorXAndesPerf())
342 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
343
344 setOperationAction(ISD::EH_DWARF_CFA, MVT::i32, Custom);
345
346 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
347 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
348 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
349 setOperationAction(ISD::SIGN_EXTEND_INREG, {MVT::i8, MVT::i16}, Expand);
350
351 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
352 setOperationAction(ISD::LOAD, MVT::i64, Custom);
353 setOperationAction(ISD::STORE, MVT::i64, Custom);
354 }
355
356 if (Subtarget.is64Bit()) {
357 setOperationAction(ISD::EH_DWARF_CFA, MVT::i64, Custom);
358
359 setOperationAction(ISD::LOAD, MVT::i32, Custom);
360 setOperationAction({ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
361 MVT::i32, Custom);
362 setOperationAction({ISD::UADDO, ISD::USUBO}, MVT::i32, Custom);
363 setOperationAction({ISD::SADDO, ISD::SSUBO}, MVT::i32, Custom);
364 } else if (Subtarget.hasStdExtP()) {
365 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
366 setOperationAction({ISD::ADD, ISD::SUB}, MVT::i64, Custom);
367 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, MVT::i64, Custom);
368 }
369 if (!Subtarget.hasStdExtZmmul()) {
370 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, XLenVT, Expand);
371 } else if (Subtarget.is64Bit()) {
372 setOperationAction(ISD::MUL, MVT::i128, Custom);
373 setOperationAction(ISD::MUL, MVT::i32, Custom);
374 } else {
375 setOperationAction(ISD::MUL, MVT::i64, Custom);
376 }
377
378 if (!Subtarget.hasStdExtM()) {
379 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, XLenVT,
380 Expand);
381 } else if (Subtarget.is64Bit()) {
382 setOperationAction({ISD::SDIV, ISD::UDIV, ISD::UREM},
383 {MVT::i8, MVT::i16, MVT::i32}, Custom);
384 }
385
386 setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, XLenVT, Expand);
387
388 // On RV32, the P extension has a WMUL(U) instruction we can use for
389 // (S/U)MUL_LOHI.
390 // FIXME: Does P imply Zmmul?
391 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
392 Subtarget.is64Bit())
393 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, XLenVT, Expand);
394
395 setOperationAction({ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, XLenVT,
396 Custom);
397
398 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
399 if (Subtarget.is64Bit())
400 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
401 } else if (Subtarget.hasVendorXTHeadBb()) {
402 if (Subtarget.is64Bit())
403 setOperationAction({ISD::ROTL, ISD::ROTR}, MVT::i32, Custom);
404 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Custom);
405 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
406 setOperationAction(ISD::ROTL, XLenVT, Expand);
407 } else {
408 setOperationAction({ISD::ROTL, ISD::ROTR}, XLenVT, Expand);
409 }
410
411 if (Subtarget.hasStdExtP())
412 setOperationAction({ISD::FSHL, ISD::FSHR}, XLenVT, Legal);
413
414 setOperationAction(ISD::BSWAP, XLenVT,
415 Subtarget.hasREV8Like() ? Legal : Expand);
416
417 if (Subtarget.hasREVLike()) {
418 setOperationAction(ISD::BITREVERSE, XLenVT, Legal);
419 } else {
420 // Zbkb can use rev8+brev8 to implement bitreverse.
421 setOperationAction(ISD::BITREVERSE, XLenVT,
422 Subtarget.hasStdExtZbkb() ? Custom : Expand);
423 if (Subtarget.hasStdExtZbkb())
424 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
425 }
426
427 if (Subtarget.hasStdExtZbb() ||
428 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
429 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, XLenVT,
430 Legal);
431 }
432
433 if (Subtarget.hasCTZLike()) {
434 if (Subtarget.is64Bit())
435 setOperationAction({ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, MVT::i32, Custom);
436 } else {
437 setOperationAction(ISD::CTTZ, XLenVT, Expand);
438 }
439
440 if (!Subtarget.hasCPOPLike()) {
441 // TODO: These should be set to LibCall, but this currently breaks
442 // the Linux kernel build. See #101786. Lacks i128 tests, too.
443 if (Subtarget.is64Bit())
444 setOperationAction(ISD::CTPOP, MVT::i128, Expand);
445 else
446 setOperationAction(ISD::CTPOP, MVT::i32, Expand);
447 setOperationAction(ISD::CTPOP, MVT::i64, Expand);
448 }
449
450 if (Subtarget.hasCLZLike()) {
451 // We need the custom lowering to make sure that the resulting sequence
452 // for the 32bit case is efficient on 64bit targets.
453 // Use default promotion for i32 without Zbb.
454 if (Subtarget.is64Bit() &&
455 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
456 setOperationAction({ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, MVT::i32, Custom);
457 } else {
458 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
459 setOperationAction(ISD::CTLZ_ZERO_POISON, XLenVT, Legal);
460 setOperationAction(ISD::CTLZ, XLenVT, Expand);
461 }
462
463 if (Subtarget.hasStdExtP()) {
464 setOperationAction(ISD::CTLS, XLenVT, Legal);
465 if (Subtarget.is64Bit())
466 setOperationAction(ISD::CTLS, MVT::i32, Custom);
467 }
468
469 if (Subtarget.hasStdExtP() ||
470 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
471 setOperationAction(ISD::ABS, XLenVT, Legal);
472 if (Subtarget.is64Bit())
473 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
474 } else if (Subtarget.hasShortForwardBranchIALU()) {
475 // We can use PseudoCCSUB to implement ABS.
476 setOperationAction(ISD::ABS, XLenVT, Legal);
477 } else if (Subtarget.is64Bit()) {
478 setOperationAction({ISD::ABS, ISD::ABS_MIN_POISON}, MVT::i32, Custom);
479 }
480
481 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
482 setOperationAction(ISD::SELECT, XLenVT, Custom);
483
484 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
485 !Subtarget.is64Bit()) {
486 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
487 MVT::i32, Legal);
488 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
489 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
490 MVT::i32, Custom);
491 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
492 setOperationAction({ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
493 MVT::i32, Custom);
494 }
495
496 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
497 !Subtarget.is64Bit()) {
498 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
499 // pssha.w/psshl.w and extracting.
500 setOperationAction(ISD::SSHLSAT, MVT::i32, Legal);
501 setOperationAction(ISD::USHLSAT, MVT::i32, Legal);
502 }
503
504 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
505 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
506 // paadd.w, paaddu.w and extracting.
507 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, MVT::i32, Legal);
508 }
509
510 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc())
511 setOperationAction({ISD::CLMUL, ISD::CLMULH}, XLenVT, Legal);
512 if (Subtarget.hasStdExtZbc())
513 setOperationAction(ISD::CLMULR, XLenVT, Legal);
514
515 static const unsigned FPLegalNodeTypes[] = {
516 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
517 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
518 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
519 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
520 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
521 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
522 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
523
524 static const ISD::CondCode FPCCToExpand[] = {
525 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
526 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
527 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
528
529 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
530 ISD::FPOW};
531 static const unsigned FPOpToLibCall[] = {ISD::FREM};
532
533 static const unsigned FPRndMode[] = {
534 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
535 ISD::FROUNDEVEN};
536
537 static const unsigned ZfhminZfbfminPromoteOps[] = {
538 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUM,
539 ISD::FMAXIMUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM,
540 ISD::FADD, ISD::FSUB, ISD::FMUL,
541 ISD::FMA, ISD::FDIV, ISD::FSQRT,
542 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
543 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
544 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
545 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC,
546 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN,
547 ISD::FCANONICALIZE};
548
549 if (Subtarget.hasStdExtP()) {
550 static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
551 static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
552 ArrayRef<MVT> VTs;
553 if (Subtarget.is64Bit()) {
554 VTs = P64VecVTs;
555 // There's no instruction for vector shamt in P extension so we unroll to
556 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
557 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
558 // widen for those operations that will be unrolled.
559 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA},
560 {MVT::v2i16, MVT::v4i8}, Custom);
561 setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::v2i16, MVT::v4i8},
562 Custom);
563 } else {
564 VTs = P32VecVTs;
565 }
566 // By default everything must be expanded.
567 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
568 setOperationAction(Op, VTs, Expand);
569
570 for (MVT VT : VTs) {
571 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
572 setTruncStoreAction(VT, OtherVT, Expand);
573 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
574 OtherVT, Expand);
575 }
576 }
577
578 setOperationAction({ISD::LOAD, ISD::STORE}, VTs, Legal);
579 setOperationAction({ISD::ADD, ISD::SUB}, VTs, Legal);
580 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VTs, Legal);
581 setOperationAction(ISD::UADDSAT, VTs, Legal);
582 setOperationAction(ISD::SADDSAT, VTs, Legal);
583 setOperationAction(ISD::USUBSAT, VTs, Legal);
584 setOperationAction(ISD::SSUBSAT, VTs, Legal);
585 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Legal);
586 setOperationAction(ISD::BITREVERSE, VTs, Legal);
587 setOperationAction(ISD::VECTOR_SHUFFLE, VTs, Custom);
588 setOperationAction(ISD::VECTOR_REVERSE, VTs, Legal);
589 for (MVT VT : VTs) {
590 if (VT != MVT::v2i32)
591 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Legal);
592 if (VT.getVectorElementType() != MVT::i8) {
593 setOperationAction(ISD::SSHLSAT, VT, Custom);
594 setOperationAction(ISD::BSWAP, VT, Legal);
595 }
596 }
597 setOperationAction(ISD::SPLAT_VECTOR, VTs, Legal);
598 setOperationAction(ISD::BUILD_VECTOR, VTs, Legal);
599 setOperationAction(ISD::SCALAR_TO_VECTOR, VTs, Legal);
600 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Custom);
601 setOperationAction(ISD::BITCAST, VTs, Custom);
602 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
603 Custom);
604 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
605 Legal);
606 setOperationAction(ISD::SELECT, VTs, Custom);
607 setOperationAction(ISD::VSELECT, VTs, Legal);
608 setOperationAction(ISD::SETCC, VTs, Legal);
609 setCondCodeAction(
610 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
611 Expand);
612 setCondCodeAction({ISD::SETNE, ISD::SETGT}, VTs, Custom);
613
614 if (!Subtarget.is64Bit())
615 setOperationAction(ISD::BUILD_VECTOR, MVT::v4i8, Custom);
616
617 // P extension vector comparisons produce all 1s for true, all 0s for false
618 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
619
620 if (!Subtarget.is64Bit()) {
621 // By default everything must be expanded.
622 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
623 setOperationAction(Op, P64VecVTs, Expand);
624
625 for (MVT VT : P64VecVTs) {
626 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
627 setTruncStoreAction(VT, OtherVT, Expand);
628 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
629 OtherVT, Expand);
630 }
631 }
632
633 setOperationAction({ISD::LOAD, ISD::STORE}, P64VecVTs, Custom);
634 setOperationAction(ISD::BITCAST, P64VecVTs, Custom);
635 setOperationAction({ISD::ADD, ISD::SUB}, P64VecVTs, Legal);
636 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, {MVT::v4i16, MVT::v8i8},
637 Custom);
638 setOperationAction(
639 {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, P64VecVTs,
640 Legal);
641 setOperationAction(ISD::INTRINSIC_WO_CHAIN, P64VecVTs, Legal);
642 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
643 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU}, P64VecVTs, Legal);
644 setOperationAction({ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
645 P64VecVTs, Legal);
646 setOperationAction({ISD::ABS, ISD::ABDS, ISD::ABDU},
647 {MVT::v4i16, MVT::v8i8}, Legal);
648 setOperationAction({ISD::SHL, ISD::SRL, ISD::SRA}, P64VecVTs, Custom);
649 setOperationAction(ISD::SSHLSAT, {MVT::v2i32, MVT::v4i16}, Custom);
650 setOperationAction(ISD::BSWAP, MVT::v4i16, Legal);
651 setOperationAction(ISD::BITREVERSE, {MVT::v4i16, MVT::v8i8}, Legal);
652 setOperationAction(ISD::VECTOR_SHUFFLE, P64VecVTs, Custom);
653 setOperationAction(ISD::VECTOR_REVERSE, P64VecVTs, Custom);
654 setOperationAction(ISD::SPLAT_VECTOR, P64VecVTs, Legal);
655 setOperationAction(ISD::BUILD_VECTOR, P64VecVTs, Legal);
656 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v2i32, Legal);
657 setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
658 {MVT::v4i16, MVT::v8i8}, Custom);
659 setOperationAction(ISD::CONCAT_VECTORS, {MVT::v4i16, MVT::v8i8}, Legal);
660 setOperationAction(ISD::EXTRACT_SUBVECTOR, {MVT::v2i16, MVT::v4i8},
661 Legal);
662 setOperationAction({ISD::SELECT, ISD::VSELECT}, {MVT::v4i16, MVT::v8i8},
663 Custom);
664 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU},
665 {MVT::v4i16, MVT::v8i8}, Custom);
666 setOperationAction(ISD::MUL, P32VecVTs, Custom);
667 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v4i8, Custom);
668 setOperationAction({ISD::MULHS, ISD::MULHU}, MVT::v2i16, Legal);
669 setOperationAction({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
670 {MVT::v4i16, MVT::v2i32}, Legal);
671 setOperationAction(ISD::TRUNCATE, {MVT::v4i8, MVT::v2i16}, Legal);
672 setOperationAction(ISD::SETCC, P64VecVTs, Legal);
673 setCondCodeAction(
674 {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
675 P64VecVTs, Expand);
676 setCondCodeAction({ISD::SETNE, ISD::SETGT}, P64VecVTs, Custom);
677 } else {
678 setOperationAction({ISD::MUL, ISD::MULHS, ISD::MULHU}, P64VecVTs, Legal);
679 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG,
680 {MVT::v4i16, MVT::v2i32}, Legal);
681 setOperationAction(ISD::ANY_EXTEND_VECTOR_INREG, {MVT::v4i16, MVT::v2i32},
682 Custom);
683 // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
684 setOperationAction(ISD::SIGN_EXTEND_INREG,
685 {MVT::v2i16, MVT::v4i8, MVT::v2i32, MVT::v4i16},
686 Legal);
687 }
688 }
689
690 if (Subtarget.hasStdExtZfbfmin()) {
691 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
692 setOperationAction(ISD::ConstantFP, MVT::bf16, Expand);
693 setOperationAction(ISD::SELECT_CC, MVT::bf16, Expand);
694 setOperationAction(ISD::SELECT, MVT::bf16, Custom);
695 setOperationAction(ISD::BR_CC, MVT::bf16, Expand);
696 setOperationAction(ZfhminZfbfminPromoteOps, MVT::bf16, Promote);
697 setOperationAction(ISD::FREM, MVT::bf16, Promote);
698 setOperationAction(ISD::FABS, MVT::bf16, Custom);
699 setOperationAction(ISD::FNEG, MVT::bf16, Custom);
700 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
701 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
702 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
703 }
704
705 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
706 if (Subtarget.hasStdExtZfhOrZhinx()) {
707 setOperationAction(FPLegalNodeTypes, MVT::f16, Legal);
708 setOperationAction(FPRndMode, MVT::f16,
709 Subtarget.hasStdExtZfa() ? Legal : Custom);
710 setOperationAction(ISD::IS_FPCLASS, MVT::f16, Custom);
711 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16,
712 Subtarget.hasStdExtZfa() ? Legal : Custom);
713 if (Subtarget.hasStdExtZfa())
714 setOperationAction(ISD::ConstantFP, MVT::f16, Custom);
715 } else {
716 setOperationAction(ZfhminZfbfminPromoteOps, MVT::f16, Promote);
717 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Promote);
718 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
719 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
720 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
721 setOperationAction(Op, MVT::f16, Custom);
722 setOperationAction(ISD::FABS, MVT::f16, Custom);
723 setOperationAction(ISD::FNEG, MVT::f16, Custom);
724 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
725 setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, XLenVT, Custom);
726 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, XLenVT, Custom);
727 }
728
729 if (!Subtarget.hasStdExtD()) {
730 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
731 // instruction runs into double rounding issues, so this is wrong.
732 // Normally we'd use an f64 fma, but without the D extension the f64 type
733 // is not legal. This should probably be a libcall.
734 AddPromotedToType(ISD::FMA, MVT::f16, MVT::f32);
735 AddPromotedToType(ISD::STRICT_FMA, MVT::f16, MVT::f32);
736 }
737
738 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
739
740 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Legal);
741 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
742 setCondCodeAction(FPCCToExpand, MVT::f16, Expand);
743 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
744 setOperationAction(ISD::SELECT, MVT::f16, Custom);
745 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
746
747 setOperationAction(
748 ISD::FNEARBYINT, MVT::f16,
749 Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
750 setOperationAction({ISD::FREM, ISD::FPOW, ISD::FPOWI,
751 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
752 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
753 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
754 MVT::f16, Promote);
755
756 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
757 // complete support for all operations in LegalizeDAG.
758 setOperationAction({ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
759 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
760 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
761 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
762 MVT::f16, Promote);
763
764 // We need to custom promote this.
765 if (Subtarget.is64Bit())
766 setOperationAction(ISD::FPOWI, MVT::i32, Custom);
767 }
768
769 if (Subtarget.hasStdExtFOrZfinx()) {
770 setOperationAction(FPLegalNodeTypes, MVT::f32, Legal);
771 setOperationAction(FPRndMode, MVT::f32,
772 Subtarget.hasStdExtZfa() ? Legal : Custom);
773 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
774 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
775 setOperationAction(ISD::SELECT, MVT::f32, Custom);
776 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
777 setOperationAction(FPOpToExpand, MVT::f32, Expand);
778 setOperationAction(FPOpToLibCall, MVT::f32, LibCall);
779 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
780 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
781 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
782 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
783 setOperationAction(ISD::IS_FPCLASS, MVT::f32, Custom);
784 setOperationAction(ISD::BF16_TO_FP, MVT::f32, Custom);
785 setOperationAction(ISD::FP_TO_BF16, MVT::f32,
786 Subtarget.isSoftFPABI() ? LibCall : Custom);
787 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Custom);
788 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Custom);
789 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f32, Custom);
790 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f32, Custom);
791
792 if (Subtarget.hasStdExtZfa()) {
793 setOperationAction(ISD::ConstantFP, MVT::f32, Custom);
794 setOperationAction(ISD::FNEARBYINT, MVT::f32, Legal);
795 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal);
796 } else {
797 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Custom);
798 }
799 }
800
801 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
802 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
803
804 if (Subtarget.hasStdExtDOrZdinx()) {
805 setOperationAction(FPLegalNodeTypes, MVT::f64, Legal);
806
807 if (!Subtarget.is64Bit())
808 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
809
810 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
811 !Subtarget.is64Bit()) {
812 setOperationAction(ISD::LOAD, MVT::f64, Custom);
813 setOperationAction(ISD::STORE, MVT::f64, Custom);
814 }
815
816 if (Subtarget.hasStdExtZfa()) {
817 setOperationAction(ISD::ConstantFP, MVT::f64, Custom);
818 setOperationAction(FPRndMode, MVT::f64, Legal);
819 setOperationAction(ISD::FNEARBYINT, MVT::f64, Legal);
820 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Legal);
821 } else {
822 if (Subtarget.is64Bit())
823 setOperationAction(FPRndMode, MVT::f64, Custom);
824
825 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f64, Custom);
826 }
827
828 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
829 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
830 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
831 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
832 setOperationAction(ISD::SELECT, MVT::f64, Custom);
833 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
834 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
835 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
836 setOperationAction(FPOpToExpand, MVT::f64, Expand);
837 setOperationAction(FPOpToLibCall, MVT::f64, LibCall);
838 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
839 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
840 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
841 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
842 setOperationAction(ISD::IS_FPCLASS, MVT::f64, Custom);
843 setOperationAction(ISD::BF16_TO_FP, MVT::f64, Custom);
844 setOperationAction(ISD::FP_TO_BF16, MVT::f64,
845 Subtarget.isSoftFPABI() ? LibCall : Custom);
846 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom);
847 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
848 setOperationAction(ISD::STRICT_FP_TO_FP16, MVT::f64, Custom);
849 setOperationAction(ISD::STRICT_FP16_TO_FP, MVT::f64, Expand);
850 }
851
852 if (Subtarget.is64Bit()) {
853 setOperationAction({ISD::FP_TO_UINT, ISD::FP_TO_SINT,
854 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
855 MVT::i32, Custom);
856 setOperationAction(ISD::LROUND, MVT::i32, Custom);
857 }
858
859 if (Subtarget.hasStdExtFOrZfinx()) {
860 setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, XLenVT,
861 Custom);
862
863 // f16/bf16 require custom handling.
864 setOperationAction({ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, XLenVT,
865 Custom);
866 setOperationAction({ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, XLenVT,
867 Custom);
868
869 setOperationAction(ISD::GET_ROUNDING, XLenVT, Custom);
870 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
871 setOperationAction(ISD::GET_FPENV, XLenVT, Custom);
872 setOperationAction(ISD::SET_FPENV, XLenVT, Custom);
873 setOperationAction(ISD::RESET_FPENV, MVT::Other, Custom);
874 setOperationAction(ISD::GET_FPMODE, XLenVT, Custom);
875 setOperationAction(ISD::SET_FPMODE, XLenVT, Custom);
876 setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
877 }
878
879 setOperationAction({ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
880 ISD::JumpTable},
881 XLenVT, Custom);
882
883 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
884
885 if (Subtarget.is64Bit())
886 setOperationAction(ISD::Constant, MVT::i64, Custom);
887
888 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
889 // Unfortunately this can't be determined just from the ISA naming string.
890 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
891 Subtarget.is64Bit() ? Legal : Custom);
892 setOperationAction(ISD::READSTEADYCOUNTER, MVT::i64,
893 Subtarget.is64Bit() ? Legal : Custom);
894
895 if (Subtarget.is64Bit()) {
896 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
897 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
898 }
899
900 setOperationAction({ISD::TRAP, ISD::DEBUGTRAP}, MVT::Other, Legal);
901 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
902 if (Subtarget.is64Bit())
903 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
904
905 if (Subtarget.hasVendorXMIPSCBOP())
906 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
907 else
908 setOperationAction(ISD::PREFETCH, MVT::Other, Legal);
909
910 if (Subtarget.hasStdExtZalrsc()) {
911 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
912 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
913 setMinCmpXchgSizeInBits(8);
914 else
915 setMinCmpXchgSizeInBits(32);
916 } else if (Subtarget.hasForcedAtomics()) {
917 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
918 } else {
919 setMaxAtomicSizeInBitsSupported(0);
920 }
921
922 setOperationAction(ISD::ATOMIC_FENCE, MVT::Other, Custom);
923
924 setBooleanContents(ZeroOrOneBooleanContent);
925
926 if (getTargetMachine().getTargetTriple().isOSLinux()) {
927 // Custom lowering of llvm.clear_cache.
928 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Custom);
929 }
930
931 if (Subtarget.hasVInstructions()) {
932 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
933
934 setOperationAction(ISD::VSCALE, XLenVT, Custom);
935
936 // RVV intrinsics may have illegal operands.
937 // We also need to custom legalize vmv.x.s.
938 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
939 ISD::INTRINSIC_VOID},
940 {MVT::i8, MVT::i16}, Custom);
941 if (Subtarget.is64Bit())
942 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
943 MVT::i32, Custom);
944 else
945 setOperationAction({ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
946 MVT::i64, Custom);
947
948 setOperationAction({ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
949 MVT::Other, Custom);
950
951 static const unsigned IntegerVPOps[] = {
952 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
953 ISD::VP_UREM, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
954 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
955 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
956 ISD::VP_MERGE,
957 ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
958 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON};
959
960 static const unsigned FloatingPointVPOps[] = {
961 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
962 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
963 ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
964
965 static const unsigned IntegerVecReduceOps[] = {
966 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
967 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
968 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
969
970 static const unsigned FloatingPointVecReduceOps[] = {
971 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
972 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
973
974 static const unsigned FloatingPointLibCallOps[] = {
975 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
976 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
977
978 if (!Subtarget.is64Bit()) {
979 // We must custom-lower certain vXi64 operations on RV32 due to the vector
980 // element type being illegal.
981 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
982 MVT::i64, Custom);
983
984 setOperationAction(IntegerVecReduceOps, MVT::i64, Custom);
985
986 setOperationAction({ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
987 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
988 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
989 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
990 MVT::i64, Custom);
991 }
992
993 for (MVT VT : BoolVecVTs) {
994 if (!isTypeLegal(VT))
995 continue;
996
997 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
998
999 // Mask VTs are custom-expanded into a series of standard nodes
1000 setOperationAction({ISD::TRUNCATE, ISD::CONCAT_VECTORS,
1001 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1002 ISD::SCALAR_TO_VECTOR},
1003 VT, Custom);
1004
1005 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1006 Custom);
1007
1008 setOperationAction(ISD::SELECT, VT, Custom);
1009 setOperationAction({ISD::SELECT_CC, ISD::VSELECT}, VT,
1010 Expand);
1011 setOperationAction(ISD::VP_MERGE, VT, Custom);
1012
1013 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
1014 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
1015 VT, Custom);
1016
1017 setOperationAction(
1018 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1019 Custom);
1020
1021 setOperationAction(
1022 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1023 Custom);
1024
1025 // RVV has native int->float & float->int conversions where the
1026 // element type sizes are within one power-of-two of each other. Any
1027 // wider distances between type sizes have to be lowered as sequences
1028 // which progressively narrow the gap in stages.
1029 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1030 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1031 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1032 ISD::STRICT_FP_TO_UINT},
1033 VT, Custom);
1034 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1035 Custom);
1036
1037 // Expand all extending loads to types larger than this, and truncating
1038 // stores from types larger than this.
1039 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1040 setTruncStoreAction(VT, OtherVT, Expand);
1041 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1042 OtherVT, Expand);
1043 }
1044
1045 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1046 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1047
1048 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
1049
1050 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1051 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1052
1053 setOperationPromotedToType(
1054 {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, VT,
1055 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount()));
1056 }
1057
1058 for (MVT VT : IntVecVTs) {
1059 if (!isTypeLegal(VT))
1060 continue;
1061
1062 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1063 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1064
1065 // Vectors implement MULHS/MULHU.
1066 setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand);
1067
1068 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1069 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1070 setOperationAction({ISD::MULHU, ISD::MULHS}, VT, Expand);
1071
1072 setOperationAction({ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1073 Legal);
1074
1075 if (Subtarget.hasStdExtZvabd()) {
1076 setOperationAction(ISD::ABS, VT, Legal);
1077 // Only SEW=8/16 are supported in Zvabd.
1078 if (VT.getVectorElementType() == MVT::i8 ||
1079 VT.getVectorElementType() == MVT::i16)
1080 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Legal);
1081 else
1082 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1083 } else
1084 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1085
1086 // Custom-lower extensions and truncations from/to mask types.
1087 setOperationAction({ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1088 VT, Custom);
1089
1090 // RVV has native int->float & float->int conversions where the
1091 // element type sizes are within one power-of-two of each other. Any
1092 // wider distances between type sizes have to be lowered as sequences
1093 // which progressively narrow the gap in stages.
1094 setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1095 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1096 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1097 ISD::STRICT_FP_TO_UINT},
1098 VT, Custom);
1099 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1100 Custom);
1101 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1102 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1103 ISD::SSUBSAT, ISD::USUBSAT},
1104 VT, Legal);
1105
1106 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1107 // nodes which truncate by one power of two at a time.
1108 setOperationAction(
1109 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1110 Custom);
1111
1112 // Custom-lower insert/extract operations to simplify patterns.
1113 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1114 Custom);
1115
1116 // Custom-lower reduction operations to set up the corresponding custom
1117 // nodes' operands.
1118 setOperationAction(IntegerVecReduceOps, VT, Custom);
1119
1120 setOperationAction(IntegerVPOps, VT, Custom);
1121
1122 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1123
1124 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1125 VT, Custom);
1126
1127 setOperationAction(
1128 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1129 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1130 VT, Custom);
1131 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1132
1133 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1134 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1135 VT, Custom);
1136
1137 setOperationAction(ISD::SELECT, VT, Custom);
1138 setOperationAction(ISD::SELECT_CC, VT, Expand);
1139
1140 setOperationAction({ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Custom);
1141
1142 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1143 setTruncStoreAction(VT, OtherVT, Expand);
1144 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1145 OtherVT, Expand);
1146 }
1147
1148 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1149 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1150
1151 setOperationAction({ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1152 VT, Custom);
1153
1154 if (Subtarget.hasStdExtZvkb()) {
1155 setOperationAction(ISD::BSWAP, VT, Legal);
1156 } else {
1157 setOperationAction(ISD::BSWAP, VT, Expand);
1158 setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand);
1159 }
1160
1161 if (Subtarget.hasStdExtZvbb()) {
1162 setOperationAction(ISD::BITREVERSE, VT, Legal);
1163 } else {
1164 setOperationAction(ISD::BITREVERSE, VT, Expand);
1165 setOperationAction({ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Expand);
1166
1167 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1168 // range of f32.
1169 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1170 if (isTypeLegal(FloatVT)) {
1171 setOperationAction(
1172 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1173 Custom);
1174 }
1175 }
1176
1177 if (VT.getVectorElementType() == MVT::i64) {
1178 if (Subtarget.hasStdExtZvbc())
1179 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1180 } else {
1181 if (Subtarget.hasStdExtZvbc32e()) {
1182 setOperationAction({ISD::CLMUL, ISD::CLMULH}, VT, Legal);
1183 } else if (Subtarget.hasStdExtZvbc()) {
1184 // Promote to i64 if the lmul is small enough.
1185 // FIXME: Split if necessary to widen.
1186 // FIXME: Promote clmulh directly without legalizing to clmul first.
1187 MVT I64VecVT = MVT::getVectorVT(MVT::i64, VT.getVectorElementCount());
1188 if (isTypeLegal(I64VecVT))
1189 setOperationAction(ISD::CLMUL, VT, Custom);
1190 }
1191 }
1192
1193 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1194 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
1195 ISD::MASKED_SREM},
1196 VT, Legal);
1197 }
1198
1199 for (MVT VT : VecTupleVTs) {
1200 if (!isTypeLegal(VT))
1201 continue;
1202
1203 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1204 }
1205
1206 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1207 // but no other unordered comparisons, and supports all ordered comparisons
1208 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1209 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1210 // and we pattern-match those back to the "original", swapping operands once
1211 // more. This way we catch both operations and both "vf" and "fv" forms with
1212 // fewer patterns.
1213 static const ISD::CondCode VFPCCToExpand[] = {
1214 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1215 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1216 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1217 };
1218
1219 // TODO: support more ops.
1220 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1221 ISD::FMINNUM,
1222 ISD::FMAXNUM,
1223 ISD::FMINIMUMNUM,
1224 ISD::FMAXIMUMNUM,
1225 ISD::FADD,
1226 ISD::FSUB,
1227 ISD::FMUL,
1228 ISD::FMA,
1229 ISD::FDIV,
1230 ISD::FSQRT,
1231 ISD::FCEIL,
1232 ISD::FTRUNC,
1233 ISD::FFLOOR,
1234 ISD::FROUND,
1235 ISD::FROUNDEVEN,
1236 ISD::FRINT,
1237 ISD::FNEARBYINT,
1238 ISD::IS_FPCLASS,
1239 ISD::SETCC,
1240 ISD::FMAXIMUM,
1241 ISD::FMINIMUM,
1242 ISD::STRICT_FADD,
1243 ISD::STRICT_FSUB,
1244 ISD::STRICT_FMUL,
1245 ISD::STRICT_FDIV,
1246 ISD::STRICT_FSQRT,
1247 ISD::STRICT_FMA,
1248 ISD::VECREDUCE_FADD,
1249 ISD::VECREDUCE_FMIN,
1250 ISD::VECREDUCE_FMAX,
1251 ISD::VECREDUCE_FMINIMUM,
1252 ISD::VECREDUCE_FMAXIMUM,
1253 ISD::FCANONICALIZE};
1254
1255 // TODO: Make more of these ops legal.
1256 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1257 ISD::FSQRT,
1258 ISD::FCEIL,
1259 ISD::FTRUNC,
1260 ISD::FFLOOR,
1261 ISD::FROUND,
1262 ISD::FROUNDEVEN,
1263 ISD::FRINT,
1264 ISD::FNEARBYINT,
1265 ISD::STRICT_FDIV,
1266 ISD::STRICT_FSQRT,
1267 ISD::VECREDUCE_FADD,
1268 ISD::VECREDUCE_FMIN,
1269 ISD::VECREDUCE_FMAX,
1270 ISD::VECREDUCE_FMINIMUM,
1271 ISD::VECREDUCE_FMAXIMUM};
1272
1273 // TODO: support more vp ops.
1274 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1275 ISD::VP_REDUCE_FMIN,
1276 ISD::VP_REDUCE_FMAX,
1277 ISD::VP_REDUCE_FMINIMUM,
1278 ISD::VP_REDUCE_FMAXIMUM};
1279
1280 // Sets common operation actions on RVV floating-point vector types.
1281 const auto SetCommonVFPActions = [&](MVT VT) {
1282 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1283 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1284 // sizes are within one power-of-two of each other. Therefore conversions
1285 // between vXf16 and vXf64 must be lowered as sequences which convert via
1286 // vXf32.
1287 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1288 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1289 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1290 // Custom-lower insert/extract operations to simplify patterns.
1291 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1292 Custom);
1293 // Expand various condition codes (explained above).
1294 setCondCodeAction(VFPCCToExpand, VT, Expand);
1295
1296 setOperationAction(
1297 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1298 Legal);
1299 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1300
1301 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1302 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1303 ISD::IS_FPCLASS},
1304 VT, Custom);
1305
1306 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1307
1308 // Expand FP operations that need libcalls.
1309 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1310
1311 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1312
1313 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1314
1315 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1316
1317 setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1318 VT, Custom);
1319
1320 setOperationAction(
1321 {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1322 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1323 VT, Custom);
1324 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1325
1326 setOperationAction(ISD::SELECT, VT, Custom);
1327 setOperationAction(ISD::SELECT_CC, VT, Expand);
1328
1329 setOperationAction({ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1330 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1331 VT, Custom);
1332
1333 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1334 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1335
1336 setOperationAction({ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1337 ISD::VECTOR_SPLICE_RIGHT},
1338 VT, Custom);
1339 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1340 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1341
1342 setOperationAction(FloatingPointVPOps, VT, Custom);
1343
1344 setOperationAction({ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1345 Custom);
1346 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1347 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1348 VT, Legal);
1349 setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1350 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1351 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1352 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1353 VT, Custom);
1354
1355 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1356 };
1357
1358 // Sets common extload/truncstore actions on RVV floating-point vector
1359 // types.
1360 const auto SetCommonVFPExtLoadTruncStoreActions =
1361 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1362 for (auto SmallVT : SmallerVTs) {
1363 setTruncStoreAction(VT, SmallVT, Expand);
1364 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
1365 }
1366 };
1367
1368 // Sets common actions for f16 and bf16 for when there's only
1369 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1370 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1371 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1372 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1373 Custom);
1374 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1375 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1376 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1377 Custom);
1378 setOperationAction(ISD::SELECT_CC, VT, Expand);
1379 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1380 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1381 ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
1382 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1383 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1384 VT, Custom);
1385 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1386 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1387 MVT EltVT = VT.getVectorElementType();
1388 if (isTypeLegal(EltVT))
1389 setOperationAction({ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1390 VT, Custom);
1391 else
1392 setOperationAction(ISD::SPLAT_VECTOR, EltVT, Custom);
1393 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1394 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1395 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1396 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1397 ISD::VP_SCATTER},
1398 VT, Custom);
1399 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1400
1401 setOperationAction(ISD::FNEG, VT, Expand);
1402 setOperationAction(ISD::FABS, VT, Expand);
1403 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1404
1405 // Expand FP operations that need libcalls.
1406 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1407
1408 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1409
1410 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1411 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1412 setOperationAction(ZvfhminZvfbfminPromoteOps, VT, Custom);
1413 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1414 } else {
1415 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1416 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1417 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1418 }
1419 };
1420
1421 // Sets common actions for zvfbfa, some of instructions are supported
1422 // natively so that we don't need to promote them.
1423 const auto SetZvfbfaActions = [&](MVT VT) {
1424 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1425 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1426 Custom);
1427 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1428 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1429 setOperationAction({ISD::VP_MERGE, ISD::SELECT}, VT,
1430 Custom);
1431 setOperationAction(ISD::SELECT_CC, VT, Expand);
1432 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1433 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1434 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
1435 ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
1436 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1437 ISD::VECTOR_COMPRESS},
1438 VT, Custom);
1439 setOperationAction(
1440 {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1441 Legal);
1442 setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Custom);
1443 setOperationAction(ISD::IS_FPCLASS, VT, Custom);
1444 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1445 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1446
1447 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
1448 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1449 setOperationAction({ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1450 ISD::STRICT_FMA},
1451 VT, Legal);
1452 setCondCodeAction(VFPCCToExpand, VT, Expand);
1453
1454 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1455 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1456 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1457 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1458 ISD::VP_SCATTER},
1459 VT, Custom);
1460 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1461
1462 // Expand FP operations that need libcalls.
1463 setOperationAction(FloatingPointLibCallOps, VT, Expand);
1464
1465 setOperationAction(ISD::FCANONICALIZE, VT, Expand);
1466
1467 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1468 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1469 setOperationAction(ZvfbfaPromoteOps, VT, Custom);
1470 setOperationAction(ZvfhminZvfbfminPromoteVPOps, VT, Custom);
1471 } else {
1472 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1473 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1474 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1475 }
1476 };
1477
1478 if (Subtarget.hasVInstructionsF16()) {
1479 for (MVT VT : F16VecVTs) {
1480 if (!isTypeLegal(VT))
1481 continue;
1482 SetCommonVFPActions(VT);
1483 }
1484 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1485 for (MVT VT : F16VecVTs) {
1486 if (!isTypeLegal(VT))
1487 continue;
1488 SetCommonPromoteToF32Actions(VT);
1489 }
1490 }
1491
1492 if (Subtarget.hasVInstructionsBF16()) {
1493 for (MVT VT : BF16VecVTs) {
1494 if (!isTypeLegal(VT))
1495 continue;
1496 SetZvfbfaActions(VT);
1497 }
1498 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1499 for (MVT VT : BF16VecVTs) {
1500 if (!isTypeLegal(VT))
1501 continue;
1502 SetCommonPromoteToF32Actions(VT);
1503 }
1504 }
1505
1506 if (Subtarget.hasVInstructionsF32()) {
1507 for (MVT VT : F32VecVTs) {
1508 if (!isTypeLegal(VT))
1509 continue;
1510 SetCommonVFPActions(VT);
1511 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1512 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1513 }
1514 }
1515
1516 if (Subtarget.hasVInstructionsF64()) {
1517 for (MVT VT : F64VecVTs) {
1518 if (!isTypeLegal(VT))
1519 continue;
1520 SetCommonVFPActions(VT);
1521 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1522 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1523 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1524 }
1525 }
1526
1527 if (Subtarget.useRVVForFixedLengthVectors()) {
1528 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1529 if (!useRVVForFixedLengthVectorVT(VT))
1530 continue;
1531
1532 // By default everything must be expanded.
1533 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1534 setOperationAction(Op, VT, Expand);
1535 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1536 setTruncStoreAction(VT, OtherVT, Expand);
1537 setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT,
1538 OtherVT, Expand);
1539 }
1540
1541 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1542 // expansion to a build_vector of 0s.
1543 setOperationAction(ISD::UNDEF, VT, Custom);
1544
1545 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1546 setOperationAction({ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1547 Custom);
1548
1549 setOperationAction(
1550 {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1551 Custom);
1552
1553 setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1554 VT, Custom);
1555
1556 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1557 VT, Custom);
1558
1559 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1560
1561 setOperationAction({ISD::LOAD, ISD::STORE}, VT, Custom);
1562
1563 setOperationAction(ISD::SETCC, VT, Custom);
1564
1565 setOperationAction(ISD::SELECT, VT, Custom);
1566
1567 setOperationAction(
1568 {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1569 Custom);
1570
1571 setOperationAction(ISD::BITCAST, VT, Custom);
1572
1573 setOperationAction(
1574 {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1575 Custom);
1576
1577 setOperationAction(
1578 {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1579 Custom);
1580
1581 setOperationAction(
1582 {
1583 ISD::SINT_TO_FP,
1584 ISD::UINT_TO_FP,
1585 ISD::FP_TO_SINT,
1586 ISD::FP_TO_UINT,
1587 ISD::STRICT_SINT_TO_FP,
1588 ISD::STRICT_UINT_TO_FP,
1589 ISD::STRICT_FP_TO_SINT,
1590 ISD::STRICT_FP_TO_UINT,
1591 },
1592 VT, Custom);
1593 setOperationAction({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1594 Custom);
1595
1596 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1597
1598 // Operations below are different for between masks and other vectors.
1599 if (VT.getVectorElementType() == MVT::i1) {
1600 setOperationAction({ISD::AND, ISD::OR, ISD::XOR}, VT, Custom);
1601
1602 setOperationAction(ISD::VP_MERGE, VT, Custom);
1603
1604 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1605 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1606
1607 setOperationAction({ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1608 Custom);
1609 continue;
1610 }
1611
1612 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1613 // it before type legalization for i64 vectors on RV32. It will then be
1614 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1615 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1616 // improvements first.
1617 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1618 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1619 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
1620
1621 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1622 setOperationAction(ISD::BUILD_VECTOR, MVT::i64, Custom);
1623 }
1624
1625 setOperationAction(
1626 {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Custom);
1627
1628 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE,
1629 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1630 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1631 ISD::VP_SCATTER},
1632 VT, Custom);
1633 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1634
1635 setOperationAction({ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1636 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1637 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1638 VT, Custom);
1639
1640 setOperationAction(
1641 {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Custom);
1642
1643 setOperationAction({ISD::ABDS, ISD::ABDU}, VT, Custom);
1644
1645 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1646 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1647 setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Custom);
1648
1649 setOperationAction({ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1650 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1651 ISD::SSUBSAT, ISD::USUBSAT},
1652 VT, Custom);
1653
1654 setOperationAction(ISD::VSELECT, VT, Custom);
1655
1656 setOperationAction(
1657 {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Custom);
1658
1659 // Custom-lower reduction operations to set up the corresponding custom
1660 // nodes' operands.
1661 setOperationAction({ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1662 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1663 ISD::VECREDUCE_UMIN},
1664 VT, Custom);
1665
1666 setOperationAction(IntegerVPOps, VT, Custom);
1667
1668 if (Subtarget.hasStdExtZvkb())
1669 setOperationAction({ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Custom);
1670
1671 if (Subtarget.hasStdExtZvbb()) {
1672 setOperationAction({ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
1673 ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
1674 VT, Custom);
1675 } else {
1676 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1677 // range of f32.
1678 EVT FloatVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1679 if (isTypeLegal(FloatVT))
1680 setOperationAction(
1681 {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1682 Custom);
1683 }
1684
1685 setOperationAction(ISD::VECTOR_COMPRESS, VT, Custom);
1686 setOperationAction({ISD::MASKED_UDIV, ISD::MASKED_SDIV,
1687 ISD::MASKED_UREM, ISD::MASKED_SREM},
1688 VT, Custom);
1689 }
1690
1691 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1692 // There are no extending loads or truncating stores.
1693 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1694 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1695 setTruncStoreAction(VT, InnerVT, Expand);
1696 }
1697
1698 if (!useRVVForFixedLengthVectorVT(VT))
1699 continue;
1700
1701 // By default everything must be expanded.
1702 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1703 setOperationAction(Op, VT, Expand);
1704
1705 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1706 // expansion to a build_vector of 0s.
1707 setOperationAction(ISD::UNDEF, VT, Custom);
1708
1709 setOperationAction({ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1710 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1711 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1712 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1713 VT, Custom);
1714 setOperationAction(ISD::EXPERIMENTAL_VP_SPLICE, VT, Custom);
1715 setOperationAction(ISD::EXPERIMENTAL_VP_REVERSE, VT, Custom);
1716
1717 setOperationAction({ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1718 VT, Custom);
1719
1720 setOperationAction({ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1721 ISD::MGATHER, ISD::MSCATTER},
1722 VT, Custom);
1723 setOperationAction({ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1724 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1725 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1726 VT, Custom);
1727 setOperationAction(ISD::VP_LOAD_FF, VT, Custom);
1728
1729 setOperationAction({ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Custom);
1730 setOperationAction({ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1731 Custom);
1732
1733 setOperationAction(ISD::BITCAST, VT, Custom);
1734
1735 if (VT.getVectorElementType() == MVT::f16 &&
1736 !Subtarget.hasVInstructionsF16()) {
1737 setOperationAction(
1738 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1739 Custom);
1740 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1741 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1742 if (Subtarget.hasStdExtZfhmin()) {
1743 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1744 } else {
1745 // We need to custom legalize f16 build vectors if Zfhmin isn't
1746 // available.
1747 setOperationAction(ISD::BUILD_VECTOR, MVT::f16, Custom);
1748 }
1749 setOperationAction(ISD::FNEG, VT, Expand);
1750 setOperationAction(ISD::FABS, VT, Expand);
1751 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
1752 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1753 // Don't promote f16 vector operations to f32 if f32 vector type is
1754 // not legal.
1755 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1756 // TODO: Support more operations.
1757 if (!isTypeLegal(F32VecVT)) {
1758 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1759 ISD::VECREDUCE_FMAXIMUM,
1760 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1761 VT, Custom);
1762 setOperationAction(ISD::SETCC, VT, Custom);
1763 continue;
1764 }
1765 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1766 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1767 continue;
1768 }
1769
1770 if (VT.getVectorElementType() == MVT::bf16) {
1771 setOperationAction({ISD::LRINT, ISD::LLRINT}, VT, Custom);
1772 setOperationAction({ISD::LROUND, ISD::LLROUND}, VT, Custom);
1773 if (Subtarget.hasStdExtZfbfmin()) {
1774 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1775 } else {
1776 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1777 // available.
1778 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
1779 }
1780 if (Subtarget.hasVInstructionsBF16()) {
1781 setOperationAction(ZvfbfaOps, VT, Custom);
1782 setCondCodeAction(VFPCCToExpand, VT, Expand);
1783 }
1784 setOperationAction(
1785 {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1786 Custom);
1787 MVT F32VecVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
1788 // Don't promote bf16 vector operations to f32 if f32 vector type is
1789 // not legal.
1790 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1791 // TODO: Support more operations.
1792 if (!isTypeLegal(F32VecVT)) {
1793 setOperationAction({ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1794 ISD::VECREDUCE_FMAXIMUM,
1795 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1796 VT, Custom);
1797 setOperationAction(ISD::SETCC, VT, Custom);
1798 continue;
1799 }
1800
1801 if (Subtarget.hasVInstructionsBF16())
1802 setOperationPromotedToType(ZvfbfaPromoteOps, VT, F32VecVT);
1803 else
1804 setOperationPromotedToType(ZvfhminZvfbfminPromoteOps, VT, F32VecVT);
1805 setOperationPromotedToType(ZvfhminZvfbfminPromoteVPOps, VT, F32VecVT);
1806 continue;
1807 }
1808
1809 setOperationAction({ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1810 Custom);
1811
1812 setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1813 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1814 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1815 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1816 ISD::FMAXIMUM, ISD::FMINIMUM},
1817 VT, Custom);
1818
1819 setOperationAction({ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1820 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1821 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1822 ISD::FNEARBYINT, ISD::FCANONICALIZE},
1823 VT, Custom);
1824
1825 setCondCodeAction(VFPCCToExpand, VT, Expand);
1826
1827 setOperationAction(ISD::SETCC, VT, Custom);
1828 setOperationAction({ISD::VSELECT, ISD::SELECT}, VT, Custom);
1829
1830 setOperationAction(FloatingPointVecReduceOps, VT, Custom);
1831
1832 setOperationAction(FloatingPointVPOps, VT, Custom);
1833
1834 setOperationAction(
1835 {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1836 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1837 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1838 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1839 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1840 VT, Custom);
1841 }
1842
1843 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1844 setOperationAction(ISD::BITCAST, {MVT::i8, MVT::i16, MVT::i32}, Custom);
1845 if (Subtarget.is64Bit())
1846 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
1847 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1848 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1849 if (Subtarget.hasStdExtZfbfmin())
1850 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
1851 if (Subtarget.hasStdExtFOrZfinx())
1852 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
1853 if (Subtarget.hasStdExtDOrZdinx())
1854 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
1855 }
1856 }
1857
1858 if (Subtarget.hasStdExtZaamo())
1859 setOperationAction(ISD::ATOMIC_LOAD_SUB, XLenVT, Expand);
1860
1861 if (Subtarget.hasForcedAtomics()) {
1862 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1863 setOperationAction(
1864 {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1865 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1866 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1867 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1868 XLenVT, LibCall);
1869 }
1870
1871 if (Subtarget.hasVendorXTHeadMemIdx()) {
1872 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1873 setIndexedLoadAction(im, MVT::i8, Legal);
1874 setIndexedStoreAction(im, MVT::i8, Legal);
1875 setIndexedLoadAction(im, MVT::i16, Legal);
1876 setIndexedStoreAction(im, MVT::i16, Legal);
1877 setIndexedLoadAction(im, MVT::i32, Legal);
1878 setIndexedStoreAction(im, MVT::i32, Legal);
1879
1880 if (Subtarget.is64Bit()) {
1881 setIndexedLoadAction(im, MVT::i64, Legal);
1882 setIndexedStoreAction(im, MVT::i64, Legal);
1883 }
1884 }
1885 }
1886
1887 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1888 setIndexedLoadAction(ISD::POST_INC, MVT::i8, Legal);
1889 setIndexedLoadAction(ISD::POST_INC, MVT::i16, Legal);
1890 setIndexedLoadAction(ISD::POST_INC, MVT::i32, Legal);
1891
1892 setIndexedStoreAction(ISD::POST_INC, MVT::i8, Legal);
1893 setIndexedStoreAction(ISD::POST_INC, MVT::i16, Legal);
1894 setIndexedStoreAction(ISD::POST_INC, MVT::i32, Legal);
1895 }
1896
1897 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1898 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1899 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1900 ISD::PARTIAL_REDUCE_UMLA,
1901 ISD::PARTIAL_REDUCE_SUMLA};
1902 setPartialReduceMLAAction(MLAOps, MVT::nxv1i32, MVT::nxv4i8, Custom);
1903 setPartialReduceMLAAction(MLAOps, MVT::nxv2i32, MVT::nxv8i8, Custom);
1904 setPartialReduceMLAAction(MLAOps, MVT::nxv4i32, MVT::nxv16i8, Custom);
1905 setPartialReduceMLAAction(MLAOps, MVT::nxv8i32, MVT::nxv32i8, Custom);
1906 setPartialReduceMLAAction(MLAOps, MVT::nxv16i32, MVT::nxv64i8, Custom);
1907
1908 if (Subtarget.useRVVForFixedLengthVectors()) {
1909 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1910 if (VT.getVectorElementType() != MVT::i32 ||
1911 !useRVVForFixedLengthVectorVT(VT))
1912 continue;
1913 ElementCount EC = VT.getVectorElementCount();
1914 MVT ArgVT = MVT::getVectorVT(MVT::i8, EC.multiplyCoefficientBy(4));
1915 setPartialReduceMLAAction(MLAOps, VT, ArgVT, Custom);
1916 }
1917 }
1918 }
1919
1920 // Customize load and store operation for bf16 if zfh isn't enabled.
1921 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
1922 setOperationAction(ISD::LOAD, MVT::bf16, Custom);
1923 setOperationAction(ISD::STORE, MVT::bf16, Custom);
1924 }
1925
1926 // Function alignments.
1927 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
1928 setMinFunctionAlignment(FunctionAlignment);
1929 // Set preferred alignments.
1930 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
1931 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
1932
1933 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
1934 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
1935 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
1936 ISD::SRA});
1937 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1938
1939 if (Subtarget.hasStdExtFOrZfinx())
1940 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
1941
1942 // Allow scalar min/max to be combined with vector reductions.
1943 if (Subtarget.hasVInstructions())
1944 setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
1945 if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
1946 setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
1947
1948 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
1949 Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
1950 setTargetDAGCombine(ISD::TRUNCATE);
1951
1952 if (Subtarget.hasStdExtZbkb())
1953 setTargetDAGCombine(ISD::BITREVERSE);
1954
1955 if (Subtarget.hasStdExtFOrZfinx())
1956 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1957 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
1958 if (Subtarget.hasVInstructions())
1959 setTargetDAGCombine({ISD::FCOPYSIGN,
1960 ISD::MGATHER,
1961 ISD::MSCATTER,
1962 ISD::VP_GATHER,
1963 ISD::VP_SCATTER,
1964 ISD::SRL,
1965 ISD::SHL,
1966 ISD::STORE,
1967 ISD::SPLAT_VECTOR,
1968 ISD::BUILD_VECTOR,
1969 ISD::CONCAT_VECTORS,
1970 ISD::VP_STORE,
1971 ISD::EXPERIMENTAL_VP_REVERSE,
1972 ISD::SDIV,
1973 ISD::UDIV,
1974 ISD::SREM,
1975 ISD::UREM,
1976 ISD::INSERT_VECTOR_ELT,
1977 ISD::ABS,
1978 ISD::ABS_MIN_POISON,
1979 ISD::CTPOP,
1980 ISD::VECTOR_SHUFFLE,
1981 ISD::FMA,
1982 ISD::VSELECT,
1983 ISD::VECREDUCE_ADD,
1984 ISD::VECTOR_SPLICE_RIGHT});
1985
1986 if (Subtarget.hasVendorXTHeadMemPair())
1987 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
1988 if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
1989 setTargetDAGCombine(ISD::BITCAST);
1990
1991 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
1992
1993 setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
1994
1995 setJumpIsExpensive(Subtarget.isJumpExpensive());
1996
1997 // Disable strict node mutation.
1998 IsStrictFPEnabled = true;
1999 EnableExtLdPromotion = true;
2000
2001 // Let the subtarget decide if a predictable select is more expensive than the
2002 // corresponding branch. This information is used in CGP/SelectOpt to decide
2003 // when to convert selects into branches.
2004 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
2005
2006 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
2007 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
2008
2009 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
2010 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
2011 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
2012
2013 MaxStoresPerMemmoveOptSize =
2014 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
2015 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
2016
2017 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
2018 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
2019}
2020
2021TargetLoweringBase::LegalizeTypeAction
2022RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
2023 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
2024 if (VT == MVT::v2i16 || VT == MVT::v4i8)
2025 return TypeWidenVector;
2026
2027 return TargetLoweringBase::getPreferredVectorAction(VT);
2028}
2029
2030EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
2031 LLVMContext &Context,
2032 EVT VT) const {
2033 if (!VT.isVector())
2034 return getPointerTy(DL);
2035 if (Subtarget.hasVInstructions() &&
2036 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
2037 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
2038 return VT.changeVectorElementTypeToInteger();
2039}
2040
2041MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
2042 return Subtarget.getXLenVT();
2043}
2044
2045// Return false if we can lower get_vector_length to a vsetvli intrinsic.
2046bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
2047 unsigned VF,
2048 bool IsScalable) const {
2049 if (!Subtarget.hasVInstructions())
2050 return true;
2051
2052 if (!IsScalable)
2053 return true;
2054
2055 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2056 return true;
2057
2058 // Don't allow VF=1 if those types are't legal.
2059 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2060 return true;
2061
2062 // VLEN=32 support is incomplete.
2063 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2064 return true;
2065
2066 // The maximum VF is for the smallest element width with LMUL=8.
2067 // VF must be a power of 2.
2068 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2069 return VF > MaxVF || !isPowerOf2_32(VF);
2070}
2071
2072bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
2073 return !Subtarget.hasVInstructions() ||
2074 VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
2075}
2076
2077void RISCVTargetLowering::getTgtMemIntrinsic(
2078 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2079 MachineFunction &MF, unsigned Intrinsic) const {
2080 IntrinsicInfo Info;
2081 auto &DL = I.getDataLayout();
2082
2083 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2084 bool IsUnitStrided, bool UsePtrVal = false) {
2085 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2086 // We can't use ptrVal if the intrinsic can access memory before the
2087 // pointer. This means we can't use it for strided or indexed intrinsics.
2088 if (UsePtrVal)
2089 Info.ptrVal = I.getArgOperand(PtrOp);
2090 else
2091 Info.fallbackAddressSpace =
2092 I.getArgOperand(PtrOp)->getType()->getPointerAddressSpace();
2093 Type *MemTy;
2094 if (IsStore) {
2095 // Store value is the first operand.
2096 MemTy = I.getArgOperand(0)->getType();
2097 } else {
2098 // Use return type. If it's segment load, return type is a struct.
2099 MemTy = I.getType();
2100 if (MemTy->isStructTy())
2101 MemTy = MemTy->getStructElementType(0);
2102 }
2103 if (!IsUnitStrided)
2104 MemTy = MemTy->getScalarType();
2105
2106 Info.memVT = getValueType(DL, MemTy);
2107 if (MemTy->isTargetExtTy()) {
2108 // RISC-V vector tuple type's alignment type should be its element type.
2109 if (cast<TargetExtType>(MemTy)->getName() == "riscv.vector.tuple")
2110 MemTy = Type::getIntNTy(
2111 MemTy->getContext(),
2112 1 << cast<ConstantInt>(I.getArgOperand(I.arg_size() - 1))
2113 ->getZExtValue());
2114 Info.align = DL.getABITypeAlign(MemTy);
2115 } else {
2116 Info.align = Align(DL.getTypeStoreSize(MemTy->getScalarType()));
2117 }
2118 Info.size = MemoryLocation::UnknownSize;
2119 Info.flags |=
2120 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2121 Infos.push_back(Info);
2122 };
2123
2124 if (I.hasMetadata(LLVMContext::MD_nontemporal))
2125 Info.flags |= MachineMemOperand::MONonTemporal;
2126
2127 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2128 switch (Intrinsic) {
2129 default:
2130 return;
2131 case Intrinsic::riscv_masked_atomicrmw_xchg:
2132 case Intrinsic::riscv_masked_atomicrmw_add:
2133 case Intrinsic::riscv_masked_atomicrmw_sub:
2134 case Intrinsic::riscv_masked_atomicrmw_nand:
2135 case Intrinsic::riscv_masked_atomicrmw_max:
2136 case Intrinsic::riscv_masked_atomicrmw_min:
2137 case Intrinsic::riscv_masked_atomicrmw_umax:
2138 case Intrinsic::riscv_masked_atomicrmw_umin:
2139 case Intrinsic::riscv_masked_cmpxchg:
2140 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2141 // narrow atomic operation. These will be expanded to an LR/SC loop that
2142 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2143 // will be used to modify the appropriate part of the 4 byte data and
2144 // preserve the rest.
2145 Info.opc = ISD::INTRINSIC_W_CHAIN;
2146 Info.memVT = MVT::i32;
2147 Info.ptrVal = I.getArgOperand(0);
2148 Info.offset = 0;
2149 Info.align = Align(4);
2150 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2151 MachineMemOperand::MOVolatile;
2152 Infos.push_back(Info);
2153 return;
2154 case Intrinsic::riscv_seg2_load_mask:
2155 case Intrinsic::riscv_seg3_load_mask:
2156 case Intrinsic::riscv_seg4_load_mask:
2157 case Intrinsic::riscv_seg5_load_mask:
2158 case Intrinsic::riscv_seg6_load_mask:
2159 case Intrinsic::riscv_seg7_load_mask:
2160 case Intrinsic::riscv_seg8_load_mask:
2161 case Intrinsic::riscv_sseg2_load_mask:
2162 case Intrinsic::riscv_sseg3_load_mask:
2163 case Intrinsic::riscv_sseg4_load_mask:
2164 case Intrinsic::riscv_sseg5_load_mask:
2165 case Intrinsic::riscv_sseg6_load_mask:
2166 case Intrinsic::riscv_sseg7_load_mask:
2167 case Intrinsic::riscv_sseg8_load_mask:
2168 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2169 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2170 return;
2171 case Intrinsic::riscv_seg2_store_mask:
2172 case Intrinsic::riscv_seg3_store_mask:
2173 case Intrinsic::riscv_seg4_store_mask:
2174 case Intrinsic::riscv_seg5_store_mask:
2175 case Intrinsic::riscv_seg6_store_mask:
2176 case Intrinsic::riscv_seg7_store_mask:
2177 case Intrinsic::riscv_seg8_store_mask:
2178 // Operands are (vec, ..., vec, ptr, mask, vl)
2179 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2180 /*IsStore*/ true,
2181 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2182 return;
2183 case Intrinsic::riscv_sseg2_store_mask:
2184 case Intrinsic::riscv_sseg3_store_mask:
2185 case Intrinsic::riscv_sseg4_store_mask:
2186 case Intrinsic::riscv_sseg5_store_mask:
2187 case Intrinsic::riscv_sseg6_store_mask:
2188 case Intrinsic::riscv_sseg7_store_mask:
2189 case Intrinsic::riscv_sseg8_store_mask:
2190 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2191 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2192 /*IsStore*/ true,
2193 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2194 return;
2195 case Intrinsic::riscv_vlm:
2196 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2197 /*IsStore*/ false,
2198 /*IsUnitStrided*/ true,
2199 /*UsePtrVal*/ true);
2200 return;
2201 case Intrinsic::riscv_vle:
2202 case Intrinsic::riscv_vle_mask:
2203 case Intrinsic::riscv_vleff:
2204 case Intrinsic::riscv_vleff_mask:
2205 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2206 /*IsStore*/ false,
2207 /*IsUnitStrided*/ true,
2208 /*UsePtrVal*/ true);
2209 return;
2210 case Intrinsic::riscv_vsm:
2211 case Intrinsic::riscv_vse:
2212 case Intrinsic::riscv_vse_mask:
2213 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2214 /*IsStore*/ true,
2215 /*IsUnitStrided*/ true,
2216 /*UsePtrVal*/ true);
2217 return;
2218 case Intrinsic::riscv_vlse:
2219 case Intrinsic::riscv_vlse_mask:
2220 case Intrinsic::riscv_vloxei:
2221 case Intrinsic::riscv_vloxei_mask:
2222 case Intrinsic::riscv_vluxei:
2223 case Intrinsic::riscv_vluxei_mask:
2224 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2225 /*IsStore*/ false,
2226 /*IsUnitStrided*/ false);
2227 return;
2228 case Intrinsic::riscv_vsse:
2229 case Intrinsic::riscv_vsse_mask:
2230 case Intrinsic::riscv_vsoxei:
2231 case Intrinsic::riscv_vsoxei_mask:
2232 case Intrinsic::riscv_vsuxei:
2233 case Intrinsic::riscv_vsuxei_mask:
2234 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2235 /*IsStore*/ true,
2236 /*IsUnitStrided*/ false);
2237 return;
2238 case Intrinsic::riscv_vlseg2:
2239 case Intrinsic::riscv_vlseg3:
2240 case Intrinsic::riscv_vlseg4:
2241 case Intrinsic::riscv_vlseg5:
2242 case Intrinsic::riscv_vlseg6:
2243 case Intrinsic::riscv_vlseg7:
2244 case Intrinsic::riscv_vlseg8:
2245 case Intrinsic::riscv_vlseg2ff:
2246 case Intrinsic::riscv_vlseg3ff:
2247 case Intrinsic::riscv_vlseg4ff:
2248 case Intrinsic::riscv_vlseg5ff:
2249 case Intrinsic::riscv_vlseg6ff:
2250 case Intrinsic::riscv_vlseg7ff:
2251 case Intrinsic::riscv_vlseg8ff:
2252 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2253 /*IsStore*/ false,
2254 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2255 return;
2256 case Intrinsic::riscv_vlseg2_mask:
2257 case Intrinsic::riscv_vlseg3_mask:
2258 case Intrinsic::riscv_vlseg4_mask:
2259 case Intrinsic::riscv_vlseg5_mask:
2260 case Intrinsic::riscv_vlseg6_mask:
2261 case Intrinsic::riscv_vlseg7_mask:
2262 case Intrinsic::riscv_vlseg8_mask:
2263 case Intrinsic::riscv_vlseg2ff_mask:
2264 case Intrinsic::riscv_vlseg3ff_mask:
2265 case Intrinsic::riscv_vlseg4ff_mask:
2266 case Intrinsic::riscv_vlseg5ff_mask:
2267 case Intrinsic::riscv_vlseg6ff_mask:
2268 case Intrinsic::riscv_vlseg7ff_mask:
2269 case Intrinsic::riscv_vlseg8ff_mask:
2270 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2271 /*IsStore*/ false,
2272 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2273 return;
2274 case Intrinsic::riscv_vlsseg2:
2275 case Intrinsic::riscv_vlsseg3:
2276 case Intrinsic::riscv_vlsseg4:
2277 case Intrinsic::riscv_vlsseg5:
2278 case Intrinsic::riscv_vlsseg6:
2279 case Intrinsic::riscv_vlsseg7:
2280 case Intrinsic::riscv_vlsseg8:
2281 case Intrinsic::riscv_vloxseg2:
2282 case Intrinsic::riscv_vloxseg3:
2283 case Intrinsic::riscv_vloxseg4:
2284 case Intrinsic::riscv_vloxseg5:
2285 case Intrinsic::riscv_vloxseg6:
2286 case Intrinsic::riscv_vloxseg7:
2287 case Intrinsic::riscv_vloxseg8:
2288 case Intrinsic::riscv_vluxseg2:
2289 case Intrinsic::riscv_vluxseg3:
2290 case Intrinsic::riscv_vluxseg4:
2291 case Intrinsic::riscv_vluxseg5:
2292 case Intrinsic::riscv_vluxseg6:
2293 case Intrinsic::riscv_vluxseg7:
2294 case Intrinsic::riscv_vluxseg8:
2295 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2296 /*IsStore*/ false,
2297 /*IsUnitStrided*/ false);
2298 return;
2299 case Intrinsic::riscv_vlsseg2_mask:
2300 case Intrinsic::riscv_vlsseg3_mask:
2301 case Intrinsic::riscv_vlsseg4_mask:
2302 case Intrinsic::riscv_vlsseg5_mask:
2303 case Intrinsic::riscv_vlsseg6_mask:
2304 case Intrinsic::riscv_vlsseg7_mask:
2305 case Intrinsic::riscv_vlsseg8_mask:
2306 case Intrinsic::riscv_vloxseg2_mask:
2307 case Intrinsic::riscv_vloxseg3_mask:
2308 case Intrinsic::riscv_vloxseg4_mask:
2309 case Intrinsic::riscv_vloxseg5_mask:
2310 case Intrinsic::riscv_vloxseg6_mask:
2311 case Intrinsic::riscv_vloxseg7_mask:
2312 case Intrinsic::riscv_vloxseg8_mask:
2313 case Intrinsic::riscv_vluxseg2_mask:
2314 case Intrinsic::riscv_vluxseg3_mask:
2315 case Intrinsic::riscv_vluxseg4_mask:
2316 case Intrinsic::riscv_vluxseg5_mask:
2317 case Intrinsic::riscv_vluxseg6_mask:
2318 case Intrinsic::riscv_vluxseg7_mask:
2319 case Intrinsic::riscv_vluxseg8_mask:
2320 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2321 /*IsStore*/ false,
2322 /*IsUnitStrided*/ false);
2323 return;
2324 case Intrinsic::riscv_vsseg2:
2325 case Intrinsic::riscv_vsseg3:
2326 case Intrinsic::riscv_vsseg4:
2327 case Intrinsic::riscv_vsseg5:
2328 case Intrinsic::riscv_vsseg6:
2329 case Intrinsic::riscv_vsseg7:
2330 case Intrinsic::riscv_vsseg8:
2331 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2332 /*IsStore*/ true,
2333 /*IsUnitStrided*/ false);
2334 return;
2335 case Intrinsic::riscv_vsseg2_mask:
2336 case Intrinsic::riscv_vsseg3_mask:
2337 case Intrinsic::riscv_vsseg4_mask:
2338 case Intrinsic::riscv_vsseg5_mask:
2339 case Intrinsic::riscv_vsseg6_mask:
2340 case Intrinsic::riscv_vsseg7_mask:
2341 case Intrinsic::riscv_vsseg8_mask:
2342 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2343 /*IsStore*/ true,
2344 /*IsUnitStrided*/ false);
2345 return;
2346 case Intrinsic::riscv_vssseg2:
2347 case Intrinsic::riscv_vssseg3:
2348 case Intrinsic::riscv_vssseg4:
2349 case Intrinsic::riscv_vssseg5:
2350 case Intrinsic::riscv_vssseg6:
2351 case Intrinsic::riscv_vssseg7:
2352 case Intrinsic::riscv_vssseg8:
2353 case Intrinsic::riscv_vsoxseg2:
2354 case Intrinsic::riscv_vsoxseg3:
2355 case Intrinsic::riscv_vsoxseg4:
2356 case Intrinsic::riscv_vsoxseg5:
2357 case Intrinsic::riscv_vsoxseg6:
2358 case Intrinsic::riscv_vsoxseg7:
2359 case Intrinsic::riscv_vsoxseg8:
2360 case Intrinsic::riscv_vsuxseg2:
2361 case Intrinsic::riscv_vsuxseg3:
2362 case Intrinsic::riscv_vsuxseg4:
2363 case Intrinsic::riscv_vsuxseg5:
2364 case Intrinsic::riscv_vsuxseg6:
2365 case Intrinsic::riscv_vsuxseg7:
2366 case Intrinsic::riscv_vsuxseg8:
2367 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2368 /*IsStore*/ true,
2369 /*IsUnitStrided*/ false);
2370 return;
2371 case Intrinsic::riscv_vssseg2_mask:
2372 case Intrinsic::riscv_vssseg3_mask:
2373 case Intrinsic::riscv_vssseg4_mask:
2374 case Intrinsic::riscv_vssseg5_mask:
2375 case Intrinsic::riscv_vssseg6_mask:
2376 case Intrinsic::riscv_vssseg7_mask:
2377 case Intrinsic::riscv_vssseg8_mask:
2378 case Intrinsic::riscv_vsoxseg2_mask:
2379 case Intrinsic::riscv_vsoxseg3_mask:
2380 case Intrinsic::riscv_vsoxseg4_mask:
2381 case Intrinsic::riscv_vsoxseg5_mask:
2382 case Intrinsic::riscv_vsoxseg6_mask:
2383 case Intrinsic::riscv_vsoxseg7_mask:
2384 case Intrinsic::riscv_vsoxseg8_mask:
2385 case Intrinsic::riscv_vsuxseg2_mask:
2386 case Intrinsic::riscv_vsuxseg3_mask:
2387 case Intrinsic::riscv_vsuxseg4_mask:
2388 case Intrinsic::riscv_vsuxseg5_mask:
2389 case Intrinsic::riscv_vsuxseg6_mask:
2390 case Intrinsic::riscv_vsuxseg7_mask:
2391 case Intrinsic::riscv_vsuxseg8_mask:
2392 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2393 /*IsStore*/ true,
2394 /*IsUnitStrided*/ false);
2395 return;
2396 case Intrinsic::riscv_sf_vlte8:
2397 case Intrinsic::riscv_sf_vlte16:
2398 case Intrinsic::riscv_sf_vlte32:
2399 case Intrinsic::riscv_sf_vlte64:
2400 Info.opc = ISD::INTRINSIC_VOID;
2401 Info.ptrVal = I.getArgOperand(1);
2402 switch (Intrinsic) {
2403 case Intrinsic::riscv_sf_vlte8:
2404 Info.memVT = MVT::i8;
2405 Info.align = Align(1);
2406 break;
2407 case Intrinsic::riscv_sf_vlte16:
2408 Info.memVT = MVT::i16;
2409 Info.align = Align(2);
2410 break;
2411 case Intrinsic::riscv_sf_vlte32:
2412 Info.memVT = MVT::i32;
2413 Info.align = Align(4);
2414 break;
2415 case Intrinsic::riscv_sf_vlte64:
2416 Info.memVT = MVT::i64;
2417 Info.align = Align(8);
2418 break;
2419 }
2420 Info.size = MemoryLocation::UnknownSize;
2421 Info.flags |= MachineMemOperand::MOLoad;
2422 Infos.push_back(Info);
2423 return;
2424 case Intrinsic::riscv_sf_vste8:
2425 case Intrinsic::riscv_sf_vste16:
2426 case Intrinsic::riscv_sf_vste32:
2427 case Intrinsic::riscv_sf_vste64:
2428 Info.opc = ISD::INTRINSIC_VOID;
2429 Info.ptrVal = I.getArgOperand(1);
2430 switch (Intrinsic) {
2431 case Intrinsic::riscv_sf_vste8:
2432 Info.memVT = MVT::i8;
2433 Info.align = Align(1);
2434 break;
2435 case Intrinsic::riscv_sf_vste16:
2436 Info.memVT = MVT::i16;
2437 Info.align = Align(2);
2438 break;
2439 case Intrinsic::riscv_sf_vste32:
2440 Info.memVT = MVT::i32;
2441 Info.align = Align(4);
2442 break;
2443 case Intrinsic::riscv_sf_vste64:
2444 Info.memVT = MVT::i64;
2445 Info.align = Align(8);
2446 break;
2447 }
2448 Info.size = MemoryLocation::UnknownSize;
2449 Info.flags |= MachineMemOperand::MOStore;
2450 Infos.push_back(Info);
2451 return;
2452 }
2453}
2454
2455bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2456 const AddrMode &AM, Type *Ty,
2457 unsigned AS,
2458 Instruction *I) const {
2459 // No global is ever allowed as a base.
2460 if (AM.BaseGV)
2461 return false;
2462
2463 // None of our addressing modes allows a scalable offset
2464 if (AM.ScalableOffset)
2465 return false;
2466
2467 // RVV instructions only support register addressing.
2468 if (Subtarget.hasVInstructions() && isa<VectorType>(Ty))
2469 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2470
2471 // The Xqcilo extension provides load/store instructions with a 26-bit signed
2472 // offset.
2473 if (Subtarget.hasVendorXqcilo()) {
2474 if (!isInt<26>(AM.BaseOffs))
2475 return false;
2476 } else if (!isInt<12>(AM.BaseOffs)) {
2477 // Otherwise require a 12-bit signed offset.
2478 return false;
2479 }
2480
2481 switch (AM.Scale) {
2482 case 0: // "r+i" or just "i", depending on HasBaseReg.
2483 break;
2484 case 1:
2485 if (!AM.HasBaseReg) // allow "r+i".
2486 break;
2487 return false; // disallow "r+r" or "r+r+i".
2488 default:
2489 return false;
2490 }
2491
2492 return true;
2493}
2494
2495bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2496 return isInt<12>(Imm);
2497}
2498
2499bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2500 // The Xqcilia extension provides add-immediate instructions with a 26-bit
2501 // signed immediate.
2502 if (Subtarget.hasVendorXqcilia())
2503 return isInt<26>(Imm);
2504 return isInt<12>(Imm);
2505}
2506
2507// On RV32, 64-bit integers are split into their high and low parts and held
2508// in two different registers, so the trunc is free since the low register can
2509// just be used.
2510// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2511// isTruncateFree?
2512bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2513 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2514 return false;
2515 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2516 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2517 return (SrcBits == 64 && DestBits == 32);
2518}
2519
2520bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2521 // We consider i64->i32 free on RV64 since we have good selection of W
2522 // instructions that make promoting operations back to i64 free in many cases.
2523 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2524 !DstVT.isInteger())
2525 return false;
2526 unsigned SrcBits = SrcVT.getSizeInBits();
2527 unsigned DestBits = DstVT.getSizeInBits();
2528 return (SrcBits == 64 && DestBits == 32);
2529}
2530
2531bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2532 EVT SrcVT = Val.getValueType();
2533 // free truncate from vnsrl and vnsra
2534 if (Subtarget.hasVInstructions() &&
2535 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2536 SrcVT.isVector() && VT2.isVector()) {
2537 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2538 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2539 if (SrcBits == DestBits * 2) {
2540 return true;
2541 }
2542 }
2543 return TargetLowering::isTruncateFree(Val, VT2);
2544}
2545
2546bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2547 // Zexts are free if they can be combined with a load.
2548 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2549 // poorly with type legalization of compares preferring sext.
2550 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2551 EVT MemVT = LD->getMemoryVT();
2552 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2553 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2554 LD->getExtensionType() == ISD::ZEXTLOAD))
2555 return true;
2556 }
2557
2558 return TargetLowering::isZExtFree(Val, VT2);
2559}
2560
2561bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2562 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2563}
2564
2565bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2566 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
2567}
2568
2569bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2570 return Subtarget.hasCTZLike();
2571}
2572
2573bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2574 return Subtarget.hasCLZLike();
2575}
2576
2577bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2578 const Instruction &AndI) const {
2579 // We expect to be able to match a bit extraction instruction if the Zbs
2580 // extension is supported and the mask is a power of two. However, we
2581 // conservatively return false if the mask would fit in an ANDI instruction,
2582 // on the basis that it's possible the sinking+duplication of the AND in
2583 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2584 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2585 if (!Subtarget.hasBEXTILike())
2586 return false;
2587 ConstantInt *Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
2588 if (!Mask)
2589 return false;
2590 return !Mask->getValue().isSignedIntN(12) && Mask->getValue().isPowerOf2();
2591}
2592
2593bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2594 EVT VT = Y.getValueType();
2595
2596 if (VT.isVector())
2597 return false;
2598
2599 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2600 (!isa<ConstantSDNode>(Y) || cast<ConstantSDNode>(Y)->isOpaque());
2601}
2602
2603bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2604 EVT VT = Y.getValueType();
2605
2606 if (!VT.isVector())
2607 return hasAndNotCompare(Y);
2608
2609 return Subtarget.hasStdExtZvkb();
2610}
2611
2612bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2613 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2614 if (Subtarget.hasStdExtZbs())
2615 return X.getValueType().isScalarInteger();
2616 auto *C = dyn_cast<ConstantSDNode>(Y);
2617 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2618 if (Subtarget.hasVendorXTHeadBs())
2619 return C != nullptr;
2620 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2621 return C && C->getAPIntValue().ule(10);
2622}
2623
2624bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2625 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2626 SDValue Y) const {
2627 if (SelectOpcode != ISD::VSELECT)
2628 return false;
2629
2630 // Only enable for rvv.
2631 if (!VT.isVector() || !Subtarget.hasVInstructions())
2632 return false;
2633
2634 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2635 return false;
2636
2637 return true;
2638}
2639
2640bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2641 Type *Ty) const {
2642 assert(Ty->isIntegerTy());
2643
2644 unsigned BitSize = Ty->getIntegerBitWidth();
2645 if (BitSize > Subtarget.getXLen())
2646 return false;
2647
2648 // Fast path, assume 32-bit immediates are cheap.
2649 int64_t Val = Imm.getSExtValue();
2650 if (isInt<32>(Val))
2651 return true;
2652
2653 // A constant pool entry may be more aligned than the load we're trying to
2654 // replace. If we don't support unaligned scalar mem, prefer the constant
2655 // pool.
2656 // TODO: Can the caller pass down the alignment?
2657 if (!Subtarget.enableUnalignedScalarMem())
2658 return true;
2659
2660 // Prefer to keep the load if it would require many instructions.
2661 // This uses the same threshold we use for constant pools but doesn't
2662 // check useConstantPoolForLargeInts.
2663 // TODO: Should we keep the load only when we're definitely going to emit a
2664 // constant pool?
2665
2666 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, Subtarget);
2667 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2668}
2669
2670bool RISCVTargetLowering::
2671 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2672 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2673 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2674 SelectionDAG &DAG) const {
2675 // One interesting pattern that we'd want to form is 'bit extract':
2676 // ((1 >> Y) & 1) ==/!= 0
2677 // But we also need to be careful not to try to reverse that fold.
2678
2679 // Is this '((1 >> Y) & 1)'?
2680 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2681 return false; // Keep the 'bit extract' pattern.
2682
2683 // Will this be '((1 >> Y) & 1)' after the transform?
2684 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2685 return true; // Do form the 'bit extract' pattern.
2686
2687 // If 'X' is a constant, and we transform, then we will immediately
2688 // try to undo the fold, thus causing endless combine loop.
2689 // So only do the transform if X is not a constant. This matches the default
2690 // implementation of this function.
2691 return !XC;
2692}
2693
2694bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2695 unsigned Opc = VecOp.getOpcode();
2696
2697 // Assume target opcodes can't be scalarized.
2698 // TODO - do we have any exceptions?
2699 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
2700 return false;
2701
2702 // If the vector op is not supported, try to convert to scalar.
2703 EVT VecVT = VecOp.getValueType();
2704 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
2705 return true;
2706
2707 // If the vector op is supported, but the scalar op is not, the transform may
2708 // not be worthwhile.
2709 // Permit a vector binary operation can be converted to scalar binary
2710 // operation which is custom lowered with illegal type.
2711 EVT ScalarVT = VecVT.getScalarType();
2712 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT) ||
2713 isOperationCustom(Opc, ScalarVT);
2714}
2715
2716bool RISCVTargetLowering::isOffsetFoldingLegal(
2717 const GlobalAddressSDNode *GA) const {
2718 // In order to maximise the opportunity for common subexpression elimination,
2719 // keep a separate ADD node for the global address offset instead of folding
2720 // it in the global address node. Later peephole optimisations may choose to
2721 // fold it back in when profitable.
2722 return false;
2723}
2724
2725// Returns 0-31 if the fli instruction is available for the type and this is
2726// legal FP immediate for the type. Returns -1 otherwise.
2727int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2728 if (!Subtarget.hasStdExtZfa())
2729 return -1;
2730
2731 bool IsSupportedVT = false;
2732 if (VT == MVT::f16) {
2733 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2734 } else if (VT == MVT::f32) {
2735 IsSupportedVT = true;
2736 } else if (VT == MVT::f64) {
2737 assert(Subtarget.hasStdExtD() && "Expect D extension");
2738 IsSupportedVT = true;
2739 }
2740
2741 if (!IsSupportedVT)
2742 return -1;
2743
2744 return RISCVLoadFPImm::getLoadFPImm(Imm);
2745}
2746
2747bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2748 bool ForCodeSize) const {
2749 bool IsLegalVT = false;
2750 if (VT == MVT::f16)
2751 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2752 else if (VT == MVT::f32)
2753 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2754 else if (VT == MVT::f64)
2755 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2756 else if (VT == MVT::bf16)
2757 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2758
2759 if (!IsLegalVT)
2760 return false;
2761
2762 if (getLegalZfaFPImm(Imm, VT) >= 0)
2763 return true;
2764
2765 // Some constants can be produced by fli+fneg.
2766 if (Imm.isNegative() && getLegalZfaFPImm(-Imm, VT) >= 0)
2767 return true;
2768
2769 // Cannot create a 64 bit floating-point immediate value for rv32.
2770 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2771 // td can handle +0.0 or -0.0 already.
2772 // -0.0 can be created by fmv + fneg.
2773 return Imm.isZero();
2774 }
2775
2776 // Special case: fmv + fneg
2777 if (Imm.isNegZero())
2778 return true;
2779
2780 // Building an integer and then converting requires a fmv at the end of
2781 // the integer sequence. The fmv is not required for Zfinx.
2782 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2783 const int Cost =
2784 FmvCost + RISCVMatInt::getIntMatCost(Imm.bitcastToAPInt(),
2785 Subtarget.getXLen(), Subtarget);
2786 return Cost <= FPImmCost;
2787}
2788
2789// TODO: This is very conservative.
2790bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2791 unsigned Index) const {
2792 if (!Subtarget.hasVInstructions())
2793 return false;
2794
2795 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
2796 return false;
2797
2798 // Extracts from index 0 are just subreg extracts.
2799 if (Index == 0)
2800 return true;
2801
2802 // Only support extracting a fixed from a fixed vector for now.
2803 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2804 return false;
2805
2806 EVT EltVT = ResVT.getVectorElementType();
2807 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2808
2809 // The smallest type we can slide is i8.
2810 if (EltVT == MVT::i1)
2811 return false;
2812
2813 unsigned ResElts = ResVT.getVectorNumElements();
2814 unsigned SrcElts = SrcVT.getVectorNumElements();
2815
2816 unsigned MinVLen = Subtarget.getRealMinVLen();
2817 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2818
2819 // If we're extracting only data from the first VLEN bits of the source
2820 // then we can always do this with an m1 vslidedown.vx. Restricting the
2821 // Index ensures we can use a vslidedown.vi.
2822 // TODO: We can generalize this when the exact VLEN is known.
2823 if (Index + ResElts <= MinVLMAX && Index < 31)
2824 return true;
2825
2826 // Convervatively only handle extracting half of a vector.
2827 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2828 // the upper half of a vector until we have more test coverage.
2829 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2830 // a cheap extract. However, this case is important in practice for
2831 // shuffled extracts of longer vectors. How resolve?
2832 return (ResElts * 2) == SrcElts && Index == ResElts;
2833}
2834
2835MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2836 CallingConv::ID CC,
2837 EVT VT) const {
2838 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2839 // We might still end up using a GPR but that will be decided based on ABI.
2840 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2841 !Subtarget.hasStdExtZfhminOrZhinxmin())
2842 return MVT::f32;
2843
2844 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2845}
2846
2847unsigned
2848RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2849 std::optional<MVT> RegisterVT) const {
2850 // Pair inline assembly operand
2851 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2852 *RegisterVT == MVT::Untyped)
2853 return 1;
2854
2855 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2856}
2857
2858unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2859 CallingConv::ID CC,
2860 EVT VT) const {
2861 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2862 // We might still end up using a GPR but that will be decided based on ABI.
2863 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2864 !Subtarget.hasStdExtZfhminOrZhinxmin())
2865 return 1;
2866
2867 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2868}
2869
2870// Changes the condition code and swaps operands if necessary, so the SetCC
2871// operation matches one of the comparisons supported directly by branches
2872// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2873// with 1/-1.
2874static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2875 ISD::CondCode &CC, SelectionDAG &DAG,
2876 const RISCVSubtarget &Subtarget) {
2877 // If this is a single bit test that can't be handled by ANDI, shift the
2878 // bit to be tested to the MSB and perform a signed compare with 0.
2879 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
2880 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2881 isa<ConstantSDNode>(LHS.getOperand(1)) &&
2882 // XAndesPerf supports branch on test bit.
2883 !Subtarget.hasVendorXAndesPerf()) {
2884 uint64_t Mask = LHS.getConstantOperandVal(1);
2885 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
2886 unsigned ShAmt = 0;
2887 if (isPowerOf2_64(Mask)) {
2888 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
2889 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
2890 } else {
2891 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
2892 }
2893
2894 LHS = LHS.getOperand(0);
2895 if (ShAmt != 0)
2896 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
2897 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
2898 return;
2899 }
2900 }
2901
2902 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2903 int64_t C = RHSC->getSExtValue();
2904 switch (CC) {
2905 default: break;
2906 case ISD::SETGT:
2907 // Convert X > -1 to X >= 0.
2908 if (C == -1) {
2909 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2910 CC = ISD::SETGE;
2911 return;
2912 }
2913 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
2914 C != INT64_MAX && isInt<5>(C + 1)) {
2915 // We have a conditional move instruction for SETGE but not SETGT.
2916 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
2917 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
2918 CC = ISD::SETGE;
2919 return;
2920 }
2921 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(C + 1)) {
2922 // We have a branch immediate instruction for SETGE but not SETGT.
2923 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
2924 RHS = DAG.getSignedConstant(C + 1, DL, RHS.getValueType());
2925 CC = ISD::SETGE;
2926 return;
2927 }
2928 break;
2929 case ISD::SETLT:
2930 // Convert X < 1 to 0 >= X.
2931 if (C == 1) {
2932 RHS = LHS;
2933 LHS = DAG.getConstant(0, DL, RHS.getValueType());
2934 CC = ISD::SETGE;
2935 return;
2936 }
2937 break;
2938 case ISD::SETUGT:
2939 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
2940 C != INT64_MAX && isUInt<5>(C + 1)) {
2941 // We have a conditional move instruction for SETUGE but not SETUGT.
2942 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
2943 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
2944 CC = ISD::SETUGE;
2945 return;
2946 }
2947 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(C + 1)) {
2948 // We have a branch immediate instruction for SETUGE but not SETUGT.
2949 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
2950 // immediate.
2951 RHS = DAG.getConstant(C + 1, DL, RHS.getValueType());
2952 CC = ISD::SETUGE;
2953 return;
2954 }
2955 break;
2956 }
2957 }
2958
2959 switch (CC) {
2960 default:
2961 break;
2962 case ISD::SETGT:
2963 case ISD::SETLE:
2964 case ISD::SETUGT:
2965 case ISD::SETULE:
2966 CC = ISD::getSetCCSwappedOperands(CC);
2967 std::swap(LHS, RHS);
2968 break;
2969 }
2970}
2971
2972RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
2973 if (VT.isRISCVVectorTuple()) {
2974 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
2975 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
2976 return RISCVVType::LMUL_F8;
2977 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
2978 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
2979 return RISCVVType::LMUL_F4;
2980 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
2981 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
2982 return RISCVVType::LMUL_F2;
2983 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
2984 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
2985 return RISCVVType::LMUL_1;
2986 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
2987 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
2988 return RISCVVType::LMUL_2;
2989 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
2990 return RISCVVType::LMUL_4;
2991 llvm_unreachable("Invalid vector tuple type LMUL.");
2992 }
2993
2994 assert(VT.isScalableVector() && "Expecting a scalable vector type");
2995 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
2996 if (VT.getVectorElementType() == MVT::i1)
2997 KnownSize *= 8;
2998
2999 switch (KnownSize) {
3000 default:
3001 llvm_unreachable("Invalid LMUL.");
3002 case 8:
3003 return RISCVVType::LMUL_F8;
3004 case 16:
3005 return RISCVVType::LMUL_F4;
3006 case 32:
3007 return RISCVVType::LMUL_F2;
3008 case 64:
3009 return RISCVVType::LMUL_1;
3010 case 128:
3011 return RISCVVType::LMUL_2;
3012 case 256:
3013 return RISCVVType::LMUL_4;
3014 case 512:
3015 return RISCVVType::LMUL_8;
3016 }
3017}
3018
3019unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
3020 switch (LMul) {
3021 default:
3022 llvm_unreachable("Invalid LMUL.");
3023 case RISCVVType::LMUL_F8:
3024 case RISCVVType::LMUL_F4:
3025 case RISCVVType::LMUL_F2:
3026 case RISCVVType::LMUL_1:
3027 return RISCV::VRRegClassID;
3028 case RISCVVType::LMUL_2:
3029 return RISCV::VRM2RegClassID;
3030 case RISCVVType::LMUL_4:
3031 return RISCV::VRM4RegClassID;
3032 case RISCVVType::LMUL_8:
3033 return RISCV::VRM8RegClassID;
3034 }
3035}
3036
3037unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
3038 RISCVVType::VLMUL LMUL = getLMUL(VT);
3039 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
3040 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
3041 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
3042 "Unexpected subreg numbering");
3043 return RISCV::sub_vrm1_0 + Index;
3044 }
3045 if (LMUL == RISCVVType::LMUL_2) {
3046 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
3047 "Unexpected subreg numbering");
3048 return RISCV::sub_vrm2_0 + Index;
3049 }
3050 if (LMUL == RISCVVType::LMUL_4) {
3051 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
3052 "Unexpected subreg numbering");
3053 return RISCV::sub_vrm4_0 + Index;
3054 }
3055 llvm_unreachable("Invalid vector type.");
3056}
3057
3058unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
3059 if (VT.isRISCVVectorTuple()) {
3060 unsigned NF = VT.getRISCVVectorTupleNumFields();
3061 unsigned RegsPerField =
3062 std::max(1U, (unsigned)VT.getSizeInBits().getKnownMinValue() /
3063 (NF * RISCV::RVVBitsPerBlock));
3064 switch (RegsPerField) {
3065 case 1:
3066 if (NF == 2)
3067 return RISCV::VRN2M1RegClassID;
3068 if (NF == 3)
3069 return RISCV::VRN3M1RegClassID;
3070 if (NF == 4)
3071 return RISCV::VRN4M1RegClassID;
3072 if (NF == 5)
3073 return RISCV::VRN5M1RegClassID;
3074 if (NF == 6)
3075 return RISCV::VRN6M1RegClassID;
3076 if (NF == 7)
3077 return RISCV::VRN7M1RegClassID;
3078 if (NF == 8)
3079 return RISCV::VRN8M1RegClassID;
3080 break;
3081 case 2:
3082 if (NF == 2)
3083 return RISCV::VRN2M2RegClassID;
3084 if (NF == 3)
3085 return RISCV::VRN3M2RegClassID;
3086 if (NF == 4)
3087 return RISCV::VRN4M2RegClassID;
3088 break;
3089 case 4:
3090 assert(NF == 2);
3091 return RISCV::VRN2M4RegClassID;
3092 default:
3093 break;
3094 }
3095 llvm_unreachable("Invalid vector tuple type RegClass.");
3096 }
3097
3098 if (VT.getVectorElementType() == MVT::i1)
3099 return RISCV::VRRegClassID;
3100 return getRegClassIDForLMUL(getLMUL(VT));
3101}
3102
3103// Attempt to decompose a subvector insert/extract between VecVT and
3104// SubVecVT via subregister indices. Returns the subregister index that
3105// can perform the subvector insert/extract with the given element index, as
3106// well as the index corresponding to any leftover subvectors that must be
3107// further inserted/extracted within the register class for SubVecVT.
3108std::pair<unsigned, unsigned>
3109RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3110 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3111 const RISCVRegisterInfo *TRI) {
3112 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3113 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3114 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3115 "Register classes not ordered");
3116 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
3117 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
3118
3119 // If VecVT is a vector tuple type, either it's the tuple type with same
3120 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3121 if (VecVT.isRISCVVectorTuple()) {
3122 if (VecRegClassID == SubRegClassID)
3123 return {RISCV::NoSubRegister, 0};
3124
3125 assert(SubVecVT.isScalableVector() &&
3126 "Only allow scalable vector subvector.");
3127 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3128 "Invalid vector tuple insert/extract for vector and subvector with "
3129 "different LMUL.");
3130 return {getSubregIndexByMVT(VecVT, InsertExtractIdx), 0};
3131 }
3132
3133 // Try to compose a subregister index that takes us from the incoming
3134 // LMUL>1 register class down to the outgoing one. At each step we half
3135 // the LMUL:
3136 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3137 // Note that this is not guaranteed to find a subregister index, such as
3138 // when we are extracting from one VR type to another.
3139 unsigned SubRegIdx = RISCV::NoSubRegister;
3140 for (const unsigned RCID :
3141 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3142 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3143 VecVT = VecVT.getHalfNumVectorElementsVT();
3144 bool IsHi =
3145 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3146 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
3147 getSubregIndexByMVT(VecVT, IsHi));
3148 if (IsHi)
3149 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3150 }
3151 return {SubRegIdx, InsertExtractIdx};
3152}
3153
3154// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3155// stores for those types.
3156bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3157 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
3158 (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
3159 return false;
3160
3161 return !Subtarget.useRVVForFixedLengthVectors() ||
3162 VT.isFixedLengthVectorOf(MVT::i1);
3163}
3164
3165bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3166 if (!ScalarTy.isSimple())
3167 return false;
3168 switch (ScalarTy.getSimpleVT().SimpleTy) {
3169 case MVT::iPTR:
3170 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3171 case MVT::i8:
3172 case MVT::i16:
3173 case MVT::i32:
3174 return Subtarget.hasVInstructions();
3175 case MVT::i64:
3176 return Subtarget.hasVInstructionsI64();
3177 case MVT::f16:
3178 return Subtarget.hasVInstructionsF16Minimal();
3179 case MVT::bf16:
3180 return Subtarget.hasVInstructionsBF16Minimal();
3181 case MVT::f32:
3182 return Subtarget.hasVInstructionsF32();
3183 case MVT::f64:
3184 return Subtarget.hasVInstructionsF64();
3185 default:
3186 return false;
3187 }
3188}
3189
3190
3191unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3192 return NumRepeatedDivisors;
3193}
3194
3195static SDValue getVLOperand(SDValue Op) {
3196 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3197 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3198 "Unexpected opcode");
3199 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3200 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
3201 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3202 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
3203 if (!II)
3204 return SDValue();
3205 return Op.getOperand(II->VLOperand + 1 + HasChain);
3206}
3207
3208static bool useRVVForFixedLengthVectorVT(MVT VT,
3209 const RISCVSubtarget &Subtarget) {
3210 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3211 if (!Subtarget.useRVVForFixedLengthVectors())
3212 return false;
3213
3214 // We only support a set of vector types with a consistent maximum fixed size
3215 // across all supported vector element types to avoid legalization issues.
3216 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3217 // fixed-length vector type we support is 1024 bytes.
3218 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3219 return false;
3220
3221 unsigned MinVLen = Subtarget.getRealMinVLen();
3222
3223 MVT EltVT = VT.getVectorElementType();
3224
3225 // Don't use RVV for vectors we cannot scalarize if required.
3226 switch (EltVT.SimpleTy) {
3227 // i1 is supported but has different rules.
3228 default:
3229 return false;
3230 case MVT::i1:
3231 // Masks can only use a single register.
3232 if (VT.getVectorNumElements() > MinVLen)
3233 return false;
3234 MinVLen /= 8;
3235 break;
3236 case MVT::i8:
3237 case MVT::i16:
3238 case MVT::i32:
3239 break;
3240 case MVT::i64:
3241 if (!Subtarget.hasVInstructionsI64())
3242 return false;
3243 break;
3244 case MVT::f16:
3245 if (!Subtarget.hasVInstructionsF16Minimal())
3246 return false;
3247 break;
3248 case MVT::bf16:
3249 if (!Subtarget.hasVInstructionsBF16Minimal())
3250 return false;
3251 break;
3252 case MVT::f32:
3253 if (!Subtarget.hasVInstructionsF32())
3254 return false;
3255 break;
3256 case MVT::f64:
3257 if (!Subtarget.hasVInstructionsF64())
3258 return false;
3259 break;
3260 }
3261
3262 // Reject elements larger than ELEN.
3263 if (EltVT.getSizeInBits() > Subtarget.getELen())
3264 return false;
3265
3266 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
3267 // Don't use RVV for types that don't fit.
3268 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3269 return false;
3270
3271 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3272 // the base fixed length RVV support in place.
3273 if (!VT.isPow2VectorType())
3274 return false;
3275
3276 return true;
3277}
3278
3279bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3280 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3281}
3282
3283// Return the largest legal scalable vector type that matches VT's element type.
3284static MVT getContainerForFixedLengthVector(MVT VT,
3285 const RISCVSubtarget &Subtarget) {
3286 // This may be called before legal types are setup.
3287 assert(((VT.isFixedLengthVector() &&
3288 Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
3289 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3290 "Expected legal fixed length vector!");
3291
3292 unsigned MinVLen = Subtarget.getRealMinVLen();
3293 unsigned MaxELen = Subtarget.getELen();
3294
3295 MVT EltVT = VT.getVectorElementType();
3296 switch (EltVT.SimpleTy) {
3297 default:
3298 llvm_unreachable("unexpected element type for RVV container");
3299 case MVT::i1:
3300 case MVT::i8:
3301 case MVT::i16:
3302 case MVT::i32:
3303 case MVT::i64:
3304 case MVT::bf16:
3305 case MVT::f16:
3306 case MVT::f32:
3307 case MVT::f64: {
3308 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3309 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3310 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3311 unsigned NumElts =
3312 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3313 NumElts = std::max(NumElts, RISCV::RVVBitsPerBlock / MaxELen);
3314 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3315 return MVT::getScalableVectorVT(EltVT, NumElts);
3316 }
3317 }
3318}
3319
3320MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3321 return ::getContainerForFixedLengthVector(VT, getSubtarget());
3322}
3323
3324// Grow V to consume an entire RVV register.
3325static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3326 const RISCVSubtarget &Subtarget) {
3327 assert(VT.isScalableVector() &&
3328 "Expected to convert into a scalable vector!");
3329 assert(V.getValueType().isFixedLengthVector() &&
3330 "Expected a fixed length vector operand!");
3331 SDLoc DL(V);
3332 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), V, 0);
3333}
3334
3335// Shrink V so it's just big enough to maintain a VT's worth of data.
3336static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3337 const RISCVSubtarget &Subtarget) {
3338 assert(VT.isFixedLengthVector() &&
3339 "Expected to convert into a fixed length vector!");
3340 assert(V.getValueType().isScalableVector() &&
3341 "Expected a scalable vector operand!");
3342 SDLoc DL(V);
3343 return DAG.getExtractSubvector(DL, VT, V, 0);
3344}
3345
3346/// Return the type of the mask type suitable for masking the provided
3347/// vector type. This is simply an i1 element type vector of the same
3348/// (possibly scalable) length.
3349static MVT getMaskTypeFor(MVT VecVT) {
3350 assert(VecVT.isVector());
3351 ElementCount EC = VecVT.getVectorElementCount();
3352 return MVT::getVectorVT(MVT::i1, EC);
3353}
3354
3355/// Creates an all ones mask suitable for masking a vector of type VecTy with
3356/// vector length VL. .
3357static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3358 SelectionDAG &DAG) {
3359 MVT MaskVT = getMaskTypeFor(VecVT);
3360 return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
3361}
3362
3363static std::pair<SDValue, SDValue>
3364getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3365 const RISCVSubtarget &Subtarget) {
3366 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3367 SDValue VL = DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
3368 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3369 return {Mask, VL};
3370}
3371
3372static std::pair<SDValue, SDValue>
3373getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3374 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3375 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3376 SDValue VL = DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
3377 SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
3378 return {Mask, VL};
3379}
3380
3381// Gets the two common "VL" operands: an all-ones mask and the vector length.
3382// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3383// the vector type that the fixed-length vector is contained in. Otherwise if
3384// VecVT is scalable, then ContainerVT should be the same as VecVT.
3385static std::pair<SDValue, SDValue>
3386getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3387 const RISCVSubtarget &Subtarget) {
3388 if (VecVT.isFixedLengthVector())
3389 return getDefaultVLOps(VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3390 Subtarget);
3391 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3392 return getDefaultScalableVLOps(ContainerVT, DL, DAG, Subtarget);
3393}
3394
3395SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3396 SelectionDAG &DAG) const {
3397 assert(VecVT.isScalableVector() && "Expected scalable vector");
3398 return DAG.getElementCount(DL, Subtarget.getXLenVT(),
3399 VecVT.getVectorElementCount());
3400}
3401
3402std::pair<unsigned, unsigned>
3403RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3404 const RISCVSubtarget &Subtarget) {
3405 assert(VecVT.isScalableVector() && "Expected scalable vector");
3406
3407 unsigned EltSize = VecVT.getScalarSizeInBits();
3408 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3409
3410 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3411 unsigned MaxVLMAX =
3412 RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
3413
3414 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3415 unsigned MinVLMAX =
3416 RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
3417
3418 return std::make_pair(MinVLMAX, MaxVLMAX);
3419}
3420
3421// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3422// of either is (currently) supported. This can get us into an infinite loop
3423// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3424// as a ..., etc.
3425// Until either (or both) of these can reliably lower any node, reporting that
3426// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3427// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3428// which is not desirable.
3429bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3430 EVT VT, unsigned DefinedValues) const {
3431 return false;
3432}
3433
3434InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3435 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3436 // implementation-defined.
3437 if (!VT.isVector())
3438 return InstructionCost::getInvalid();
3439 unsigned DLenFactor = Subtarget.getDLenFactor();
3440 unsigned Cost;
3441 if (VT.isScalableVector()) {
3442 unsigned LMul;
3443 bool Fractional;
3444 std::tie(LMul, Fractional) =
3445 RISCVVType::decodeVLMUL(RISCVTargetLowering::getLMUL(VT));
3446 if (Fractional)
3447 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3448 else
3449 Cost = (LMul * DLenFactor);
3450 } else {
3451 Cost = divideCeil(VT.getSizeInBits(), Subtarget.getRealMinVLen() / DLenFactor);
3452 }
3453 return Cost;
3454}
3455
3456
3457/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3458/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3459/// be by default. VRGatherCostModel reflects available options. Note that
3460/// operand (index and possibly mask) are handled separately.
3461InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3462 auto LMULCost = getLMULCost(VT);
3463 bool Log2CostModel =
3464 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3465 if (Log2CostModel && LMULCost.isValid()) {
3466 unsigned Log = Log2_64(LMULCost.getValue());
3467 if (Log > 0)
3468 return LMULCost * Log;
3469 }
3470 return LMULCost * LMULCost;
3471}
3472
3473/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3474/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3475/// or may track the vrgather.vv cost. It is implementation-dependent.
3479
3480/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3481/// for the type VT. (This does not cover the vslide1up or vslide1down
3482/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3483/// or may track the vrgather.vv cost. It is implementation-dependent.
3487
3488/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3489/// for the type VT. (This does not cover the vslide1up or vslide1down
3490/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3491/// or may track the vrgather.vv cost. It is implementation-dependent.
3495
3496static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3497 const RISCVSubtarget &Subtarget) {
3498 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3499 // bf16 conversions are always promoted to f32.
3500 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3501 Op.getValueType() == MVT::bf16) {
3502 bool IsStrict = Op->isStrictFPOpcode();
3503
3504 SDLoc DL(Op);
3505 if (IsStrict) {
3506 SDValue Val = DAG.getNode(Op.getOpcode(), DL, {MVT::f32, MVT::Other},
3507 {Op.getOperand(0), Op.getOperand(1)});
3508 return DAG.getNode(ISD::STRICT_FP_ROUND, DL,
3509 {Op.getValueType(), MVT::Other},
3510 {Val.getValue(1), Val.getValue(0),
3511 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)});
3512 }
3513 return DAG.getNode(
3514 ISD::FP_ROUND, DL, Op.getValueType(),
3515 DAG.getNode(Op.getOpcode(), DL, MVT::f32, Op.getOperand(0)),
3516 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
3517 }
3518
3519 // Other operations are legal.
3520 return Op;
3521}
3522
3523static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3524 const RISCVSubtarget &Subtarget) {
3525 // RISC-V FP-to-int conversions saturate to the destination register size, but
3526 // don't produce 0 for nan. We can use a conversion instruction and fix the
3527 // nan case with a compare and a select.
3528 SDValue Src = Op.getOperand(0);
3529
3530 MVT DstVT = Op.getSimpleValueType();
3531 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3532
3533 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3534
3535 if (!DstVT.isVector()) {
3536 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3537 // the result.
3538 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3539 Src.getValueType() == MVT::bf16) {
3540 Src = DAG.getNode(ISD::FP_EXTEND, SDLoc(Op), MVT::f32, Src);
3541 }
3542
3543 unsigned Opc;
3544 if (SatVT == DstVT)
3545 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3546 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3547 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3548 else
3549 return SDValue();
3550 // FIXME: Support other SatVTs by clamping before or after the conversion.
3551
3552 SDLoc DL(Op);
3553 SDValue FpToInt = DAG.getNode(
3554 Opc, DL, DstVT, Src,
3555 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT()));
3556
3557 if (Opc == RISCVISD::FCVT_WU_RV64)
3558 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
3559
3560 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
3561 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt,
3562 ISD::CondCode::SETUO);
3563 }
3564
3565 // Vectors.
3566
3567 MVT DstEltVT = DstVT.getVectorElementType();
3568 MVT SrcVT = Src.getSimpleValueType();
3569 MVT SrcEltVT = SrcVT.getVectorElementType();
3570 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3571 unsigned DstEltSize = DstEltVT.getSizeInBits();
3572
3573 // Only handle saturating to the destination type.
3574 if (SatVT != DstEltVT)
3575 return SDValue();
3576
3577 MVT DstContainerVT = DstVT;
3578 MVT SrcContainerVT = SrcVT;
3579 if (DstVT.isFixedLengthVector()) {
3580 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
3581 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
3582 assert(DstContainerVT.getVectorElementCount() ==
3583 SrcContainerVT.getVectorElementCount() &&
3584 "Expected same element count");
3585 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3586 }
3587
3588 SDLoc DL(Op);
3589
3590 auto [Mask, VL] = getDefaultVLOps(DstVT, DstContainerVT, DL, DAG, Subtarget);
3591
3592 SDValue IsNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
3593 {Src, Src, DAG.getCondCode(ISD::SETNE),
3594 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
3595
3596 // Need to widen by more than 1 step, promote the FP type, then do a widening
3597 // convert.
3598 if (DstEltSize > (2 * SrcEltSize)) {
3599 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3600 MVT InterVT = SrcContainerVT.changeVectorElementType(MVT::f32);
3601 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, InterVT, Src, Mask, VL);
3602 }
3603
3604 MVT CvtContainerVT = DstContainerVT;
3605 MVT CvtEltVT = DstEltVT;
3606 if (SrcEltSize > (2 * DstEltSize)) {
3607 CvtEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
3608 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3609 }
3610
3611 unsigned RVVOpc =
3612 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3613 SDValue Res = DAG.getNode(RVVOpc, DL, CvtContainerVT, Src, Mask, VL);
3614
3615 while (CvtContainerVT != DstContainerVT) {
3616 CvtEltVT = MVT::getIntegerVT(CvtEltVT.getSizeInBits() / 2);
3617 CvtContainerVT = CvtContainerVT.changeVectorElementType(CvtEltVT);
3618 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3619 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3620 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3621 Res = DAG.getNode(ClipOpc, DL, CvtContainerVT, Res, Mask, VL);
3622 }
3623
3624 SDValue SplatZero = DAG.getNode(
3625 RISCVISD::VMV_V_X_VL, DL, DstContainerVT, DAG.getUNDEF(DstContainerVT),
3626 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
3627 Res = DAG.getNode(RISCVISD::VMERGE_VL, DL, DstContainerVT, IsNan, SplatZero,
3628 Res, DAG.getUNDEF(DstContainerVT), VL);
3629
3630 if (DstVT.isFixedLengthVector())
3631 Res = convertFromScalableVector(DstVT, Res, DAG, Subtarget);
3632
3633 return Res;
3634}
3635
3636static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3637 const RISCVSubtarget &Subtarget) {
3638 bool IsStrict = Op->isStrictFPOpcode();
3639 SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3640
3641 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3642 // bf16 conversions are always promoted to f32.
3643 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3644 SrcVal.getValueType() == MVT::bf16) {
3645 SDLoc DL(Op);
3646 if (IsStrict) {
3647 SDValue Ext =
3648 DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
3649 {Op.getOperand(0), SrcVal});
3650 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
3651 {Ext.getValue(1), Ext.getValue(0)});
3652 }
3653 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
3654 DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, SrcVal));
3655 }
3656
3657 // Other operations are legal.
3658 return Op;
3659}
3660
3661static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3662 switch (Opc) {
3663 case ISD::FROUNDEVEN:
3664 case ISD::STRICT_FROUNDEVEN:
3665 return RISCVFPRndMode::RNE;
3666 case ISD::FTRUNC:
3667 case ISD::STRICT_FTRUNC:
3668 return RISCVFPRndMode::RTZ;
3669 case ISD::FFLOOR:
3670 case ISD::STRICT_FFLOOR:
3671 return RISCVFPRndMode::RDN;
3672 case ISD::FCEIL:
3673 case ISD::STRICT_FCEIL:
3674 return RISCVFPRndMode::RUP;
3675 case ISD::FROUND:
3676 case ISD::LROUND:
3677 case ISD::LLROUND:
3678 case ISD::STRICT_FROUND:
3679 case ISD::STRICT_LROUND:
3680 case ISD::STRICT_LLROUND:
3681 return RISCVFPRndMode::RMM;
3682 case ISD::FRINT:
3683 case ISD::LRINT:
3684 case ISD::LLRINT:
3685 case ISD::STRICT_FRINT:
3686 case ISD::STRICT_LRINT:
3687 case ISD::STRICT_LLRINT:
3688 return RISCVFPRndMode::DYN;
3689 }
3690
3691 return RISCVFPRndMode::Invalid;
3692}
3693
3694// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
3695// the integer domain and back. Taking care to avoid converting values that are
3696// nan or already correct.
3697static SDValue
3698lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3699 const RISCVSubtarget &Subtarget) {
3700 MVT VT = Op.getSimpleValueType();
3701 assert(VT.isVector() && "Unexpected type");
3702
3703 SDLoc DL(Op);
3704
3705 SDValue Src = Op.getOperand(0);
3706
3707 // Freeze the source since we are increasing the number of uses.
3708 Src = DAG.getFreeze(Src);
3709
3710 MVT ContainerVT = VT;
3711 if (VT.isFixedLengthVector()) {
3712 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3713 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3714 }
3715
3716 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3717
3718 // We do the conversion on the absolute value and fix the sign at the end.
3719 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3720
3721 // Determine the largest integer that can be represented exactly. This and
3722 // values larger than it don't have any fractional bits so don't need to
3723 // be converted.
3724 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3725 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3726 APFloat MaxVal = APFloat(FltSem);
3727 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3728 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3729 SDValue MaxValNode =
3730 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3731 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3732 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3733
3734 // If abs(Src) was larger than MaxVal or nan, keep it.
3735 MVT SetccVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3736 Mask =
3737 DAG.getNode(RISCVISD::SETCC_VL, DL, SetccVT,
3738 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT),
3739 Mask, Mask, VL});
3740
3741 // Truncate to integer and convert back to FP.
3742 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3743 MVT XLenVT = Subtarget.getXLenVT();
3744 SDValue Truncated;
3745
3746 switch (Op.getOpcode()) {
3747 default:
3748 llvm_unreachable("Unexpected opcode");
3749 case ISD::FRINT:
3750 case ISD::FCEIL:
3751 case ISD::FFLOOR:
3752 case ISD::FROUND:
3753 case ISD::FROUNDEVEN: {
3754 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3755 assert(FRM != RISCVFPRndMode::Invalid);
3756 Truncated = DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, IntVT, Src, Mask,
3757 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
3758 break;
3759 }
3760 case ISD::FTRUNC:
3761 Truncated = DAG.getNode(RISCVISD::VFCVT_RTZ_X_F_VL, DL, IntVT, Src,
3762 Mask, VL);
3763 break;
3764 case ISD::FNEARBYINT:
3765 Truncated = DAG.getNode(RISCVISD::VFROUND_NOEXCEPT_VL, DL, ContainerVT, Src,
3766 Mask, VL);
3767 break;
3768 }
3769
3770 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3771 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3772 Truncated = DAG.getNode(RISCVISD::SINT_TO_FP_VL, DL, ContainerVT, Truncated,
3773 Mask, VL);
3774
3775 // Restore the original sign so that -0.0 is preserved.
3776 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3777 Src, Src, Mask, VL);
3778
3779 if (!VT.isFixedLengthVector())
3780 return Truncated;
3781
3782 return convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3783}
3784
3785// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3786// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3787// qNan and converting the new source to integer and back to FP.
3788static SDValue
3789lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3790 const RISCVSubtarget &Subtarget) {
3791 SDLoc DL(Op);
3792 MVT VT = Op.getSimpleValueType();
3793 SDValue Chain = Op.getOperand(0);
3794 SDValue Src = Op.getOperand(1);
3795
3796 MVT ContainerVT = VT;
3797 if (VT.isFixedLengthVector()) {
3798 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3799 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3800 }
3801
3802 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3803
3804 // Freeze the source since we are increasing the number of uses.
3805 Src = DAG.getFreeze(Src);
3806
3807 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3808 MVT MaskVT = Mask.getSimpleValueType();
3809 SDValue Unorder = DAG.getNode(RISCVISD::STRICT_FSETCC_VL, DL,
3810 DAG.getVTList(MaskVT, MVT::Other),
3811 {Chain, Src, Src, DAG.getCondCode(ISD::SETUNE),
3812 DAG.getUNDEF(MaskVT), Mask, VL});
3813 Chain = Unorder.getValue(1);
3814 Src = DAG.getNode(RISCVISD::STRICT_FADD_VL, DL,
3815 DAG.getVTList(ContainerVT, MVT::Other),
3816 {Chain, Src, Src, Src, Unorder, VL});
3817 Chain = Src.getValue(1);
3818
3819 // We do the conversion on the absolute value and fix the sign at the end.
3820 SDValue Abs = DAG.getNode(RISCVISD::FABS_VL, DL, ContainerVT, Src, Mask, VL);
3821
3822 // Determine the largest integer that can be represented exactly. This and
3823 // values larger than it don't have any fractional bits so don't need to
3824 // be converted.
3825 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3826 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3827 APFloat MaxVal = APFloat(FltSem);
3828 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3829 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3830 SDValue MaxValNode =
3831 DAG.getConstantFP(MaxVal, DL, ContainerVT.getVectorElementType());
3832 SDValue MaxValSplat = DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, ContainerVT,
3833 DAG.getUNDEF(ContainerVT), MaxValNode, VL);
3834
3835 // If abs(Src) was larger than MaxVal or nan, keep it.
3836 Mask = DAG.getNode(
3837 RISCVISD::SETCC_VL, DL, MaskVT,
3838 {Abs, MaxValSplat, DAG.getCondCode(ISD::SETOLT), Mask, Mask, VL});
3839
3840 // Truncate to integer and convert back to FP.
3841 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3842 MVT XLenVT = Subtarget.getXLenVT();
3843 SDValue Truncated;
3844
3845 switch (Op.getOpcode()) {
3846 default:
3847 llvm_unreachable("Unexpected opcode");
3848 case ISD::STRICT_FCEIL:
3849 case ISD::STRICT_FFLOOR:
3850 case ISD::STRICT_FROUND:
3851 case ISD::STRICT_FROUNDEVEN: {
3852 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3853 assert(FRM != RISCVFPRndMode::Invalid);
3854 Truncated = DAG.getNode(
3855 RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, DAG.getVTList(IntVT, MVT::Other),
3856 {Chain, Src, Mask, DAG.getTargetConstant(FRM, DL, XLenVT), VL});
3857 break;
3858 }
3859 case ISD::STRICT_FTRUNC:
3860 Truncated =
3861 DAG.getNode(RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3862 DAG.getVTList(IntVT, MVT::Other), Chain, Src, Mask, VL);
3863 break;
3864 case ISD::STRICT_FNEARBYINT:
3865 Truncated = DAG.getNode(RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3866 DAG.getVTList(ContainerVT, MVT::Other), Chain, Src,
3867 Mask, VL);
3868 break;
3869 }
3870 Chain = Truncated.getValue(1);
3871
3872 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3873 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3874 Truncated = DAG.getNode(RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3875 DAG.getVTList(ContainerVT, MVT::Other), Chain,
3876 Truncated, Mask, VL);
3877 Chain = Truncated.getValue(1);
3878 }
3879
3880 // Restore the original sign so that -0.0 is preserved.
3881 Truncated = DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Truncated,
3882 Src, Src, Mask, VL);
3883
3884 if (VT.isFixedLengthVector())
3885 Truncated = convertFromScalableVector(VT, Truncated, DAG, Subtarget);
3886 return DAG.getMergeValues({Truncated, Chain}, DL);
3887}
3888
3889static SDValue
3890lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3891 const RISCVSubtarget &Subtarget) {
3892 MVT VT = Op.getSimpleValueType();
3893 if (VT.isVector())
3894 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
3895
3896 if (DAG.shouldOptForSize())
3897 return SDValue();
3898
3899 SDLoc DL(Op);
3900 SDValue Src = Op.getOperand(0);
3901
3902 // Create an integer the size of the mantissa with the MSB set. This and all
3903 // values larger than it don't have any fractional bits so don't need to be
3904 // converted.
3905 const fltSemantics &FltSem = VT.getFltSemantics();
3906 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3907 APFloat MaxVal = APFloat(FltSem);
3908 MaxVal.convertFromAPInt(APInt::getOneBitSet(Precision, Precision - 1),
3909 /*IsSigned*/ false, APFloat::rmNearestTiesToEven);
3910 SDValue MaxValNode = DAG.getConstantFP(MaxVal, DL, VT);
3911
3912 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Op.getOpcode());
3913 return DAG.getNode(RISCVISD::FROUND, DL, VT, Src, MaxValNode,
3914 DAG.getTargetConstant(FRM, DL, Subtarget.getXLenVT()));
3915}
3916
3917// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
3918static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
3919 const RISCVSubtarget &Subtarget) {
3920 SDLoc DL(Op);
3921 MVT DstVT = Op.getSimpleValueType();
3922 SDValue Src = Op.getOperand(0);
3923 MVT SrcVT = Src.getSimpleValueType();
3924 assert(SrcVT.isVector() && DstVT.isVector() &&
3925 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
3926 "Unexpected type");
3927
3928 MVT DstContainerVT = DstVT;
3929 MVT SrcContainerVT = SrcVT;
3930
3931 if (DstVT.isFixedLengthVector()) {
3932 DstContainerVT = getContainerForFixedLengthVector(DstVT, Subtarget);
3933 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
3934 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
3935 }
3936
3937 auto [Mask, VL] = getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
3938
3939 // [b]f16 -> f32
3940 MVT SrcElemType = SrcVT.getVectorElementType();
3941 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
3942 MVT F32VT = SrcContainerVT.changeVectorElementType(MVT::f32);
3943 Src = DAG.getNode(RISCVISD::FP_EXTEND_VL, DL, F32VT, Src, Mask, VL);
3944 }
3945
3946 SDValue Res =
3947 DAG.getNode(RISCVISD::VFCVT_RM_X_F_VL, DL, DstContainerVT, Src, Mask,
3948 DAG.getTargetConstant(matchRoundingOp(Op.getOpcode()), DL,
3949 Subtarget.getXLenVT()),
3950 VL);
3951
3952 if (!DstVT.isFixedLengthVector())
3953 return Res;
3954
3955 return convertFromScalableVector(DstVT, Res, DAG, Subtarget);
3956}
3957
3958static SDValue
3959getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
3960 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
3961 SDValue Offset, SDValue Mask, SDValue VL,
3962 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3963 if (Passthru.isUndef())
3964 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
3965 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3966 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3967 return DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3968}
3969
3970static SDValue
3971getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
3972 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
3973 SDValue VL,
3974 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3975 if (Passthru.isUndef())
3976 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
3977 SDValue PolicyOp = DAG.getTargetConstant(Policy, DL, Subtarget.getXLenVT());
3978 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3979 return DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3980}
3981
3982struct VIDSequence {
3983 int64_t StepNumerator;
3984 unsigned StepDenominator;
3985 int64_t Addend;
3986};
3987
3988static std::optional<APInt> getExactInteger(const APFloat &APF,
3989 uint32_t BitWidth) {
3990 // We will use a SINT_TO_FP to materialize this constant so we should use a
3991 // signed APSInt here.
3992 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
3993 // We use an arbitrary rounding mode here. If a floating-point is an exact
3994 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
3995 // the rounding mode changes the output value, then it is not an exact
3996 // integer.
3997 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
3998 bool IsExact;
3999 // If it is out of signed integer range, it will return an invalid operation.
4000 // If it is not an exact integer, IsExact is false.
4001 if ((APF.convertToInteger(ValInt, ArbitraryRM, &IsExact) ==
4002 APFloatBase::opInvalidOp) ||
4003 !IsExact)
4004 return std::nullopt;
4005 return ValInt.extractBits(BitWidth, 0);
4006}
4007
4008// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
4009// to the (non-zero) step S and start value X. This can be then lowered as the
4010// RVV sequence (VID * S) + X, for example.
4011// The step S is represented as an integer numerator divided by a positive
4012// denominator. Note that the implementation currently only identifies
4013// sequences in which either the numerator is +/- 1 or the denominator is 1. It
4014// cannot detect 2/3, for example.
4015// Note that this method will also match potentially unappealing index
4016// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
4017// determine whether this is worth generating code for.
4018//
4019// EltSizeInBits is the size of the type that the sequence will be calculated
4020// in, i.e. SEW for build_vectors or XLEN for address calculations.
4021static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
4022 unsigned EltSizeInBits) {
4023 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
4024 if (!cast<BuildVectorSDNode>(Op)->isConstant())
4025 return std::nullopt;
4026 bool IsInteger = Op.getValueType().isInteger();
4027
4028 std::optional<unsigned> SeqStepDenom;
4029 std::optional<APInt> SeqStepNum;
4030 std::optional<APInt> SeqAddend;
4031 std::optional<std::pair<APInt, unsigned>> PrevElt;
4032 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
4033
4034 // First extract the ops into a list of constant integer values. This may not
4035 // be possible for floats if they're not all representable as integers.
4036 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
4037 const unsigned OpSize = Op.getScalarValueSizeInBits();
4038 for (auto [Idx, Elt] : enumerate(Op->op_values())) {
4039 if (Elt.isUndef()) {
4040 Elts[Idx] = std::nullopt;
4041 continue;
4042 }
4043 if (IsInteger) {
4044 Elts[Idx] = Elt->getAsAPIntVal().trunc(OpSize).zext(EltSizeInBits);
4045 } else {
4046 auto ExactInteger =
4047 getExactInteger(cast<ConstantFPSDNode>(Elt)->getValueAPF(), OpSize);
4048 if (!ExactInteger)
4049 return std::nullopt;
4050 Elts[Idx] = *ExactInteger;
4051 }
4052 }
4053
4054 for (auto [Idx, Elt] : enumerate(Elts)) {
4055 // Assume undef elements match the sequence; we just have to be careful
4056 // when interpolating across them.
4057 if (!Elt)
4058 continue;
4059
4060 if (PrevElt) {
4061 // Calculate the step since the last non-undef element, and ensure
4062 // it's consistent across the entire sequence.
4063 unsigned IdxDiff = Idx - PrevElt->second;
4064 APInt ValDiff = *Elt - PrevElt->first;
4065
4066 // A zero-value value difference means that we're somewhere in the middle
4067 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4068 // step change before evaluating the sequence.
4069 if (ValDiff == 0)
4070 continue;
4071
4072 int64_t Remainder = ValDiff.srem(IdxDiff);
4073 // Normalize the step if it's greater than 1.
4074 if (Remainder != ValDiff.getSExtValue()) {
4075 // The difference must cleanly divide the element span.
4076 if (Remainder != 0)
4077 return std::nullopt;
4078 ValDiff = ValDiff.sdiv(IdxDiff);
4079 IdxDiff = 1;
4080 }
4081
4082 if (!SeqStepNum)
4083 SeqStepNum = ValDiff;
4084 else if (ValDiff != SeqStepNum)
4085 return std::nullopt;
4086
4087 if (!SeqStepDenom)
4088 SeqStepDenom = IdxDiff;
4089 else if (IdxDiff != *SeqStepDenom)
4090 return std::nullopt;
4091 }
4092
4093 // Record this non-undef element for later.
4094 if (!PrevElt || PrevElt->first != *Elt)
4095 PrevElt = std::make_pair(*Elt, Idx);
4096 }
4097
4098 // We need to have logged a step for this to count as a legal index sequence.
4099 if (!SeqStepNum || !SeqStepDenom)
4100 return std::nullopt;
4101
4102 // Loop back through the sequence and validate elements we might have skipped
4103 // while waiting for a valid step. While doing this, log any sequence addend.
4104 for (auto [Idx, Elt] : enumerate(Elts)) {
4105 if (!Elt)
4106 continue;
4107 APInt ExpectedVal =
4108 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4109 *SeqStepNum)
4110 .sdiv(*SeqStepDenom);
4111
4112 APInt Addend = *Elt - ExpectedVal;
4113 if (!SeqAddend)
4114 SeqAddend = Addend;
4115 else if (Addend != SeqAddend)
4116 return std::nullopt;
4117 }
4118
4119 assert(SeqAddend && "Must have an addend if we have a step");
4120
4121 return VIDSequence{SeqStepNum->getSExtValue(), *SeqStepDenom,
4122 SeqAddend->getSExtValue()};
4123}
4124
4125// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4126// and lower it as a VRGATHER_VX_VL from the source vector.
4127static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4128 SelectionDAG &DAG,
4129 const RISCVSubtarget &Subtarget) {
4130 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4131 return SDValue();
4132 SDValue Src = SplatVal.getOperand(0);
4133 // Don't perform this optimization for i1 vectors, or if the element types are
4134 // different
4135 // FIXME: Support i1 vectors, maybe by promoting to i8?
4136 MVT EltTy = VT.getVectorElementType();
4137 if (EltTy == MVT::i1 ||
4138 !DAG.getTargetLoweringInfo().isTypeLegal(Src.getValueType()))
4139 return SDValue();
4140 MVT SrcVT = Src.getSimpleValueType();
4141 if (EltTy != SrcVT.getVectorElementType())
4142 return SDValue();
4143 SDValue Idx = SplatVal.getOperand(1);
4144 // The index must be a legal type.
4145 if (Idx.getValueType() != Subtarget.getXLenVT())
4146 return SDValue();
4147
4148 // Check that we know Idx lies within VT
4149 if (!TypeSize::isKnownLE(SrcVT.getSizeInBits(), VT.getSizeInBits())) {
4150 auto *CIdx = dyn_cast<ConstantSDNode>(Idx);
4151 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4152 return SDValue();
4153 }
4154
4155 // Convert fixed length vectors to scalable
4156 MVT ContainerVT = VT;
4157 if (VT.isFixedLengthVector())
4158 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4159
4160 MVT SrcContainerVT = SrcVT;
4161 if (SrcVT.isFixedLengthVector()) {
4162 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
4163 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
4164 }
4165
4166 // Put Vec in a VT sized vector
4167 if (SrcContainerVT.getVectorMinNumElements() <
4168 ContainerVT.getVectorMinNumElements())
4169 Src = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), Src, 0);
4170 else
4171 Src = DAG.getExtractSubvector(DL, ContainerVT, Src, 0);
4172
4173 // We checked that Idx fits inside VT earlier
4174 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4175 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, Src,
4176 Idx, DAG.getUNDEF(ContainerVT), Mask, VL);
4177 if (VT.isFixedLengthVector())
4178 Gather = convertFromScalableVector(VT, Gather, DAG, Subtarget);
4179 return Gather;
4180}
4181
4182static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4183 const RISCVSubtarget &Subtarget) {
4184 MVT VT = Op.getSimpleValueType();
4185 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4186
4187 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4188
4189 SDLoc DL(Op);
4190 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4191
4192 if (auto SimpleVID = isSimpleVIDSequence(Op, Op.getScalarValueSizeInBits())) {
4193 int64_t StepNumerator = SimpleVID->StepNumerator;
4194 unsigned StepDenominator = SimpleVID->StepDenominator;
4195 int64_t Addend = SimpleVID->Addend;
4196
4197 assert(StepNumerator != 0 && "Invalid step");
4198 bool Negate = false;
4199 int64_t SplatStepVal = StepNumerator;
4200 unsigned StepOpcode = ISD::MUL;
4201 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4202 // anyway as the shift of 63 won't fit in uimm5.
4203 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4204 isPowerOf2_64(std::abs(StepNumerator))) {
4205 Negate = StepNumerator < 0;
4206 StepOpcode = ISD::SHL;
4207 SplatStepVal = Log2_64(std::abs(StepNumerator));
4208 }
4209
4210 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4211 // since it's the immediate value many RVV instructions accept. There is
4212 // no vmul.vi instruction so ensure multiply constant can fit in a
4213 // single addi instruction. For the addend, we allow up to 32 bits..
4214 if (((StepOpcode == ISD::MUL && isInt<12>(SplatStepVal)) ||
4215 (StepOpcode == ISD::SHL && isUInt<5>(SplatStepVal))) &&
4216 isPowerOf2_32(StepDenominator) &&
4217 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(Addend)) {
4218 MVT VIDVT =
4219 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4220 MVT VIDContainerVT = getContainerForFixedLengthVector(VIDVT, Subtarget);
4221 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VIDContainerVT, Mask, VL);
4222 // Convert right out of the scalable type so we can use standard ISD
4223 // nodes for the rest of the computation. If we used scalable types with
4224 // these, we'd lose the fixed-length vector info and generate worse
4225 // vsetvli code.
4226 VID = convertFromScalableVector(VIDVT, VID, DAG, Subtarget);
4227 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4228 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4229 SDValue SplatStep = DAG.getSignedConstant(SplatStepVal, DL, VIDVT);
4230 VID = DAG.getNode(StepOpcode, DL, VIDVT, VID, SplatStep);
4231 }
4232 if (StepDenominator != 1) {
4233 SDValue SplatStep =
4234 DAG.getConstant(Log2_64(StepDenominator), DL, VIDVT);
4235 VID = DAG.getNode(ISD::SRL, DL, VIDVT, VID, SplatStep);
4236 }
4237 if (Addend != 0 || Negate) {
4238 SDValue SplatAddend = DAG.getSignedConstant(Addend, DL, VIDVT);
4239 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VIDVT, SplatAddend,
4240 VID);
4241 }
4242 if (VT.isFloatingPoint()) {
4243 // TODO: Use vfwcvt to reduce register pressure.
4244 VID = DAG.getNode(ISD::SINT_TO_FP, DL, VT, VID);
4245 }
4246 return VID;
4247 }
4248 }
4249
4250 return SDValue();
4251}
4252
4253/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4254/// which constitute a large proportion of the elements. In such cases we can
4255/// splat a vector with the dominant element and make up the shortfall with
4256/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4257/// Note that this includes vectors of 2 elements by association. The
4258/// upper-most element is the "dominant" one, allowing us to use a splat to
4259/// "insert" the upper element, and an insert of the lower element at position
4260/// 0, which improves codegen.
4261static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4262 const RISCVSubtarget &Subtarget) {
4263 MVT VT = Op.getSimpleValueType();
4264 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4265
4266 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4267
4268 SDLoc DL(Op);
4269 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4270
4271 MVT XLenVT = Subtarget.getXLenVT();
4272 unsigned NumElts = Op.getNumOperands();
4273
4274 SDValue DominantValue;
4275 unsigned MostCommonCount = 0;
4276 DenseMap<SDValue, unsigned> ValueCounts;
4277 unsigned NumUndefElts =
4278 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4279
4280 // Track the number of scalar loads we know we'd be inserting, estimated as
4281 // any non-zero floating-point constant. Other kinds of element are either
4282 // already in registers or are materialized on demand. The threshold at which
4283 // a vector load is more desirable than several scalar materializion and
4284 // vector-insertion instructions is not known.
4285 unsigned NumScalarLoads = 0;
4286
4287 for (SDValue V : Op->op_values()) {
4288 if (V.isUndef())
4289 continue;
4290
4291 unsigned &Count = ValueCounts[V];
4292 if (0 == Count)
4293 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
4294 NumScalarLoads += !CFP->isPosZero();
4295
4296 // Is this value dominant? In case of a tie, prefer the highest element as
4297 // it's cheaper to insert near the beginning of a vector than it is at the
4298 // end.
4299 if (++Count >= MostCommonCount) {
4300 DominantValue = V;
4301 MostCommonCount = Count;
4302 }
4303 }
4304
4305 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4306 unsigned NumDefElts = NumElts - NumUndefElts;
4307 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4308
4309 // Don't perform this optimization when optimizing for size, since
4310 // materializing elements and inserting them tends to cause code bloat.
4311 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4312 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) &&
4313 ((MostCommonCount > DominantValueCountThreshold) ||
4314 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
4315 // Start by splatting the most common element.
4316 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
4317
4318 DenseSet<SDValue> Processed{DominantValue};
4319
4320 // We can handle an insert into the last element (of a splat) via
4321 // v(f)slide1down. This is slightly better than the vslideup insert
4322 // lowering as it avoids the need for a vector group temporary. It
4323 // is also better than using vmerge.vx as it avoids the need to
4324 // materialize the mask in a vector register.
4325 if (SDValue LastOp = Op->getOperand(Op->getNumOperands() - 1);
4326 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4327 LastOp != DominantValue) {
4328 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4329 auto OpCode =
4330 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4331 if (!VT.isFloatingPoint())
4332 LastOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, LastOp);
4333 Vec = DAG.getNode(OpCode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
4334 LastOp, Mask, VL);
4335 Vec = convertFromScalableVector(VT, Vec, DAG, Subtarget);
4336 Processed.insert(LastOp);
4337 }
4338
4339 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
4340 for (const auto &OpIdx : enumerate(Op->ops())) {
4341 const SDValue &V = OpIdx.value();
4342 if (V.isUndef() || !Processed.insert(V).second)
4343 continue;
4344 if (ValueCounts[V] == 1) {
4345 Vec = DAG.getInsertVectorElt(DL, Vec, V, OpIdx.index());
4346 } else {
4347 // Blend in all instances of this value using a VSELECT, using a
4348 // mask where each bit signals whether that element is the one
4349 // we're after.
4350 SmallVector<SDValue> Ops;
4351 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
4352 return DAG.getConstant(V == V1, DL, XLenVT);
4353 });
4354 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
4355 DAG.getBuildVector(SelMaskTy, DL, Ops),
4356 DAG.getSplatBuildVector(VT, DL, V), Vec);
4357 }
4358 }
4359
4360 return Vec;
4361 }
4362
4363 return SDValue();
4364}
4365
4366static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4367 const RISCVSubtarget &Subtarget) {
4368 MVT VT = Op.getSimpleValueType();
4369 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4370
4371 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4372
4373 SDLoc DL(Op);
4374 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4375
4376 MVT XLenVT = Subtarget.getXLenVT();
4377 unsigned NumElts = Op.getNumOperands();
4378
4379 if (VT.getVectorElementType() == MVT::i1) {
4380 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
4381 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
4382 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
4383 }
4384
4385 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
4386 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
4387 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
4388 }
4389
4390 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4391 // scalar integer chunks whose bit-width depends on the number of mask
4392 // bits and XLEN.
4393 // First, determine the most appropriate scalar integer type to use. This
4394 // is at most XLenVT, but may be shrunk to a smaller vector element type
4395 // according to the size of the final vector - use i8 chunks rather than
4396 // XLenVT if we're producing a v8i1. This results in more consistent
4397 // codegen across RV32 and RV64.
4398 unsigned NumViaIntegerBits = std::clamp(NumElts, 8u, Subtarget.getXLen());
4399 NumViaIntegerBits = std::min(NumViaIntegerBits, Subtarget.getELen());
4400 // If we have to use more than one INSERT_VECTOR_ELT then this
4401 // optimization is likely to increase code size; avoid performing it in
4402 // such a case. We can use a load from a constant pool in this case.
4403 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4404 return SDValue();
4405 // Now we can create our integer vector type. Note that it may be larger
4406 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4407 unsigned IntegerViaVecElts = divideCeil(NumElts, NumViaIntegerBits);
4408 MVT IntegerViaVecVT =
4409 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
4410 IntegerViaVecElts);
4411
4412 uint64_t Bits = 0;
4413 unsigned BitPos = 0, IntegerEltIdx = 0;
4414 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4415
4416 for (unsigned I = 0; I < NumElts;) {
4417 SDValue V = Op.getOperand(I);
4418 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4419 Bits |= ((uint64_t)BitValue << BitPos);
4420 ++BitPos;
4421 ++I;
4422
4423 // Once we accumulate enough bits to fill our scalar type or process the
4424 // last element, insert into our vector and clear our accumulated data.
4425 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4426 if (NumViaIntegerBits <= 32)
4427 Bits = SignExtend64<32>(Bits);
4428 SDValue Elt = DAG.getSignedConstant(Bits, DL, XLenVT);
4429 Elts[IntegerEltIdx] = Elt;
4430 Bits = 0;
4431 BitPos = 0;
4432 IntegerEltIdx++;
4433 }
4434 }
4435
4436 SDValue Vec = DAG.getBuildVector(IntegerViaVecVT, DL, Elts);
4437
4438 if (NumElts < NumViaIntegerBits) {
4439 // If we're producing a smaller vector than our minimum legal integer
4440 // type, bitcast to the equivalent (known-legal) mask type, and extract
4441 // our final mask.
4442 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4443 Vec = DAG.getBitcast(MVT::v8i1, Vec);
4444 Vec = DAG.getExtractSubvector(DL, VT, Vec, 0);
4445 } else {
4446 // Else we must have produced an integer type with the same size as the
4447 // mask type; bitcast for the final result.
4448 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4449 Vec = DAG.getBitcast(VT, Vec);
4450 }
4451
4452 return Vec;
4453 }
4454
4455 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4456 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4457 : RISCVISD::VMV_V_X_VL;
4458 if (!VT.isFloatingPoint())
4459 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4460 Splat =
4461 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4462 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4463 }
4464
4465 // Try and match index sequences, which we can lower to the vid instruction
4466 // with optional modifications. An all-undef vector is matched by
4467 // getSplatValue, above.
4468 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4469 return Res;
4470
4471 // For very small build_vectors, use a single scalar insert of a constant.
4472 // TODO: Base this on constant rematerialization cost, not size.
4473 const unsigned EltBitSize = VT.getScalarSizeInBits();
4474 if (VT.getSizeInBits() <= 32 &&
4475 ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
4476 MVT ViaIntVT = MVT::getIntegerVT(VT.getSizeInBits());
4477 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4478 "Unexpected sequence type");
4479 // If we can use the original VL with the modified element type, this
4480 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4481 // be moved into InsertVSETVLI?
4482 unsigned ViaVecLen =
4483 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4484 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4485
4486 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4487 uint64_t SplatValue = 0;
4488 // Construct the amalgamated value at this larger vector type.
4489 for (const auto &OpIdx : enumerate(Op->op_values())) {
4490 const auto &SeqV = OpIdx.value();
4491 if (!SeqV.isUndef())
4492 SplatValue |=
4493 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4494 }
4495
4496 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4497 // achieve better constant materializion.
4498 // On RV32, we need to sign-extend to use getSignedConstant.
4499 if (ViaIntVT == MVT::i32)
4500 SplatValue = SignExtend64<32>(SplatValue);
4501
4502 SDValue Vec = DAG.getInsertVectorElt(
4503 DL, DAG.getUNDEF(ViaVecVT),
4504 DAG.getSignedConstant(SplatValue, DL, XLenVT), 0);
4505 if (ViaVecLen != 1)
4506 Vec = DAG.getExtractSubvector(DL, MVT::getVectorVT(ViaIntVT, 1), Vec, 0);
4507 return DAG.getBitcast(VT, Vec);
4508 }
4509
4510
4511 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4512 // when re-interpreted as a vector with a larger element type. For example,
4513 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4514 // could be instead splat as
4515 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4516 // TODO: This optimization could also work on non-constant splats, but it
4517 // would require bit-manipulation instructions to construct the splat value.
4518 SmallVector<SDValue> Sequence;
4519 const auto *BV = cast<BuildVectorSDNode>(Op);
4520 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4521 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
4522 BV->getRepeatedSequence(Sequence) &&
4523 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4524 unsigned SeqLen = Sequence.size();
4525 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
4526 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4527 ViaIntVT == MVT::i64) &&
4528 "Unexpected sequence type");
4529
4530 // If we can use the original VL with the modified element type, this
4531 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4532 // be moved into InsertVSETVLI?
4533 const unsigned RequiredVL = NumElts / SeqLen;
4534 const unsigned ViaVecLen =
4535 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4536 NumElts : RequiredVL;
4537 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, ViaVecLen);
4538
4539 unsigned EltIdx = 0;
4540 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
4541 uint64_t SplatValue = 0;
4542 // Construct the amalgamated value which can be splatted as this larger
4543 // vector type.
4544 for (const auto &SeqV : Sequence) {
4545 if (!SeqV.isUndef())
4546 SplatValue |=
4547 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4548 EltIdx++;
4549 }
4550
4551 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4552 // achieve better constant materializion.
4553 // On RV32, we need to sign-extend to use getSignedConstant.
4554 if (ViaIntVT == MVT::i32)
4555 SplatValue = SignExtend64<32>(SplatValue);
4556
4557 // Since we can't introduce illegal i64 types at this stage, we can only
4558 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4559 // way we can use RVV instructions to splat.
4560 assert((ViaIntVT.bitsLE(XLenVT) ||
4561 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4562 "Unexpected bitcast sequence");
4563 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
4564 SDValue ViaVL =
4565 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
4566 MVT ViaContainerVT =
4567 getContainerForFixedLengthVector(ViaVecVT, Subtarget);
4568 SDValue Splat =
4569 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
4570 DAG.getUNDEF(ViaContainerVT),
4571 DAG.getSignedConstant(SplatValue, DL, XLenVT), ViaVL);
4572 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
4573 if (ViaVecLen != RequiredVL)
4574 Splat = DAG.getExtractSubvector(
4575 DL, MVT::getVectorVT(ViaIntVT, RequiredVL), Splat, 0);
4576 return DAG.getBitcast(VT, Splat);
4577 }
4578 }
4579
4580 // If the number of signbits allows, see if we can lower as a <N x i8>.
4581 // Our main goal here is to reduce LMUL (and thus work) required to
4582 // build the constant, but we will also narrow if the resulting
4583 // narrow vector is known to materialize cheaply.
4584 // TODO: We really should be costing the smaller vector. There are
4585 // profitable cases this misses.
4586 if (EltBitSize > 8 && VT.isInteger() &&
4587 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4588 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4589 SDValue Source = DAG.getBuildVector(VT.changeVectorElementType(MVT::i8),
4590 DL, Op->ops());
4591 Source = convertToScalableVector(ContainerVT.changeVectorElementType(MVT::i8),
4592 Source, DAG, Subtarget);
4593 SDValue Res = DAG.getNode(RISCVISD::VSEXT_VL, DL, ContainerVT, Source, Mask, VL);
4594 return convertFromScalableVector(VT, Res, DAG, Subtarget);
4595 }
4596
4597 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4598 return Res;
4599
4600 // For constant vectors, use generic constant pool lowering. Otherwise,
4601 // we'd have to materialize constants in GPRs just to move them into the
4602 // vector.
4603 return SDValue();
4604}
4605
4606static unsigned getPACKOpcode(unsigned DestBW,
4607 const RISCVSubtarget &Subtarget) {
4608 switch (DestBW) {
4609 default:
4610 llvm_unreachable("Unsupported pack size");
4611 case 16:
4612 return RISCV::PACKH;
4613 case 32:
4614 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4615 case 64:
4616 assert(Subtarget.is64Bit());
4617 return RISCV::PACK;
4618 }
4619}
4620
4621/// Double the element size of the build vector to reduce the number
4622/// of vslide1down in the build vector chain. In the worst case, this
4623/// trades three scalar operations for 1 vector operation. Scalar
4624/// operations are generally lower latency, and for out-of-order cores
4625/// we also benefit from additional parallelism.
4626static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4627 const RISCVSubtarget &Subtarget) {
4628 SDLoc DL(Op);
4629 MVT VT = Op.getSimpleValueType();
4630 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4631 MVT ElemVT = VT.getVectorElementType();
4632 if (!ElemVT.isInteger())
4633 return SDValue();
4634
4635 // TODO: Relax these architectural restrictions, possibly with costing
4636 // of the actual instructions required.
4637 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4638 return SDValue();
4639
4640 unsigned NumElts = VT.getVectorNumElements();
4641 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4642 if (ElemSizeInBits >= std::min(Subtarget.getELen(), Subtarget.getXLen()) ||
4643 NumElts % 2 != 0)
4644 return SDValue();
4645
4646 // Produce [B,A] packed into a type twice as wide. Note that all
4647 // scalars are XLenVT, possibly masked (see below).
4648 MVT XLenVT = Subtarget.getXLenVT();
4649 SDValue Mask = DAG.getConstant(
4650 APInt::getLowBitsSet(XLenVT.getSizeInBits(), ElemSizeInBits), DL, XLenVT);
4651 auto pack = [&](SDValue A, SDValue B) {
4652 // Bias the scheduling of the inserted operations to near the
4653 // definition of the element - this tends to reduce register
4654 // pressure overall.
4655 SDLoc ElemDL(B);
4656 if (Subtarget.hasStdExtZbkb())
4657 // Note that we're relying on the high bits of the result being
4658 // don't care. For PACKW, the result is *sign* extended.
4659 return SDValue(
4660 DAG.getMachineNode(getPACKOpcode(ElemSizeInBits * 2, Subtarget),
4661 ElemDL, XLenVT, A, B),
4662 0);
4663
4664 A = DAG.getNode(ISD::AND, SDLoc(A), XLenVT, A, Mask);
4665 B = DAG.getNode(ISD::AND, SDLoc(B), XLenVT, B, Mask);
4666 SDValue ShtAmt = DAG.getConstant(ElemSizeInBits, ElemDL, XLenVT);
4667 return DAG.getNode(ISD::OR, ElemDL, XLenVT, A,
4668 DAG.getNode(ISD::SHL, ElemDL, XLenVT, B, ShtAmt),
4669 SDNodeFlags::Disjoint);
4670 };
4671
4672 SmallVector<SDValue> NewOperands;
4673 NewOperands.reserve(NumElts / 2);
4674 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4675 NewOperands.push_back(pack(Op.getOperand(i), Op.getOperand(i + 1)));
4676 assert(NumElts == NewOperands.size() * 2);
4677 MVT WideVT = MVT::getIntegerVT(ElemSizeInBits * 2);
4678 MVT WideVecVT = MVT::getVectorVT(WideVT, NumElts / 2);
4679 return DAG.getNode(ISD::BITCAST, DL, VT,
4680 DAG.getBuildVector(WideVecVT, DL, NewOperands));
4681}
4682
4683static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4684 const RISCVSubtarget &Subtarget) {
4685 MVT VT = Op.getSimpleValueType();
4686 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4687
4688 MVT EltVT = VT.getVectorElementType();
4689 MVT XLenVT = Subtarget.getXLenVT();
4690
4691 SDLoc DL(Op);
4692
4693 if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
4694 if (VT != MVT::v4i8)
4695 return SDValue();
4696
4697 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b, d))
4698 SDValue Val0 =
4699 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(0));
4700 SDValue Val1 =
4701 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(1));
4702 SDValue Val2 =
4703 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(2));
4704 SDValue Val3 =
4705 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i8, Op->getOperand(3));
4706 SDValue PPairDB =
4707 DAG.getNode(RISCVISD::PPAIRE_DB, DL, {MVT::v4i8, MVT::v4i8},
4708 {Val0, Val2, Val1, Val3});
4709
4710 return DAG.getNode(
4711 ISD::BITCAST, DL, MVT::v4i8,
4712 SDValue(
4713 DAG.getMachineNode(
4714 RISCV::PACK, DL, MVT::i32,
4715 {DAG.getNode(ISD::BITCAST, DL, MVT::i32, PPairDB.getValue(0)),
4716 DAG.getNode(ISD::BITCAST, DL, MVT::i32, PPairDB.getValue(1))}),
4717 0));
4718 }
4719
4720 // Proper support for f16 requires Zvfh. bf16 always requires special
4721 // handling. We need to cast the scalar to integer and create an integer
4722 // build_vector.
4723 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
4724 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4725 MVT IVT = VT.changeVectorElementType(MVT::i16);
4726 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4727 for (const auto &[I, U] : enumerate(Op->ops())) {
4728 SDValue Elem = U.get();
4729 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4730 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4731 // Called by LegalizeDAG, we need to use XLenVT operations since we
4732 // can't create illegal types.
4733 if (auto *C = dyn_cast<ConstantFPSDNode>(Elem)) {
4734 // Manually constant fold so the integer build_vector can be lowered
4735 // better. Waiting for DAGCombine will be too late.
4736 APInt V =
4737 C->getValueAPF().bitcastToAPInt().sext(XLenVT.getSizeInBits());
4738 NewOps[I] = DAG.getConstant(V, DL, XLenVT);
4739 } else {
4740 NewOps[I] = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Elem);
4741 }
4742 } else {
4743 // Called by scalar type legalizer, we can use i16.
4744 NewOps[I] = DAG.getBitcast(MVT::i16, Op.getOperand(I));
4745 }
4746 }
4747 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, DL, IVT, NewOps);
4748 return DAG.getBitcast(VT, Res);
4749 }
4750
4751 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) ||
4752 ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode()))
4753 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4754
4755 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4756
4757 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4758
4759 if (VT.getVectorElementType() == MVT::i1) {
4760 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4761 // vector type, we have a legal equivalently-sized i8 type, so we can use
4762 // that.
4763 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
4764 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
4765
4766 SDValue WideVec;
4767 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4768 // For a splat, perform a scalar truncate before creating the wider
4769 // vector.
4770 Splat = DAG.getNode(ISD::AND, DL, Splat.getValueType(), Splat,
4771 DAG.getConstant(1, DL, Splat.getValueType()));
4772 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
4773 } else {
4774 SmallVector<SDValue, 8> Ops(Op->op_values());
4775 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
4776 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
4777 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
4778 }
4779
4780 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
4781 }
4782
4783 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
4784 if (auto Gather = matchSplatAsGather(Splat, VT, DL, DAG, Subtarget))
4785 return Gather;
4786
4787 if (!VT.isFloatingPoint())
4788 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Splat);
4789
4790 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4791 // pressure at high LMUL.
4792 bool IsScalar = all_of(Op->ops().drop_front(),
4793 [](const SDUse &U) { return U.get().isUndef(); });
4794 unsigned Opc =
4795 VT.isFloatingPoint()
4796 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4797 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4798 Splat =
4799 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Splat, VL);
4800 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
4801 }
4802
4803 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4804 return Res;
4805
4806 // If we're compiling for an exact VLEN value, we can split our work per
4807 // register in the register group.
4808 if (const auto VLen = Subtarget.getRealVLen();
4809 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4810 MVT ElemVT = VT.getVectorElementType();
4811 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4812 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4813 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
4814 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
4815 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4816
4817 // The following semantically builds up a fixed length concat_vector
4818 // of the component build_vectors. We eagerly lower to scalable and
4819 // insert_subvector here to avoid DAG combining it back to a large
4820 // build_vector.
4821 SmallVector<SDValue> BuildVectorOps(Op->ops());
4822 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4823 SDValue Vec = DAG.getUNDEF(ContainerVT);
4824 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4825 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(i, ElemsPerVReg);
4826 SDValue SubBV =
4827 DAG.getNode(ISD::BUILD_VECTOR, DL, OneRegVT, OneVRegOfOps);
4828 SubBV = convertToScalableVector(M1VT, SubBV, DAG, Subtarget);
4829 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4830 Vec = DAG.getInsertSubvector(DL, Vec, SubBV, InsertIdx);
4831 }
4832 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
4833 }
4834
4835 // If we're about to resort to vslide1down (or stack usage), pack our
4836 // elements into the widest scalar type we can. This will force a VL/VTYPE
4837 // toggle, but reduces the critical path, the number of vslide1down ops
4838 // required, and possibly enables scalar folds of the values.
4839 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4840 return Res;
4841
4842 // For m1 vectors, if we have non-undef values in both halves of our vector,
4843 // split the vector into low and high halves, build them separately, then
4844 // use a vselect to combine them. For long vectors, this cuts the critical
4845 // path of the vslide1down sequence in half, and gives us an opportunity
4846 // to special case each half independently. Note that we don't change the
4847 // length of the sub-vectors here, so if both fallback to the generic
4848 // vslide1down path, we should be able to fold the vselect into the final
4849 // vslidedown (for the undef tail) for the first half w/ masking.
4850 unsigned NumElts = VT.getVectorNumElements();
4851 unsigned NumUndefElts =
4852 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
4853 unsigned NumDefElts = NumElts - NumUndefElts;
4854 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4855 ContainerVT.bitsLE(RISCVTargetLowering::getM1VT(ContainerVT))) {
4856 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4857 SmallVector<SDValue> MaskVals;
4858 SDValue UndefElem = DAG.getUNDEF(Op->getOperand(0)->getValueType(0));
4859 SubVecAOps.reserve(NumElts);
4860 SubVecBOps.reserve(NumElts);
4861 for (const auto &[Idx, U] : enumerate(Op->ops())) {
4862 SDValue Elem = U.get();
4863 if (Idx < NumElts / 2) {
4864 SubVecAOps.push_back(Elem);
4865 SubVecBOps.push_back(UndefElem);
4866 } else {
4867 SubVecAOps.push_back(UndefElem);
4868 SubVecBOps.push_back(Elem);
4869 }
4870 bool SelectMaskVal = (Idx < NumElts / 2);
4871 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
4872 }
4873 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
4874 MaskVals.size() == NumElts);
4875
4876 SDValue SubVecA = DAG.getBuildVector(VT, DL, SubVecAOps);
4877 SDValue SubVecB = DAG.getBuildVector(VT, DL, SubVecBOps);
4878 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
4879 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
4880 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, SubVecA, SubVecB);
4881 }
4882
4883 // Cap the cost at a value linear to the number of elements in the vector.
4884 // The default lowering is to use the stack. The vector store + scalar loads
4885 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
4886 // being (at least) linear in LMUL. As a result, using the vslidedown
4887 // lowering for every element ends up being VL*LMUL..
4888 // TODO: Should we be directly costing the stack alternative? Doing so might
4889 // give us a more accurate upper bound.
4890 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
4891
4892 // TODO: unify with TTI getSlideCost.
4893 InstructionCost PerSlideCost = 1;
4894 switch (RISCVTargetLowering::getLMUL(ContainerVT)) {
4895 default: break;
4896 case RISCVVType::LMUL_2:
4897 PerSlideCost = 2;
4898 break;
4899 case RISCVVType::LMUL_4:
4900 PerSlideCost = 4;
4901 break;
4902 case RISCVVType::LMUL_8:
4903 PerSlideCost = 8;
4904 break;
4905 }
4906
4907 // TODO: Should we be using the build instseq then cost + evaluate scheme
4908 // we use for integer constants here?
4909 unsigned UndefCount = 0;
4910 for (const SDValue &V : Op->ops()) {
4911 if (V.isUndef()) {
4912 UndefCount++;
4913 continue;
4914 }
4915 if (UndefCount) {
4916 LinearBudget -= PerSlideCost;
4917 UndefCount = 0;
4918 }
4919 LinearBudget -= PerSlideCost;
4920 }
4921 if (UndefCount) {
4922 LinearBudget -= PerSlideCost;
4923 }
4924
4925 if (LinearBudget < 0)
4926 return SDValue();
4927
4928 assert((!VT.isFloatingPoint() ||
4929 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
4930 "Illegal type which will result in reserved encoding");
4931
4932 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4933
4934 // General case: splat the first operand and slide other operands down one
4935 // by one to form a vector. Alternatively, if every operand is an
4936 // extraction from element 0 of a vector, we use that vector from the last
4937 // extraction as the start value and slide up instead of slide down. Such that
4938 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
4939 // vslideup of 1 later and eliminate the vector to scalar movement, which is
4940 // something we cannot do with vslide1down/vslidedown.
4941 // Of course, using vslide1up/vslideup might increase the register pressure,
4942 // and that's why we conservatively limit to cases where every operand is an
4943 // extraction from the first element.
4944 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
4945 SDValue EVec;
4946 bool SlideUp = false;
4947 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
4948 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
4949 if (SlideUp)
4950 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
4951 Mask, VL, Policy);
4952 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, Vec, Offset,
4953 Mask, VL, Policy);
4954 };
4955
4956 // The reason we don't use all_of here is because we're also capturing EVec
4957 // from the last non-undef operand. If the std::execution_policy of the
4958 // underlying std::all_of is anything but std::sequenced_policy we might
4959 // capture the wrong EVec.
4960 for (SDValue V : Operands) {
4961 using namespace SDPatternMatch;
4962 SlideUp = V.isUndef() || sd_match(V, m_ExtractElt(m_Value(EVec), m_Zero()));
4963 if (!SlideUp)
4964 break;
4965 }
4966
4967 // Do not slideup if the element type of EVec is different.
4968 if (SlideUp) {
4969 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
4970 MVT ContainerEltVT = ContainerVT.getVectorElementType();
4971 if (EVecEltVT != ContainerEltVT)
4972 SlideUp = false;
4973 }
4974
4975 if (SlideUp) {
4976 MVT EVecContainerVT = EVec.getSimpleValueType();
4977 // Make sure the original vector has scalable vector type.
4978 if (EVecContainerVT.isFixedLengthVector()) {
4979 EVecContainerVT =
4980 getContainerForFixedLengthVector(EVecContainerVT, Subtarget);
4981 EVec = convertToScalableVector(EVecContainerVT, EVec, DAG, Subtarget);
4982 }
4983
4984 // Adapt EVec's type into ContainerVT.
4985 if (EVecContainerVT.getVectorMinNumElements() <
4986 ContainerVT.getVectorMinNumElements())
4987 EVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), EVec, 0);
4988 else
4989 EVec = DAG.getExtractSubvector(DL, ContainerVT, EVec, 0);
4990
4991 // Reverse the elements as we're going to slide up from the last element.
4992 std::reverse(Operands.begin(), Operands.end());
4993 }
4994
4995 SDValue Vec;
4996 UndefCount = 0;
4997 for (SDValue V : Operands) {
4998 if (V.isUndef()) {
4999 UndefCount++;
5000 continue;
5001 }
5002
5003 // Start our sequence with either a TA splat or extract source in the
5004 // hopes that hardware is able to recognize there's no dependency on the
5005 // prior value of our temporary register.
5006 if (!Vec) {
5007 if (SlideUp) {
5008 Vec = EVec;
5009 } else {
5010 Vec = DAG.getSplatVector(VT, DL, V);
5011 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5012 }
5013
5014 UndefCount = 0;
5015 continue;
5016 }
5017
5018 if (UndefCount) {
5019 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5020 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5021 VL);
5022 UndefCount = 0;
5023 }
5024
5025 unsigned Opcode;
5026 if (VT.isFloatingPoint())
5027 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
5028 else
5029 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
5030
5031 if (!VT.isFloatingPoint())
5032 V = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), V);
5033 Vec = DAG.getNode(Opcode, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Vec,
5034 V, Mask, VL);
5035 }
5036 if (UndefCount) {
5037 const SDValue Offset = DAG.getConstant(UndefCount, DL, Subtarget.getXLenVT());
5038 Vec = getVSlide(ContainerVT, DAG.getUNDEF(ContainerVT), Vec, Offset, Mask,
5039 VL);
5040 }
5041 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5042}
5043
5044static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5045 SDValue Lo, SDValue Hi, SDValue VL,
5046 SelectionDAG &DAG) {
5047 if (!Passthru)
5048 Passthru = DAG.getUNDEF(VT);
5049 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
5050 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
5051 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
5052 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5053 // node in order to try and match RVV vector/scalar instructions.
5054 if ((LoC >> 31) == HiC)
5055 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5056
5057 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5058 // VL. This can temporarily increase VL if VL less than VLMAX.
5059 if (LoC == HiC) {
5060 SDValue NewVL;
5061 if (isa<ConstantSDNode>(VL) && isUInt<4>(VL->getAsZExtVal()))
5062 NewVL = DAG.getNode(ISD::ADD, DL, VL.getValueType(), VL, VL);
5063 else
5064 NewVL = DAG.getRegister(RISCV::X0, MVT::i32);
5065 MVT InterVT =
5066 MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
5067 auto InterVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, InterVT,
5068 DAG.getUNDEF(InterVT), Lo, NewVL);
5069 return DAG.getNode(ISD::BITCAST, DL, VT, InterVec);
5070 }
5071 }
5072
5073 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5074 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
5075 isa<ConstantSDNode>(Hi.getOperand(1)) &&
5076 Hi.getConstantOperandVal(1) == 31)
5077 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5078
5079 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5080 // even if it might be sign extended.
5081 if (Hi.isUndef())
5082 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Lo, VL);
5083
5084 // Fall back to a stack store and stride x0 vector load.
5085 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Passthru, Lo,
5086 Hi, VL);
5087}
5088
5089// Called by type legalization to handle splat of i64 on RV32.
5090// FIXME: We can optimize this when the type has sign or zero bits in one
5091// of the halves.
5092static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5093 SDValue Scalar, SDValue VL,
5094 SelectionDAG &DAG) {
5095 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5096 SDValue Lo, Hi;
5097 std::tie(Lo, Hi) = DAG.SplitScalar(Scalar, DL, MVT::i32, MVT::i32);
5098 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5099}
5100
5101// This function lowers a splat of a scalar operand Splat with the vector
5102// length VL. It ensures the final sequence is type legal, which is useful when
5103// lowering a splat after type legalization.
5104static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5105 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5106 const RISCVSubtarget &Subtarget) {
5107 bool HasPassthru = Passthru && !Passthru.isUndef();
5108 if (!HasPassthru && !Passthru)
5109 Passthru = DAG.getUNDEF(VT);
5110
5111 MVT EltVT = VT.getVectorElementType();
5112 MVT XLenVT = Subtarget.getXLenVT();
5113
5114 if (VT.isFloatingPoint()) {
5115 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
5116 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5117 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5118 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5119 Scalar = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Scalar);
5120 else
5121 Scalar = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Scalar);
5122 MVT IVT = VT.changeVectorElementType(MVT::i16);
5123 Passthru = DAG.getNode(ISD::BITCAST, DL, IVT, Passthru);
5124 SDValue Splat =
5125 lowerScalarSplat(Passthru, Scalar, VL, IVT, DL, DAG, Subtarget);
5126 return DAG.getNode(ISD::BITCAST, DL, VT, Splat);
5127 }
5128 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Passthru, Scalar, VL);
5129 }
5130
5131 // Simplest case is that the operand needs to be promoted to XLenVT.
5132 if (Scalar.getValueType().bitsLE(XLenVT)) {
5133 // If the operand is a constant, sign extend to increase our chances
5134 // of being able to use a .vi instruction. ANY_EXTEND would become a
5135 // a zero extend and the simm5 check in isel would fail.
5136 // FIXME: Should we ignore the upper bits in isel instead?
5137 unsigned ExtOpc =
5138 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5139 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5140 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Passthru, Scalar, VL);
5141 }
5142
5143 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5144 "Unexpected scalar for splat lowering!");
5145
5146 if (isOneConstant(VL) && isNullConstant(Scalar))
5147 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, Passthru,
5148 DAG.getConstant(0, DL, XLenVT), VL);
5149
5150 // Otherwise use the more complicated splatting algorithm.
5151 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5152}
5153
5154// This function lowers an insert of a scalar operand Scalar into lane
5155// 0 of the vector regardless of the value of VL. The contents of the
5156// remaining lanes of the result vector are unspecified. VL is assumed
5157// to be non-zero.
5158static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5159 const SDLoc &DL, SelectionDAG &DAG,
5160 const RISCVSubtarget &Subtarget) {
5161 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5162
5163 const MVT XLenVT = Subtarget.getXLenVT();
5164 SDValue Passthru = DAG.getUNDEF(VT);
5165
5166 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5167 isNullConstant(Scalar.getOperand(1))) {
5168 SDValue ExtractedVal = Scalar.getOperand(0);
5169 // The element types must be the same.
5170 if (ExtractedVal.getValueType().getVectorElementType() ==
5171 VT.getVectorElementType()) {
5172 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5173 MVT ExtractedContainerVT = ExtractedVT;
5174 if (ExtractedContainerVT.isFixedLengthVector()) {
5175 ExtractedContainerVT =
5176 getContainerForFixedLengthVector(ExtractedContainerVT, Subtarget);
5177 ExtractedVal = convertToScalableVector(ExtractedContainerVT,
5178 ExtractedVal, DAG, Subtarget);
5179 }
5180 if (ExtractedContainerVT.bitsLE(VT))
5181 return DAG.getInsertSubvector(DL, Passthru, ExtractedVal, 0);
5182 return DAG.getExtractSubvector(DL, VT, ExtractedVal, 0);
5183 }
5184 }
5185
5186 if (VT.isFloatingPoint())
5187 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5188 VL);
5189
5190 // Avoid the tricky legalization cases by falling back to using the
5191 // splat code which already handles it gracefully.
5192 if (!Scalar.getValueType().bitsLE(XLenVT))
5193 return lowerScalarSplat(DAG.getUNDEF(VT), Scalar,
5194 DAG.getConstant(1, DL, XLenVT),
5195 VT, DL, DAG, Subtarget);
5196
5197 // If the operand is a constant, sign extend to increase our chances
5198 // of being able to use a .vi instruction. ANY_EXTEND would become a
5199 // a zero extend and the simm5 check in isel would fail.
5200 // FIXME: Should we ignore the upper bits in isel instead?
5201 unsigned ExtOpc =
5202 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5203 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
5204 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, VT, DAG.getUNDEF(VT), Scalar,
5205 VL);
5206}
5207
5208/// If concat_vector(V1,V2) could be folded away to some existing
5209/// vector source, return it. Note that the source may be larger
5210/// than the requested concat_vector (i.e. a extract_subvector
5211/// might be required.)
5212static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5213 EVT VT = V1.getValueType();
5214 assert(VT == V2.getValueType() && "argument types must match");
5215 // Both input must be extracts.
5216 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5217 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5218 return SDValue();
5219
5220 // Extracting from the same source.
5221 SDValue Src = V1.getOperand(0);
5222 if (Src != V2.getOperand(0) ||
5223 VT.isScalableVector() != Src.getValueType().isScalableVector())
5224 return SDValue();
5225
5226 // The extracts must extract the two halves of the source.
5227 if (V1.getConstantOperandVal(1) != 0 ||
5228 V2.getConstantOperandVal(1) != VT.getVectorMinNumElements())
5229 return SDValue();
5230
5231 return Src;
5232}
5233
5234// Can this shuffle be performed on exactly one (possibly larger) input?
5235static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5236
5237 if (V2.isUndef())
5238 return V1;
5239
5240 unsigned NumElts = VT.getVectorNumElements();
5241 // Src needs to have twice the number of elements.
5242 // TODO: Update shuffle lowering to add the extract subvector
5243 if (SDValue Src = foldConcatVector(V1, V2);
5244 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5245 return Src;
5246
5247 return SDValue();
5248}
5249
5250static bool isLegalVTForZvzipOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5251 MVT ContainerVT = VT;
5252 if (VT.isFixedLengthVector())
5253 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5254 // Determine LMUL of the container vector.
5255 return RISCVTargetLowering::getLMUL(ContainerVT) != RISCVVType::LMUL_8;
5256}
5257
5258/// Is this shuffle interleaving contiguous elements from one vector into the
5259/// even elements and contiguous elements from another vector into the odd
5260/// elements. \p EvenSrc will contain the element that should be in the first
5261/// even element. \p OddSrc will contain the element that should be in the first
5262/// odd element. These can be the first element in a source or the element half
5263/// way through the source.
5264static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5265 int &OddSrc, const RISCVSubtarget &Subtarget) {
5266 // We need to be able to widen elements to the next larger integer type or
5267 // use the vzip instruction at e64.
5268 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
5269 if (!Subtarget.hasStdExtZvzip())
5270 return false;
5271 if (!isLegalVTForZvzipOperand(VT, Subtarget))
5272 return false;
5273 }
5274
5275 int Size = Mask.size();
5276 int NumElts = VT.getVectorNumElements();
5277 assert(Size == (int)NumElts && "Unexpected mask size");
5278
5279 SmallVector<unsigned, 2> StartIndexes;
5280 if (!ShuffleVectorInst::isInterleaveMask(Mask, 2, Size * 2, StartIndexes))
5281 return false;
5282
5283 EvenSrc = StartIndexes[0];
5284 OddSrc = StartIndexes[1];
5285
5286 // One source should be low half of first vector.
5287 if (EvenSrc != 0 && OddSrc != 0)
5288 return false;
5289
5290 // Subvectors will be subtracted from either at the start of the two input
5291 // vectors, or at the start and middle of the first vector if it's an unary
5292 // interleave.
5293 // In both cases, HalfNumElts will be extracted.
5294 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5295 // we'll create an illegal extract_subvector.
5296 // FIXME: We could support other values using a slidedown first.
5297 int HalfNumElts = NumElts / 2;
5298 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5299}
5300
5301/// Is this mask representing a masked combination of two slides?
5302static bool isMaskedSlidePair(ArrayRef<int> Mask,
5303 std::array<std::pair<int, int>, 2> &SrcInfo) {
5304 if (!llvm::isMaskedSlidePair(Mask, Mask.size(), SrcInfo))
5305 return false;
5306
5307 // Avoid matching vselect idioms
5308 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5309 return false;
5310 // Prefer vslideup as the second instruction, and identity
5311 // only as the initial instruction.
5312 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5313 SrcInfo[1].second == 0)
5314 std::swap(SrcInfo[0], SrcInfo[1]);
5315 assert(SrcInfo[0].first != -1 && "Must find one slide");
5316 return true;
5317}
5318
5319// Exactly matches the semantics of a previously existing custom matcher
5320// to allow migration to new matcher without changing output.
5321static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5322 unsigned NumElts) {
5323 if (SrcInfo[1].first == -1)
5324 return true;
5325 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5326 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5327}
5328
5329static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5330 ArrayRef<int> Mask, unsigned Factor,
5331 bool RequiredPolarity) {
5332 int NumElts = Mask.size();
5333 for (const auto &[Idx, M] : enumerate(Mask)) {
5334 if (M < 0)
5335 continue;
5336 int Src = M >= NumElts;
5337 int Diff = (int)Idx - (M % NumElts);
5338 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5339 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5340 "Must match exactly one of the two slides");
5341 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5342 return false;
5343 }
5344 return true;
5345}
5346
5347/// Given a shuffle which can be represented as a pair of two slides,
5348/// see if it is a pair-even idiom.
5349/// Pair-even is:
5350/// vs2: a0 a1 a2 a3
5351/// vs1: b0 b1 b2 b3
5352/// vd: a0 b0 a2 b2
5353static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5354 ArrayRef<int> Mask, unsigned &Factor) {
5355 Factor = SrcInfo[1].second;
5356 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5357 Mask.size() % Factor == 0 &&
5358 isAlternating(SrcInfo, Mask, Factor, true);
5359}
5360
5361/// Given a shuffle which can be represented as a pair of two slides,
5362/// see if it is a pair-odd idiom.
5363/// Pair-odd is:
5364/// vs2: a0 a1 a2 a3
5365/// vs1: b0 b1 b2 b3
5366/// vd: a1 b1 a3 b3
5367/// Note that the operand order is swapped due to the way we canonicalize
5368/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5369static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5370 ArrayRef<int> Mask, unsigned &Factor) {
5371 Factor = -SrcInfo[1].second;
5372 return SrcInfo[0].second == 0 && isPowerOf2_32(Factor) &&
5373 Mask.size() % Factor == 0 &&
5374 isAlternating(SrcInfo, Mask, Factor, false);
5375}
5376
5377// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5378// 2, 4, 8 and the integer type Factor-times larger than VT's
5379// element type must be a legal element type.
5380// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5381// -> [p, q, r, s] (Factor=2, Index=1)
5382static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5383 SDValue Src, unsigned Factor,
5384 unsigned Index, SelectionDAG &DAG) {
5385 unsigned EltBits = VT.getScalarSizeInBits();
5386 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5387 MVT WideSrcVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor),
5388 SrcEC.divideCoefficientBy(Factor));
5389 MVT ResVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits),
5390 SrcEC.divideCoefficientBy(Factor));
5391 Src = DAG.getBitcast(WideSrcVT, Src);
5392
5393 unsigned Shift = Index * EltBits;
5394 SDValue Res = DAG.getNode(ISD::SRL, DL, WideSrcVT, Src,
5395 DAG.getConstant(Shift, DL, WideSrcVT));
5396 Res = DAG.getNode(ISD::TRUNCATE, DL, ResVT, Res);
5397 MVT CastVT = ResVT.changeVectorElementType(VT.getVectorElementType());
5398 Res = DAG.getBitcast(CastVT, Res);
5399 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
5400}
5401
5402/// Match a single source shuffle which is an identity except that some
5403/// particular element is repeated. This can be lowered as a masked
5404/// vrgather.vi/vx. Note that the two source form of this is handled
5405/// by the recursive splitting logic and doesn't need special handling.
5406static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5407 const RISCVSubtarget &Subtarget,
5408 SelectionDAG &DAG) {
5409
5410 SDLoc DL(SVN);
5411 MVT VT = SVN->getSimpleValueType(0);
5412 SDValue V1 = SVN->getOperand(0);
5413 assert(SVN->getOperand(1).isUndef());
5414 ArrayRef<int> Mask = SVN->getMask();
5415 const unsigned NumElts = VT.getVectorNumElements();
5416 MVT XLenVT = Subtarget.getXLenVT();
5417
5418 std::optional<int> SplatIdx;
5419 for (auto [I, M] : enumerate(Mask)) {
5420 if (M == -1 || I == (unsigned)M)
5421 continue;
5422 if (SplatIdx && *SplatIdx != M)
5423 return SDValue();
5424 SplatIdx = M;
5425 }
5426
5427 if (!SplatIdx)
5428 return SDValue();
5429
5430 SmallVector<SDValue> MaskVals;
5431 for (int MaskIndex : Mask) {
5432 bool SelectMaskVal = MaskIndex == *SplatIdx;
5433 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
5434 }
5435 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5436 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
5437 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
5438 SDValue Splat = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT),
5439 SmallVector<int>(NumElts, *SplatIdx));
5440 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, Splat, V1);
5441}
5442
5443// Lower the following shuffle to vslidedown.
5444// a)
5445// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5446// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5447// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5448// b)
5449// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5450// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5451// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5452// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5453// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5454// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5455static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5456 SDValue V1, SDValue V2,
5457 ArrayRef<int> Mask,
5458 const RISCVSubtarget &Subtarget,
5459 SelectionDAG &DAG) {
5460 auto findNonEXTRACT_SUBVECTORParent =
5461 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5462 uint64_t Offset = 0;
5463 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5464 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5465 // a scalable vector. But we don't want to match the case.
5466 Parent.getOperand(0).getSimpleValueType().isFixedLengthVector()) {
5467 Offset += Parent.getConstantOperandVal(1);
5468 Parent = Parent.getOperand(0);
5469 }
5470 return std::make_pair(Parent, Offset);
5471 };
5472
5473 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5474 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5475
5476 // Extracting from the same source.
5477 SDValue Src = V1Src;
5478 if (Src != V2Src)
5479 return SDValue();
5480
5481 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5482 SmallVector<int, 16> NewMask(Mask);
5483 for (size_t i = 0; i != NewMask.size(); ++i) {
5484 if (NewMask[i] == -1)
5485 continue;
5486
5487 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5488 NewMask[i] = NewMask[i] + V1IndexOffset;
5489 } else {
5490 // Minus NewMask.size() is needed. Otherwise, the b case would be
5491 // <5,6,7,12> instead of <5,6,7,8>.
5492 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5493 }
5494 }
5495
5496 // First index must be known and non-zero. It will be used as the slidedown
5497 // amount.
5498 if (NewMask[0] <= 0)
5499 return SDValue();
5500
5501 // NewMask is also continuous.
5502 for (unsigned i = 1; i != NewMask.size(); ++i)
5503 if (NewMask[i - 1] + 1 != NewMask[i])
5504 return SDValue();
5505
5506 MVT XLenVT = Subtarget.getXLenVT();
5507 MVT SrcVT = Src.getSimpleValueType();
5508 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
5509 auto [TrueMask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
5510 SDValue Slidedown =
5511 getVSlidedown(DAG, Subtarget, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
5512 convertToScalableVector(ContainerVT, Src, DAG, Subtarget),
5513 DAG.getConstant(NewMask[0], DL, XLenVT), TrueMask, VL);
5514 return DAG.getExtractSubvector(
5515 DL, VT, convertFromScalableVector(SrcVT, Slidedown, DAG, Subtarget), 0);
5516}
5517
5518// Because vslideup leaves the destination elements at the start intact, we can
5519// use it to perform shuffles that insert subvectors:
5520//
5521// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5522// ->
5523// vsetvli zero, 8, e8, mf2, ta, ma
5524// vslideup.vi v8, v9, 4
5525//
5526// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5527// ->
5528// vsetvli zero, 5, e8, mf2, tu, ma
5529// vslideup.v1 v8, v9, 2
5530static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5531 SDValue V1, SDValue V2,
5532 ArrayRef<int> Mask,
5533 const RISCVSubtarget &Subtarget,
5534 SelectionDAG &DAG) {
5535 unsigned NumElts = VT.getVectorNumElements();
5536 int NumSubElts, Index;
5537 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumElts, NumSubElts,
5538 Index))
5539 return SDValue();
5540
5541 bool OpsSwapped = Mask[Index] < (int)NumElts;
5542 SDValue InPlace = OpsSwapped ? V2 : V1;
5543 SDValue ToInsert = OpsSwapped ? V1 : V2;
5544
5545 MVT XLenVT = Subtarget.getXLenVT();
5546 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5547 auto TrueMask = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).first;
5548 // We slide up by the index that the subvector is being inserted at, and set
5549 // VL to the index + the number of elements being inserted.
5550 unsigned Policy =
5551 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5552 // If the we're adding a suffix to the in place vector, i.e. inserting right
5553 // up to the very end of it, then we don't actually care about the tail.
5554 if (NumSubElts + Index >= (int)NumElts)
5555 Policy |= RISCVVType::TAIL_AGNOSTIC;
5556
5557 InPlace = convertToScalableVector(ContainerVT, InPlace, DAG, Subtarget);
5558 ToInsert = convertToScalableVector(ContainerVT, ToInsert, DAG, Subtarget);
5559 SDValue VL = DAG.getConstant(NumSubElts + Index, DL, XLenVT);
5560
5561 SDValue Res;
5562 // If we're inserting into the lowest elements, use a tail undisturbed
5563 // vmv.v.v.
5564 if (Index == 0)
5565 Res = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, InPlace, ToInsert,
5566 VL);
5567 else
5568 Res = getVSlideup(DAG, Subtarget, DL, ContainerVT, InPlace, ToInsert,
5569 DAG.getConstant(Index, DL, XLenVT), TrueMask, VL, Policy);
5570 return convertFromScalableVector(VT, Res, DAG, Subtarget);
5571}
5572
5573// A shuffle of shuffles where the final data only is drawn from 2 input ops
5574// can be compressed into a single shuffle
5575static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5576 const RISCVSubtarget &Subtarget,
5577 SelectionDAG &DAG) {
5578 SDValue V1 = SVN->getOperand(0);
5579 SDValue V2 = SVN->getOperand(1);
5580
5581 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5582 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5583 return SDValue();
5584
5585 if (!V1.hasOneUse() || !V2.hasOneUse())
5586 return SDValue();
5587
5588 ArrayRef<int> Mask = SVN->getMask();
5589 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(V1.getNode())->getMask();
5590 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(V2.getNode())->getMask();
5591 unsigned NumElts = Mask.size();
5592 SmallVector<int> NewMask(NumElts, -1);
5593 for (unsigned Idx : seq<unsigned>(NumElts)) {
5594 int Lane = Mask[Idx];
5595 // Don't assign if poison
5596 if (Lane == -1)
5597 continue;
5598 int OrigLane;
5599 bool SecondOp = false;
5600 if ((unsigned)Lane < NumElts) {
5601 OrigLane = V1Mask[Lane];
5602 } else {
5603 OrigLane = V2Mask[Lane - NumElts];
5604 SecondOp = true;
5605 }
5606 if (OrigLane == -1)
5607 continue;
5608 // Don't handle if shuffling from a second operand
5609 if ((unsigned)OrigLane >= NumElts)
5610 return SDValue();
5611 if (SecondOp)
5612 OrigLane += NumElts;
5613 NewMask[Idx] = OrigLane;
5614 }
5615
5616 EVT VT = SVN->getValueType(0);
5617 SDLoc DL(SVN);
5618
5619 return DAG.getVectorShuffle(VT, DL, V1->getOperand(0), V2->getOperand(0),
5620 NewMask);
5621}
5622
5623/// Match v(f)slide1up/down idioms. These operations involve sliding
5624/// N-1 elements to make room for an inserted scalar at one end.
5625static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5626 SDValue V1, SDValue V2,
5627 ArrayRef<int> Mask,
5628 const RISCVSubtarget &Subtarget,
5629 SelectionDAG &DAG) {
5630 bool OpsSwapped = false;
5631 if (!isa<BuildVectorSDNode>(V1)) {
5632 if (!isa<BuildVectorSDNode>(V2))
5633 return SDValue();
5634 std::swap(V1, V2);
5635 OpsSwapped = true;
5636 }
5637 SDValue Splat = cast<BuildVectorSDNode>(V1)->getSplatValue();
5638 if (!Splat)
5639 return SDValue();
5640
5641 // Return true if the mask could describe a slide of Mask.size() - 1
5642 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5643 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5644 const unsigned S = (Offset > 0) ? 0 : -Offset;
5645 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5646 for (unsigned i = S; i != E; ++i)
5647 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5648 return false;
5649 return true;
5650 };
5651
5652 const unsigned NumElts = VT.getVectorNumElements();
5653 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5654 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5655 return SDValue();
5656
5657 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5658 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5659 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5660 return SDValue();
5661
5662 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5663 auto [TrueMask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
5664
5665 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5666 // vslide1{down,up}.vx instead.
5667 if ((VT.getVectorElementType() == MVT::bf16 &&
5668 !Subtarget.hasVInstructionsBF16()) ||
5669 (VT.getVectorElementType() == MVT::f16 &&
5670 !Subtarget.hasVInstructionsF16())) {
5671 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5672 Splat =
5673 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(), Splat);
5674 V2 = DAG.getBitcast(
5675 IntVT, convertToScalableVector(ContainerVT, V2, DAG, Subtarget));
5676 SDValue Vec = DAG.getNode(
5677 IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5678 IntVT, DAG.getUNDEF(IntVT), V2, Splat, TrueMask, VL);
5679 Vec = DAG.getBitcast(ContainerVT, Vec);
5680 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5681 }
5682
5683 auto OpCode = IsVSlidedown ?
5684 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5685 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5686 if (!VT.isFloatingPoint())
5687 Splat = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Splat);
5688 auto Vec = DAG.getNode(OpCode, DL, ContainerVT,
5689 DAG.getUNDEF(ContainerVT),
5690 convertToScalableVector(ContainerVT, V2, DAG, Subtarget),
5691 Splat, TrueMask, VL);
5692 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
5693}
5694
5695/// Match a mask which "spreads" the leading elements of a vector evenly
5696/// across the result. Factor is the spread amount, and Index is the
5697/// offset applied. (on success, Index < Factor) This is the inverse
5698/// of a deinterleave with the same Factor and Index. This is analogous
5699/// to an interleave, except that all but one lane is undef.
5700bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5701 unsigned &Index) {
5702 SmallVector<bool> LaneIsUndef(Factor, true);
5703 for (unsigned i = 0; i < Mask.size(); i++)
5704 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5705
5706 bool Found = false;
5707 for (unsigned i = 0; i < Factor; i++) {
5708 if (LaneIsUndef[i])
5709 continue;
5710 if (Found)
5711 return false;
5712 Index = i;
5713 Found = true;
5714 }
5715 if (!Found)
5716 return false;
5717
5718 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5719 unsigned j = i * Factor + Index;
5720 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5721 return false;
5722 }
5723 return true;
5724}
5725
5726static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
5727 const SDLoc &DL, SelectionDAG &DAG,
5728 const RISCVSubtarget &Subtarget) {
5729 assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
5730 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5731
5732 MVT VT = Op0.getSimpleValueType();
5733 MVT IntVT = VT.changeVectorElementTypeToInteger();
5734 Op0 = DAG.getBitcast(IntVT, Op0);
5735 Op1 = DAG.getBitcast(IntVT, Op1);
5736
5737 MVT ContainerVT = IntVT;
5738 if (VT.isFixedLengthVector()) {
5739 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5740 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5741 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5742 }
5743
5744 MVT InnerVT = ContainerVT;
5745 auto [Mask, VL] = getDefaultVLOps(IntVT, InnerVT, DL, DAG, Subtarget);
5746
5747 SDValue Passthru = DAG.getUNDEF(InnerVT);
5748 SDValue Res = DAG.getNode(Opc, DL, InnerVT, Op0, Op1, Passthru, Mask, VL);
5749 if (IntVT.isFixedLengthVector())
5750 Res = convertFromScalableVector(IntVT, Res, DAG, Subtarget);
5751 Res = DAG.getBitcast(VT, Res);
5752 return Res;
5753}
5754
5755static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
5756 SelectionDAG &DAG,
5757 const RISCVSubtarget &Subtarget) {
5758 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5759 MVT VT = Op0.getSimpleValueType();
5760 MVT IntVT = VT.changeVectorElementTypeToInteger();
5761 Op0 = DAG.getBitcast(IntVT, Op0);
5762 Op1 = DAG.getBitcast(IntVT, Op1);
5763 MVT ContainerVT = IntVT;
5764 if (VT.isFixedLengthVector()) {
5765 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5766 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
5767 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
5768 }
5769 MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
5770 auto [Mask, VL] = getDefaultVLOps(IntVT, ContainerVT, DL, DAG, Subtarget);
5771 SDValue Passthru = DAG.getUNDEF(ResVT);
5772 SDValue Res =
5773 DAG.getNode(RISCVISD::VZIP_VL, DL, ResVT, Op0, Op1, Passthru, Mask, VL);
5774 if (IntVT.isFixedLengthVector())
5775 Res = convertFromScalableVector(IntVT.getDoubleNumVectorElementsVT(), Res,
5776 DAG, Subtarget);
5777 Res = DAG.getBitcast(VT.getDoubleNumVectorElementsVT(), Res);
5778 return Res;
5779}
5780
5781static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
5782 SelectionDAG &DAG,
5783 const RISCVSubtarget &Subtarget) {
5784 assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
5785 MVT VT = Op.getSimpleValueType();
5786 assert(VT.getVectorMinNumElements() >= 2);
5787
5788 MVT IntVT = VT.changeVectorElementTypeToInteger();
5789 Op = DAG.getBitcast(IntVT, Op);
5790 MVT ContainerVT = IntVT;
5791 if (VT.isFixedLengthVector()) {
5792 ContainerVT = getContainerForFixedLengthVector(IntVT, Subtarget);
5793 // For E64 with LMUL <= 1, we can't represent a smaller fractional LMUL for
5794 // the result (LMUL <= 1/2 is not valid for E64). We must widen the input
5795 // container to at least LMUL=2 so the result can be LMUL=1.
5796 if (ContainerVT.getVectorElementType() == MVT::i64 &&
5797 RISCVTargetLowering::getLMUL(ContainerVT) == RISCVVType::LMUL_1) {
5798 ContainerVT = MVT::getScalableVectorVT(MVT::i64, 2);
5799 }
5800 Op = convertToScalableVector(ContainerVT, Op, DAG, Subtarget);
5801 }
5802
5803 MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
5804 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5805 MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
5806 auto [Mask, VL] = getDefaultVLOps(ResVT, ResVT, DL, DAG, Subtarget);
5807 if (VT.isFixedLengthVector())
5808 VL = DAG.getConstant(VT.getVectorNumElements() / 2, DL,
5809 Subtarget.getXLenVT());
5810 SDValue Passthru = DAG.getUNDEF(ResVT);
5811 SDValue Res = DAG.getNode(Opc, DL, ResVT, Op, Passthru, Mask, VL);
5812 if (HalfIntVT.isFixedLengthVector())
5813 Res = convertFromScalableVector(HalfIntVT, Res, DAG, Subtarget);
5814 Res = DAG.getBitcast(HalfVT, Res);
5815 return Res;
5816}
5817
5818// Given a vector a, b, c, d return a vector Factor times longer
5819// with Factor-1 undef's between elements. Ex:
5820// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5821// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5822static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5823 const SDLoc &DL, SelectionDAG &DAG) {
5824
5825 MVT VT = V.getSimpleValueType();
5826 unsigned EltBits = VT.getScalarSizeInBits();
5827 ElementCount EC = VT.getVectorElementCount();
5828 V = DAG.getBitcast(VT.changeTypeToInteger(), V);
5829
5830 MVT WideVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Factor), EC);
5831
5832 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, V);
5833 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5834 // allow the SHL to fold away if Index is 0.
5835 if (Index != 0)
5836 Result = DAG.getNode(ISD::SHL, DL, WideVT, Result,
5837 DAG.getConstant(EltBits * Index, DL, WideVT));
5838 // Make sure to use original element type
5839 MVT ResultVT = MVT::getVectorVT(VT.getVectorElementType(),
5840 EC.multiplyCoefficientBy(Factor));
5841 return DAG.getBitcast(ResultVT, Result);
5842}
5843
5844// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
5845// to create an interleaved vector of <[vscale x] n*2 x ty>.
5846// This requires that the size of ty is less than the subtarget's maximum ELEN.
5847static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
5848 const SDLoc &DL, SelectionDAG &DAG,
5849 const RISCVSubtarget &Subtarget) {
5850
5851 // FIXME: Not only does this optimize the code, it fixes some correctness
5852 // issues because MIR does not have freeze.
5853 if (EvenV.isUndef())
5854 return getWideningSpread(OddV, 2, 1, DL, DAG);
5855 if (OddV.isUndef())
5856 return getWideningSpread(EvenV, 2, 0, DL, DAG);
5857
5858 MVT VecVT = EvenV.getSimpleValueType();
5859 MVT VecContainerVT = VecVT; // <vscale x n x ty>
5860 // Convert fixed vectors to scalable if needed
5861 if (VecContainerVT.isFixedLengthVector()) {
5862 VecContainerVT = getContainerForFixedLengthVector(VecVT, Subtarget);
5863 EvenV = convertToScalableVector(VecContainerVT, EvenV, DAG, Subtarget);
5864 OddV = convertToScalableVector(VecContainerVT, OddV, DAG, Subtarget);
5865 }
5866
5867 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
5868
5869 // We're working with a vector of the same size as the resulting
5870 // interleaved vector, but with half the number of elements and
5871 // twice the SEW (Hence the restriction on not using the maximum
5872 // ELEN)
5873 MVT WideVT =
5874 MVT::getVectorVT(MVT::getIntegerVT(VecVT.getScalarSizeInBits() * 2),
5875 VecVT.getVectorElementCount());
5876 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
5877 if (WideContainerVT.isFixedLengthVector())
5878 WideContainerVT = getContainerForFixedLengthVector(WideVT, Subtarget);
5879
5880 // Bitcast the input vectors to integers in case they are FP
5881 VecContainerVT = VecContainerVT.changeTypeToInteger();
5882 EvenV = DAG.getBitcast(VecContainerVT, EvenV);
5883 OddV = DAG.getBitcast(VecContainerVT, OddV);
5884
5885 auto [Mask, VL] = getDefaultVLOps(VecVT, VecContainerVT, DL, DAG, Subtarget);
5886 SDValue Passthru = DAG.getUNDEF(WideContainerVT);
5887
5888 SDValue Interleaved;
5889 if (Subtarget.hasStdExtZvbb()) {
5890 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
5891 SDValue OffsetVec =
5892 DAG.getConstant(VecVT.getScalarSizeInBits(), DL, VecContainerVT);
5893 Interleaved = DAG.getNode(RISCVISD::VWSLL_VL, DL, WideContainerVT, OddV,
5894 OffsetVec, Passthru, Mask, VL);
5895 Interleaved = DAG.getNode(RISCVISD::VWADDU_W_VL, DL, WideContainerVT,
5896 Interleaved, EvenV, Passthru, Mask, VL);
5897 } else {
5898 // FIXME: We should freeze the odd vector here. We already handled the case
5899 // of provably undef/poison above.
5900
5901 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
5902 // vwaddu.vv
5903 Interleaved = DAG.getNode(RISCVISD::VWADDU_VL, DL, WideContainerVT, EvenV,
5904 OddV, Passthru, Mask, VL);
5905
5906 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
5907 SDValue AllOnesVec = DAG.getSplatVector(
5908 VecContainerVT, DL, DAG.getAllOnesConstant(DL, Subtarget.getXLenVT()));
5909 SDValue OddsMul = DAG.getNode(RISCVISD::VWMULU_VL, DL, WideContainerVT,
5910 OddV, AllOnesVec, Passthru, Mask, VL);
5911
5912 // Add the two together so we get
5913 // (OddV * 0xff...ff) + (OddV + EvenV)
5914 // = (OddV * 0x100...00) + EvenV
5915 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
5916 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
5917 Interleaved = DAG.getNode(RISCVISD::ADD_VL, DL, WideContainerVT,
5918 Interleaved, OddsMul, Passthru, Mask, VL);
5919 }
5920
5921 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
5922 MVT ResultContainerVT = MVT::getVectorVT(
5923 VecVT.getVectorElementType(), // Make sure to use original type
5924 VecContainerVT.getVectorElementCount().multiplyCoefficientBy(2));
5925 Interleaved = DAG.getBitcast(ResultContainerVT, Interleaved);
5926
5927 // Convert back to a fixed vector if needed
5928 MVT ResultVT =
5929 MVT::getVectorVT(VecVT.getVectorElementType(),
5930 VecVT.getVectorElementCount().multiplyCoefficientBy(2));
5931 if (ResultVT.isFixedLengthVector())
5932 Interleaved =
5933 convertFromScalableVector(ResultVT, Interleaved, DAG, Subtarget);
5934
5935 return Interleaved;
5936}
5937
5938// If we have a vector of bits that we want to reverse, we can use a vbrev on a
5939// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
5940static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
5941 SelectionDAG &DAG,
5942 const RISCVSubtarget &Subtarget) {
5943 SDLoc DL(SVN);
5944 MVT VT = SVN->getSimpleValueType(0);
5945 SDValue V = SVN->getOperand(0);
5946 unsigned NumElts = VT.getVectorNumElements();
5947
5948 assert(VT.getVectorElementType() == MVT::i1);
5949
5950 if (!ShuffleVectorInst::isReverseMask(SVN->getMask(),
5951 SVN->getMask().size()) ||
5952 !SVN->getOperand(1).isUndef())
5953 return SDValue();
5954
5955 unsigned ViaEltSize = std::max((uint64_t)8, PowerOf2Ceil(NumElts));
5956 EVT ViaVT = EVT::getVectorVT(
5957 *DAG.getContext(), EVT::getIntegerVT(*DAG.getContext(), ViaEltSize), 1);
5958 EVT ViaBitVT =
5959 EVT::getVectorVT(*DAG.getContext(), MVT::i1, ViaVT.getScalarSizeInBits());
5960
5961 // If we don't have zvbb or the larger element type > ELEN, the operation will
5962 // be illegal.
5963 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(ISD::BITREVERSE,
5964 ViaVT) ||
5965 !Subtarget.getTargetLowering()->isTypeLegal(ViaBitVT))
5966 return SDValue();
5967
5968 // If the bit vector doesn't fit exactly into the larger element type, we need
5969 // to insert it into the larger vector and then shift up the reversed bits
5970 // afterwards to get rid of the gap introduced.
5971 if (ViaEltSize > NumElts)
5972 V = DAG.getInsertSubvector(DL, DAG.getUNDEF(ViaBitVT), V, 0);
5973
5974 SDValue Res =
5975 DAG.getNode(ISD::BITREVERSE, DL, ViaVT, DAG.getBitcast(ViaVT, V));
5976
5977 // Shift up the reversed bits if the vector didn't exactly fit into the larger
5978 // element type.
5979 if (ViaEltSize > NumElts)
5980 Res = DAG.getNode(ISD::SRL, DL, ViaVT, Res,
5981 DAG.getConstant(ViaEltSize - NumElts, DL, ViaVT));
5982
5983 Res = DAG.getBitcast(ViaBitVT, Res);
5984
5985 if (ViaEltSize > NumElts)
5986 Res = DAG.getExtractSubvector(DL, VT, Res, 0);
5987 return Res;
5988}
5989
5990static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
5991 const RISCVSubtarget &Subtarget,
5992 MVT &RotateVT, unsigned &RotateAmt) {
5993 unsigned NumElts = VT.getVectorNumElements();
5994 unsigned EltSizeInBits = VT.getScalarSizeInBits();
5995 unsigned NumSubElts;
5996 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, 2,
5997 NumElts, NumSubElts, RotateAmt))
5998 return false;
5999 RotateVT = MVT::getVectorVT(MVT::getIntegerVT(EltSizeInBits * NumSubElts),
6000 NumElts / NumSubElts);
6001
6002 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
6003 return Subtarget.getTargetLowering()->isTypeLegal(RotateVT);
6004}
6005
6006// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
6007// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
6008// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
6009static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
6010 SelectionDAG &DAG,
6011 const RISCVSubtarget &Subtarget) {
6012 SDLoc DL(SVN);
6013
6014 EVT VT = SVN->getValueType(0);
6015 unsigned RotateAmt;
6016 MVT RotateVT;
6017 if (!isLegalBitRotate(SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
6018 return SDValue();
6019
6020 SDValue Op = DAG.getBitcast(RotateVT, SVN->getOperand(0));
6021
6022 SDValue Rotate;
6023 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
6024 // so canonicalize to vrev8.
6025 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
6026 Rotate = DAG.getNode(ISD::BSWAP, DL, RotateVT, Op);
6027 else
6028 Rotate = DAG.getNode(ISD::ROTL, DL, RotateVT, Op,
6029 DAG.getConstant(RotateAmt, DL, RotateVT));
6030
6031 return DAG.getBitcast(VT, Rotate);
6032}
6033
6034// If compiling with an exactly known VLEN, see if we can split a
6035// shuffle on m2 or larger into a small number of m1 sized shuffles
6036// which write each destination registers exactly once.
6037static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
6038 SelectionDAG &DAG,
6039 const RISCVSubtarget &Subtarget) {
6040 SDLoc DL(SVN);
6041 MVT VT = SVN->getSimpleValueType(0);
6042 SDValue V1 = SVN->getOperand(0);
6043 SDValue V2 = SVN->getOperand(1);
6044 ArrayRef<int> Mask = SVN->getMask();
6045
6046 // If we don't know exact data layout, not much we can do. If this
6047 // is already m1 or smaller, no point in splitting further.
6048 const auto VLen = Subtarget.getRealVLen();
6049 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
6050 return SDValue();
6051
6052 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
6053 // expansion for.
6054 unsigned RotateAmt;
6055 MVT RotateVT;
6056 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
6057 return SDValue();
6058
6059 MVT ElemVT = VT.getVectorElementType();
6060 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
6061
6062 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
6063 MVT OneRegVT = MVT::getVectorVT(ElemVT, ElemsPerVReg);
6064 MVT M1VT = getContainerForFixedLengthVector(OneRegVT, Subtarget);
6065 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
6066 unsigned NumOpElts = M1VT.getVectorMinNumElements();
6067 unsigned NumElts = ContainerVT.getVectorMinNumElements();
6068 unsigned NumOfSrcRegs = NumElts / NumOpElts;
6069 unsigned NumOfDestRegs = NumElts / NumOpElts;
6070 // The following semantically builds up a fixed length concat_vector
6071 // of the component shuffle_vectors. We eagerly lower to scalable here
6072 // to avoid DAG combining it back to a large shuffle_vector again.
6073 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6074 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
6075 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
6076 Operands;
6077 processShuffleMasks(
6078 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs,
6079 [&]() { Operands.emplace_back(); },
6080 [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
6081 Operands.emplace_back().emplace_back(SrcVecIdx, UINT_MAX,
6082 SmallVector<int>(SrcSubMask));
6083 },
6084 [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
6085 if (NewReg)
6086 Operands.emplace_back();
6087 Operands.back().emplace_back(Idx1, Idx2, SmallVector<int>(SrcSubMask));
6088 });
6089 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
6090 // Note: check that we do not emit too many shuffles here to prevent code
6091 // size explosion.
6092 // TODO: investigate, if it can be improved by extra analysis of the masks to
6093 // check if the code is more profitable.
6094 unsigned NumShuffles = std::accumulate(
6095 Operands.begin(), Operands.end(), 0u,
6096 [&](unsigned N,
6097 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
6098 if (Data.empty())
6099 return N;
6100 N += Data.size();
6101 for (const auto &P : Data) {
6102 unsigned Idx2 = std::get<1>(P);
6103 ArrayRef<int> Mask = std::get<2>(P);
6104 if (Idx2 != UINT_MAX)
6105 ++N;
6106 else if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size()))
6107 --N;
6108 }
6109 return N;
6110 });
6111 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6112 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6113 return SDValue();
6114 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6115 SDValue SubVec = DAG.getExtractSubvector(DL, M1VT, SrcVec, ExtractIdx);
6116 SubVec = convertFromScalableVector(OneRegVT, SubVec, DAG, Subtarget);
6117 return SubVec;
6118 };
6119 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6120 ArrayRef<int> Mask) {
6121 SDValue SubVec = DAG.getVectorShuffle(OneRegVT, DL, SubVec1, SubVec2, Mask);
6122 return SubVec;
6123 };
6124 SDValue Vec = DAG.getUNDEF(ContainerVT);
6125 for (auto [I, Data] : enumerate(Operands)) {
6126 if (Data.empty())
6127 continue;
6128 SmallDenseMap<unsigned, SDValue, 4> Values;
6129 for (unsigned I : seq<unsigned>(Data.size())) {
6130 const auto &[Idx1, Idx2, _] = Data[I];
6131 // If the shuffle contains permutation of odd number of elements,
6132 // Idx1 might be used already in the first iteration.
6133 //
6134 // Idx1 = shuffle Idx1, Idx2
6135 // Idx1 = shuffle Idx1, Idx3
6136 SDValue &V = Values.try_emplace(Idx1).first->getSecond();
6137 if (!V)
6138 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6139 (Idx1 % NumOfSrcRegs) * NumOpElts);
6140 if (Idx2 != UINT_MAX) {
6141 SDValue &V = Values.try_emplace(Idx2).first->getSecond();
6142 if (!V)
6143 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6144 (Idx2 % NumOfSrcRegs) * NumOpElts);
6145 }
6146 }
6147 SDValue V;
6148 for (const auto &[Idx1, Idx2, Mask] : Data) {
6149 SDValue V1 = Values.at(Idx1);
6150 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Idx2);
6151 V = PerformShuffle(V1, V2, Mask);
6152 Values[Idx1] = V;
6153 }
6154
6155 unsigned InsertIdx = I * NumOpElts;
6156 V = convertToScalableVector(M1VT, V, DAG, Subtarget);
6157 Vec = DAG.getInsertSubvector(DL, Vec, V, InsertIdx);
6158 }
6159 return convertFromScalableVector(VT, Vec, DAG, Subtarget);
6160}
6161
6162// Matches a subset of compress masks with a contiguous prefix of output
6163// elements. This could be extended to allow gaps by deciding which
6164// source elements to spuriously demand.
6165static bool isCompressMask(ArrayRef<int> Mask) {
6166 int Last = -1;
6167 bool SawUndef = false;
6168 for (const auto &[Idx, M] : enumerate(Mask)) {
6169 if (M == -1) {
6170 SawUndef = true;
6171 continue;
6172 }
6173 if (SawUndef)
6174 return false;
6175 if (Idx > (unsigned)M)
6176 return false;
6177 if (M <= Last)
6178 return false;
6179 Last = M;
6180 }
6181 return true;
6182}
6183
6184/// Given a shuffle where the indices are disjoint between the two sources,
6185/// e.g.:
6186///
6187/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6188///
6189/// Merge the two sources into one and do a single source shuffle:
6190///
6191/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6192/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6193///
6194/// A vselect will either be merged into a masked instruction or be lowered as a
6195/// vmerge.vvm, which is cheaper than a vrgather.vv.
6196static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6197 SelectionDAG &DAG,
6198 const RISCVSubtarget &Subtarget) {
6199 MVT VT = SVN->getSimpleValueType(0);
6200 MVT XLenVT = Subtarget.getXLenVT();
6201 SDLoc DL(SVN);
6202
6203 const ArrayRef<int> Mask = SVN->getMask();
6204
6205 // Work out which source each lane will come from.
6206 SmallVector<int, 16> Srcs(Mask.size(), -1);
6207
6208 for (int Idx : Mask) {
6209 if (Idx == -1)
6210 continue;
6211 unsigned SrcIdx = Idx % Mask.size();
6212 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6213 if (Srcs[SrcIdx] == -1)
6214 // Mark this source as using this lane.
6215 Srcs[SrcIdx] = Src;
6216 else if (Srcs[SrcIdx] != Src)
6217 // The other source is using this lane: not disjoint.
6218 return SDValue();
6219 }
6220
6221 SmallVector<SDValue> SelectMaskVals;
6222 for (int Lane : Srcs) {
6223 if (Lane == -1)
6224 SelectMaskVals.push_back(DAG.getUNDEF(XLenVT));
6225 else
6226 SelectMaskVals.push_back(DAG.getConstant(Lane ? 0 : 1, DL, XLenVT));
6227 }
6228 MVT MaskVT = VT.changeVectorElementType(MVT::i1);
6229 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, SelectMaskVals);
6230 SDValue Select = DAG.getNode(ISD::VSELECT, DL, VT, SelectMask,
6231 SVN->getOperand(0), SVN->getOperand(1));
6232
6233 // Move all indices relative to the first source.
6234 SmallVector<int> NewMask(Mask.size());
6235 for (unsigned I = 0; I < Mask.size(); I++) {
6236 if (Mask[I] == -1)
6237 NewMask[I] = -1;
6238 else
6239 NewMask[I] = Mask[I] % Mask.size();
6240 }
6241
6242 return DAG.getVectorShuffle(VT, DL, Select, DAG.getUNDEF(VT), NewMask);
6243}
6244
6245/// Is this mask local (i.e. elements only move within their local span), and
6246/// repeating (that is, the same rearrangement is being done within each span)?
6247static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6248 // Require a prefix from the original mask until the consumer code
6249 // is adjusted to rewrite the mask instead of just taking a prefix.
6250 for (auto [I, M] : enumerate(Mask)) {
6251 if (M == -1)
6252 continue;
6253 if ((M / Span) != (int)(I / Span))
6254 return false;
6255 int SpanIdx = I % Span;
6256 int Expected = M % Span;
6257 if (Mask[SpanIdx] != Expected)
6258 return false;
6259 }
6260 return true;
6261}
6262
6263/// Is this mask only using elements from the first span of the input?
6264static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6265 return all_of(Mask, [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6266}
6267
6268/// Return true for a mask which performs an arbitrary shuffle within the first
6269/// span, and then repeats that same result across all remaining spans. Note
6270/// that this doesn't check if all the inputs come from a single span!
6271static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6272 // Require a prefix from the original mask until the consumer code
6273 // is adjusted to rewrite the mask instead of just taking a prefix.
6274 for (auto [I, M] : enumerate(Mask)) {
6275 if (M == -1)
6276 continue;
6277 int SpanIdx = I % Span;
6278 if (Mask[SpanIdx] != M)
6279 return false;
6280 }
6281 return true;
6282}
6283
6284/// Try to widen element type to get a new mask value for a better permutation
6285/// sequence. This doesn't try to inspect the widened mask for profitability;
6286/// we speculate the widened form is equal or better. This has the effect of
6287/// reducing mask constant sizes - allowing cheaper materialization sequences
6288/// - and index sequence sizes - reducing register pressure and materialization
6289/// cost, at the cost of (possibly) an extra VTYPE toggle.
6290static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6291 SDLoc DL(Op);
6292 MVT VT = Op.getSimpleValueType();
6293 MVT ScalarVT = VT.getVectorElementType();
6294 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6295 SDValue V0 = Op.getOperand(0);
6296 SDValue V1 = Op.getOperand(1);
6297 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
6298
6299 // Avoid wasted work leading to isTypeLegal check failing below
6300 if (ElementSize > 32)
6301 return SDValue();
6302
6303 SmallVector<int, 8> NewMask;
6304 if (!widenShuffleMaskElts(Mask, NewMask))
6305 return SDValue();
6306
6307 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(ElementSize * 2)
6308 : MVT::getIntegerVT(ElementSize * 2);
6309 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
6310 if (!DAG.getTargetLoweringInfo().isTypeLegal(NewVT))
6311 return SDValue();
6312 V0 = DAG.getBitcast(NewVT, V0);
6313 V1 = DAG.getBitcast(NewVT, V1);
6314 return DAG.getBitcast(VT, DAG.getVectorShuffle(NewVT, DL, V0, V1, NewMask));
6315}
6316
6317// Match an interleave shuffle that forms a P-extension packed zip:
6318// <a0, b0, a1, b1, ...> -> zip*p/wzip*p
6319static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
6320 SelectionDAG &DAG) {
6321 SDValue V1 = SVN->getOperand(0);
6322 SDValue V2 = SVN->getOperand(1);
6323 SDLoc DL(SVN);
6324 MVT VT = SVN->getSimpleValueType(0);
6325 unsigned NumElts = VT.getVectorNumElements();
6326 ArrayRef<int> Mask = SVN->getMask();
6327
6328 if (VT != MVT::v8i8 && VT != MVT::v4i16)
6329 return SDValue();
6330
6331 SmallVector<unsigned, 2> StartIndexes;
6332 if (!V2.isUndef() &&
6333 ShuffleVectorInst::isInterleaveMask(Mask, 2, NumElts * 2, StartIndexes)) {
6334 unsigned EvenSrc = StartIndexes[0];
6335 unsigned OddSrc = StartIndexes[1];
6336 if (EvenSrc == 0 && OddSrc == NumElts)
6337 return DAG.getNode(RISCVISD::PZIP, DL, VT, V1, V2);
6338 if (EvenSrc == NumElts && OddSrc == 0)
6339 return DAG.getNode(RISCVISD::PZIP, DL, VT, V2, V1);
6340 }
6341
6342 return SDValue();
6343}
6344
6345SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6346 SelectionDAG &DAG) const {
6347 SDValue V1 = Op.getOperand(0);
6348 SDValue V2 = Op.getOperand(1);
6349 SDLoc DL(Op);
6350 MVT XLenVT = Subtarget.getXLenVT();
6351 MVT VT = Op.getSimpleValueType();
6352 unsigned NumElts = VT.getVectorNumElements();
6353 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
6354
6355 // Select an element reverse shuffle to VECTOR_REVERSE (rev8/rev16/ppairoe.*).
6356 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
6357 // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
6358 // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
6359 ArrayRef<int> Mask = SVN->getMask();
6360 auto IsLowReverse = [&](unsigned L) {
6361 return V2.isUndef() &&
6362 ShuffleVectorInst::isReverseMask(Mask.take_front(L), L) &&
6363 all_of(Mask.drop_front(L), [](int M) { return M < 0; });
6364 };
6365 if (IsLowReverse(NumElts))
6366 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6367 // Widened: reversing sends the low-half lanes to the top half, so shift
6368 // them back down by half the register. Only the 64-bit packed types are
6369 // legal here, so the register is XLen (i64).
6370 if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
6371 IsLowReverse(NumElts / 2)) {
6372 SDValue Rev = DAG.getBitcast(
6373 MVT::i64, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1));
6374 SDValue Srl =
6375 DAG.getNode(ISD::SRL, DL, MVT::i64, Rev,
6376 DAG.getConstant(VT.getSizeInBits() / 2, DL, MVT::i64));
6377 return DAG.getBitcast(VT, Srl);
6378 }
6379
6380 if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, DAG))
6381 return V;
6382 return SDValue();
6383 }
6384
6385 if (VT.getVectorElementType() == MVT::i1) {
6386 // Lower to a vror.vi of a larger element type if possible before we promote
6387 // i1s to i8s.
6388 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6389 return V;
6390 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6391 return V;
6392
6393 // Promote i1 shuffle to i8 shuffle.
6394 MVT WidenVT = MVT::getVectorVT(MVT::i8, VT.getVectorElementCount());
6395 V1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V1);
6396 V2 = V2.isUndef() ? DAG.getUNDEF(WidenVT)
6397 : DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, V2);
6398 SDValue Shuffled = DAG.getVectorShuffle(WidenVT, DL, V1, V2, SVN->getMask());
6399 return DAG.getSetCC(DL, VT, Shuffled, DAG.getConstant(0, DL, WidenVT),
6400 ISD::SETNE);
6401 }
6402
6403 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6404
6405 // Store the return value in a single variable instead of structured bindings
6406 // so that we can pass it to GetSlide below, which cannot capture structured
6407 // bindings until C++20.
6408 auto TrueMaskVL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
6409 auto [TrueMask, VL] = TrueMaskVL;
6410
6411 if (SVN->isSplat()) {
6412 const int Lane = SVN->getSplatIndex();
6413 if (Lane >= 0) {
6414 MVT SVT = VT.getVectorElementType();
6415
6416 // Turn splatted vector load into a strided load with an X0 stride.
6417 SDValue V = V1;
6418 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6419 // with undef.
6420 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6421 int Offset = Lane;
6422 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6423 int OpElements =
6424 V.getOperand(0).getSimpleValueType().getVectorNumElements();
6425 V = V.getOperand(Offset / OpElements);
6426 Offset %= OpElements;
6427 }
6428
6429 // We need to ensure the load isn't atomic or volatile.
6430 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
6431 auto *Ld = cast<LoadSDNode>(V);
6432 Offset *= SVT.getStoreSize();
6433 SDValue NewAddr = DAG.getMemBasePlusOffset(
6434 Ld->getBasePtr(), TypeSize::getFixed(Offset), DL);
6435
6436 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6437 if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
6438 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
6439 SDValue IntID =
6440 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
6441 SDValue Ops[] = {Ld->getChain(),
6442 IntID,
6443 DAG.getUNDEF(ContainerVT),
6444 NewAddr,
6445 DAG.getRegister(RISCV::X0, XLenVT),
6446 VL};
6447 SDValue NewLoad = DAG.getMemIntrinsicNode(
6448 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
6449 DAG.getMachineFunction().getMachineMemOperand(
6450 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
6451 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
6452 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
6453 }
6454
6455 MVT SplatVT = ContainerVT;
6456
6457 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6458 if (SVT == MVT::bf16 ||
6459 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6460 SVT = MVT::i16;
6461 SplatVT = ContainerVT.changeVectorElementType(SVT);
6462 }
6463
6464 // Otherwise use a scalar load and splat. This will give the best
6465 // opportunity to fold a splat into the operation. ISel can turn it into
6466 // the x0 strided load if we aren't able to fold away the select.
6467 if (SVT.isFloatingPoint())
6468 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
6469 Ld->getPointerInfo().getWithOffset(Offset),
6470 Ld->getBaseAlign(), Ld->getMemOperand()->getFlags());
6471 else
6472 V = DAG.getExtLoad(ISD::EXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
6473 Ld->getPointerInfo().getWithOffset(Offset), SVT,
6474 Ld->getBaseAlign(),
6475 Ld->getMemOperand()->getFlags());
6476 DAG.makeEquivalentMemoryOrdering(Ld, V);
6477
6478 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6479 : RISCVISD::VMV_V_X_VL;
6480 SDValue Splat =
6481 DAG.getNode(Opc, DL, SplatVT, DAG.getUNDEF(ContainerVT), V, VL);
6482 Splat = DAG.getBitcast(ContainerVT, Splat);
6483 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
6484 }
6485
6486 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6487 assert(Lane < (int)NumElts && "Unexpected lane!");
6488 SDValue Gather = DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT,
6489 V1, DAG.getConstant(Lane, DL, XLenVT),
6490 DAG.getUNDEF(ContainerVT), TrueMask, VL);
6491 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6492 }
6493 }
6494
6495 // For exact VLEN m2 or greater, try to split to m1 operations if we
6496 // can split cleanly.
6497 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6498 return V;
6499
6500 ArrayRef<int> Mask = SVN->getMask();
6501
6502 if (SDValue V =
6503 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6504 return V;
6505
6506 if (SDValue V =
6507 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6508 return V;
6509
6510 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6511 // available.
6512 if (Subtarget.hasStdExtZvkb())
6513 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6514 return V;
6515
6516 if (ShuffleVectorInst::isReverseMask(Mask, NumElts) && V2.isUndef() &&
6517 NumElts != 2)
6518 return DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V1);
6519
6520 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6521 // use shift and truncate to perform the shuffle.
6522 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6523 // shift-and-trunc reducing total cost for everything except an mf8 result.
6524 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6525 // to do the entire operation.
6526 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6527 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6528 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6529 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6530 unsigned Index = 0;
6531 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6532 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6533 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6534 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6535 if (1 < count_if(Mask,
6536 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6537 1 < count_if(Mask, [&Mask](int Idx) {
6538 return Idx >= (int)Mask.size();
6539 })) {
6540 // Narrow each source and concatenate them.
6541 // FIXME: For small LMUL it is better to concatenate first.
6542 MVT EltVT = VT.getVectorElementType();
6543 auto EltCnt = VT.getVectorElementCount();
6544 MVT SubVT =
6545 MVT::getVectorVT(EltVT, EltCnt.divideCoefficientBy(Factor));
6546
6547 SDValue Lo =
6548 getDeinterleaveShiftAndTrunc(DL, SubVT, V1, Factor, Index, DAG);
6549 SDValue Hi =
6550 getDeinterleaveShiftAndTrunc(DL, SubVT, V2, Factor, Index, DAG);
6551
6552 SDValue Concat =
6553 DAG.getNode(ISD::CONCAT_VECTORS, DL,
6554 SubVT.getDoubleNumVectorElementsVT(), Lo, Hi);
6555 if (Factor == 2)
6556 return Concat;
6557
6558 SDValue Vec = DAG.getUNDEF(VT);
6559 return DAG.getInsertSubvector(DL, Vec, Concat, 0);
6560 }
6561 }
6562 }
6563 }
6564
6565 // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
6566 // e64 which can't match above.
6567 unsigned Index = 0;
6568 if (Subtarget.hasStdExtZvzip() &&
6569 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, 2, Index) &&
6570 1 < count_if(Mask, [](int Idx) { return Idx != -1; })) {
6571 bool UsesBothSources =
6572 1 < count_if(Mask,
6573 [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6574 1 < count_if(Mask,
6575 [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
6576
6577 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
6578 unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
6579 MVT NewVT = VT.getDoubleNumVectorElementsVT();
6580 if (isTypeLegal(NewVT)) {
6581 SDValue Op;
6582 if (V2.isUndef()) {
6583 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6584 } else if (auto VLEN = Subtarget.getRealVLen();
6585 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
6586 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
6587 } else if (SDValue Src = foldConcatVector(V1, V2)) {
6588 Op = DAG.getExtractSubvector(DL, NewVT, Src, 0);
6589 }
6590 if (Op)
6591 return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
6592 }
6593
6594 if (UsesBothSources &&
6595 V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
6596 V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
6597 SDValue Lo = lowerZvzipVUNZIP(Opc, V1, DL, DAG, Subtarget);
6598 SDValue Hi = lowerZvzipVUNZIP(Opc, V2, DL, DAG, Subtarget);
6599 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
6600 }
6601 }
6602 }
6603
6604 if (SDValue V =
6605 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6606 return V;
6607
6608 // Detect an interleave shuffle and lower to
6609 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6610 int EvenSrc, OddSrc;
6611 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6612 !(NumElts == 2 &&
6613 ShuffleVectorInst::isSingleSourceMask(Mask, Mask.size()))) {
6614 // Extract the halves of the vectors.
6615 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6616
6617 // Recognize if one half is actually undef; the matching above will
6618 // otherwise reuse the even stream for the undef one. This improves
6619 // spread(2) shuffles.
6620 bool LaneIsUndef[2] = { true, true};
6621 for (const auto &[Idx, M] : enumerate(Mask))
6622 LaneIsUndef[Idx % 2] &= (M == -1);
6623
6624 int Size = Mask.size();
6625 SDValue EvenV, OddV;
6626 if (LaneIsUndef[0]) {
6627 EvenV = DAG.getUNDEF(HalfVT);
6628 } else {
6629 assert(EvenSrc >= 0 && "Undef source?");
6630 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6631 EvenV = DAG.getExtractSubvector(DL, HalfVT, EvenV, EvenSrc % Size);
6632 }
6633
6634 if (LaneIsUndef[1]) {
6635 OddV = DAG.getUNDEF(HalfVT);
6636 } else {
6637 assert(OddSrc >= 0 && "Undef source?");
6638 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6639 OddV = DAG.getExtractSubvector(DL, HalfVT, OddV, OddSrc % Size);
6640 }
6641
6642 // Prefer vzip if available.
6643 // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
6644 if (Subtarget.hasStdExtZvzip() && isLegalVTForZvzipOperand(VT, Subtarget))
6645 return lowerZvzipVZIP(EvenV, OddV, DL, DAG, Subtarget);
6646 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6647 }
6648
6649 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6650 // instructions (in any combination) with masking on the second instruction.
6651 // Also handles masked slides into an identity source, and single slides
6652 // without masking. Avoid matching bit rotates (which are not also element
6653 // rotates) as slide pairs. This is a performance heuristic, not a
6654 // functional check.
6655 std::array<std::pair<int, int>, 2> SrcInfo;
6656 unsigned RotateAmt;
6657 MVT RotateVT;
6658 if (::isMaskedSlidePair(Mask, SrcInfo) &&
6659 (isElementRotate(SrcInfo, NumElts) ||
6660 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
6661 SDValue Sources[2];
6662 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
6663 int SrcIdx = Info.first;
6664 assert(SrcIdx == 0 || SrcIdx == 1);
6665 SDValue &Src = Sources[SrcIdx];
6666 if (!Src) {
6667 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
6668 Src = convertToScalableVector(ContainerVT, SrcV, DAG, Subtarget);
6669 }
6670 return Src;
6671 };
6672 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
6673 SDValue Passthru) {
6674 auto [TrueMask, VL] = TrueMaskVL;
6675 SDValue SrcV = GetSourceFor(Src);
6676 int SlideAmt = Src.second;
6677 if (SlideAmt == 0) {
6678 // Should never be second operation
6679 assert(Mask == TrueMask);
6680 return SrcV;
6681 }
6682 if (SlideAmt < 0)
6683 return getVSlidedown(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6684 DAG.getConstant(-SlideAmt, DL, XLenVT), Mask, VL,
6685 RISCVVType::TAIL_AGNOSTIC);
6686 return getVSlideup(DAG, Subtarget, DL, ContainerVT, Passthru, SrcV,
6687 DAG.getConstant(SlideAmt, DL, XLenVT), Mask, VL,
6688 RISCVVType::TAIL_AGNOSTIC);
6689 };
6690
6691 if (SrcInfo[1].first == -1) {
6692 SDValue Res = DAG.getUNDEF(ContainerVT);
6693 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6694 return convertFromScalableVector(VT, Res, DAG, Subtarget);
6695 }
6696
6697 if (Subtarget.hasStdExtZvzip()) {
6698 bool TryWiden = false;
6699 unsigned Factor;
6700 if (isPairEven(SrcInfo, Mask, Factor)) {
6701 if (Factor == 1) {
6702 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
6703 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
6704 return lowerZvzipVPAIR(RISCVISD::VPAIRE_VL, Src1, Src2, DL, DAG,
6705 Subtarget);
6706 }
6707 TryWiden = true;
6708 }
6709 if (isPairOdd(SrcInfo, Mask, Factor)) {
6710 if (Factor == 1) {
6711 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
6712 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
6713 return lowerZvzipVPAIR(RISCVISD::VPAIRO_VL, Src1, Src2, DL, DAG,
6714 Subtarget);
6715 }
6716 TryWiden = true;
6717 }
6718 // If we found a widening oppurtunity which would let us form a
6719 // pair-even or pair-odd, use the generic code to widen the shuffle
6720 // and recurse through this logic.
6721 if (TryWiden)
6722 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6723 return V;
6724 }
6725
6726 // Build the mask. Note that vslideup unconditionally preserves elements
6727 // below the slide amount in the destination, and thus those elements are
6728 // undefined in the mask. If the mask ends up all true (or undef), it
6729 // will be folded away by general logic.
6730 SmallVector<SDValue> MaskVals;
6731 for (const auto &[Idx, M] : enumerate(Mask)) {
6732 if (M < 0 ||
6733 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
6734 MaskVals.push_back(DAG.getUNDEF(XLenVT));
6735 continue;
6736 }
6737 int Src = M >= (int)NumElts;
6738 int Diff = (int)Idx - (M % NumElts);
6739 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
6740 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
6741 "Must match exactly one of the two slides");
6742 MaskVals.push_back(DAG.getConstant(C, DL, XLenVT));
6743 }
6744 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
6745 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
6746 SDValue SelectMask = convertToScalableVector(
6747 ContainerVT.changeVectorElementType(MVT::i1),
6748 DAG.getBuildVector(MaskVT, DL, MaskVals), DAG, Subtarget);
6749
6750 SDValue Res = DAG.getUNDEF(ContainerVT);
6751 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6752 Res = GetSlide(SrcInfo[1], SelectMask, Res);
6753 return convertFromScalableVector(VT, Res, DAG, Subtarget);
6754 }
6755
6756 // Handle any remaining single source shuffles
6757 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
6758 if (V2.isUndef()) {
6759 // We might be able to express the shuffle as a bitrotate. But even if we
6760 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
6761 // shifts and a vor will have a higher throughput than a vrgather.
6762 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6763 return V;
6764
6765 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
6766 return V;
6767
6768 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
6769 // is fully covered in interleave(2) above, so it is ignored here.
6770 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6771 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6772 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6773 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
6774 unsigned Index;
6775 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
6776 MVT NarrowVT =
6777 MVT::getVectorVT(VT.getVectorElementType(), NumElts / Factor);
6778 SDValue Src = DAG.getExtractSubvector(DL, NarrowVT, V1, 0);
6779 return getWideningSpread(Src, Factor, Index, DL, DAG);
6780 }
6781 }
6782 }
6783
6784 // If only a prefix of the source elements influence a prefix of the
6785 // destination elements, try to see if we can reduce the required LMUL
6786 unsigned MinVLen = Subtarget.getRealMinVLen();
6787 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
6788 if (NumElts > MinVLMAX) {
6789 unsigned MaxIdx = 0;
6790 for (auto [I, M] : enumerate(Mask)) {
6791 if (M == -1)
6792 continue;
6793 MaxIdx = std::max({(unsigned)I, (unsigned)M, MaxIdx});
6794 }
6795 unsigned NewNumElts =
6796 std::max((uint64_t)MinVLMAX, PowerOf2Ceil(MaxIdx + 1));
6797 if (NewNumElts != NumElts) {
6798 MVT NewVT = MVT::getVectorVT(VT.getVectorElementType(), NewNumElts);
6799 V1 = DAG.getExtractSubvector(DL, NewVT, V1, 0);
6800 SDValue Res = DAG.getVectorShuffle(NewVT, DL, V1, DAG.getUNDEF(NewVT),
6801 Mask.take_front(NewNumElts));
6802 return DAG.getInsertSubvector(DL, DAG.getUNDEF(VT), Res, 0);
6803 }
6804 }
6805
6806 // Before hitting generic lowering fallbacks, try to widen the mask
6807 // to a wider SEW.
6808 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6809 return V;
6810
6811 // Can we generate a vcompress instead of a vrgather? These scale better
6812 // at high LMUL, at the cost of not being able to fold a following select
6813 // into them. The mask constants are also smaller than the index vector
6814 // constants, and thus easier to materialize.
6815 if (isCompressMask(Mask)) {
6816 SmallVector<SDValue> MaskVals(NumElts,
6817 DAG.getConstant(false, DL, XLenVT));
6818 for (auto Idx : Mask) {
6819 if (Idx == -1)
6820 break;
6821 assert(Idx >= 0 && (unsigned)Idx < NumElts);
6822 MaskVals[Idx] = DAG.getConstant(true, DL, XLenVT);
6823 }
6824 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
6825 SDValue CompressMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
6826 return DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, V1, CompressMask,
6827 DAG.getUNDEF(VT));
6828 }
6829
6830 if (VT.getScalarSizeInBits() == 8 &&
6831 any_of(Mask, [&](const auto &Idx) { return Idx > 255; })) {
6832 // On such a vector we're unable to use i8 as the index type.
6833 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
6834 // may involve vector splitting if we're already at LMUL=8, or our
6835 // user-supplied maximum fixed-length LMUL.
6836 return SDValue();
6837 }
6838
6839 // Base case for the two operand recursion below - handle the worst case
6840 // single source shuffle.
6841 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
6842 MVT IndexVT = VT.changeTypeToInteger();
6843 // Since we can't introduce illegal index types at this stage, use i16 and
6844 // vrgatherei16 if the corresponding index type for plain vrgather is greater
6845 // than XLenVT.
6846 if (IndexVT.getScalarType().bitsGT(XLenVT)) {
6847 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
6848 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
6849 }
6850
6851 // If the mask allows, we can do all the index computation in 16 bits. This
6852 // requires less work and less register pressure at high LMUL, and creates
6853 // smaller constants which may be cheaper to materialize.
6854 if (IndexVT.getScalarType().bitsGT(MVT::i16) && isUInt<16>(NumElts - 1) &&
6855 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
6856 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
6857 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
6858 }
6859
6860 MVT IndexContainerVT =
6861 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
6862
6863 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
6864 SmallVector<SDValue> GatherIndicesLHS;
6865 for (int MaskIndex : Mask) {
6866 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
6867 GatherIndicesLHS.push_back(IsLHSIndex
6868 ? DAG.getConstant(MaskIndex, DL, XLenVT)
6869 : DAG.getUNDEF(XLenVT));
6870 }
6871 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
6872 LHSIndices =
6873 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
6874 // At m1 and less, there's no point trying any of the high LMUL splitting
6875 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
6876 if (NumElts <= MinVLMAX) {
6877 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
6878 DAG.getUNDEF(ContainerVT), TrueMask, VL);
6879 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6880 }
6881
6882 const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
6883 EVT SubIndexVT = M1VT.changeVectorElementType(IndexVT.getScalarType());
6884 auto [InnerTrueMask, InnerVL] =
6885 getDefaultScalableVLOps(M1VT, DL, DAG, Subtarget);
6886 int N =
6887 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
6888 assert(isPowerOf2_32(N) && N <= 8);
6889
6890 // If we have a locally repeating mask, then we can reuse the first
6891 // register in the index register group for all registers within the
6892 // source register group. TODO: This generalizes to m2, and m4.
6893 if (isLocalRepeatingShuffle(Mask, MinVLMAX)) {
6894 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
6895 SDValue Gather = DAG.getUNDEF(ContainerVT);
6896 for (int i = 0; i < N; i++) {
6897 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
6898 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, SubIdx);
6899 SDValue SubVec =
6900 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
6901 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
6902 Gather = DAG.getInsertSubvector(DL, Gather, SubVec, SubIdx);
6903 }
6904 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6905 }
6906
6907 // If we have a shuffle which only uses the first register in our source
6908 // register group, and repeats the same index across all spans, we can
6909 // use a single vrgather (and possibly some register moves).
6910 // TODO: This can be generalized for m2 or m4, or for any shuffle for
6911 // which we can do a linear number of shuffles to form an m1 which
6912 // contains all the output elements.
6913 if (isLowSourceShuffle(Mask, MinVLMAX) &&
6914 isSpanSplatShuffle(Mask, MinVLMAX)) {
6915 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
6916 SDValue SubIndex = DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
6917 SDValue SubVec = DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
6918 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
6919 SDValue Gather = DAG.getUNDEF(ContainerVT);
6920 for (int i = 0; i < N; i++)
6921 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
6922 M1VT.getVectorMinNumElements() * i);
6923 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6924 }
6925
6926 // If we have a shuffle which only uses the first register in our
6927 // source register group, we can do a linear number of m1 vrgathers
6928 // reusing the same source register (but with different indices)
6929 // TODO: This can be generalized for m2 or m4, or for any shuffle
6930 // for which we can do a vslidedown followed by this expansion.
6931 if (isLowSourceShuffle(Mask, MinVLMAX)) {
6932 SDValue SlideAmt =
6933 DAG.getElementCount(DL, XLenVT, M1VT.getVectorElementCount());
6934 SDValue SubV1 = DAG.getExtractSubvector(DL, M1VT, V1, 0);
6935 SDValue Gather = DAG.getUNDEF(ContainerVT);
6936 for (int i = 0; i < N; i++) {
6937 if (i != 0)
6938 LHSIndices = getVSlidedown(DAG, Subtarget, DL, IndexContainerVT,
6939 DAG.getUNDEF(IndexContainerVT), LHSIndices,
6940 SlideAmt, TrueMask, VL);
6941 SDValue SubIndex =
6942 DAG.getExtractSubvector(DL, SubIndexVT, LHSIndices, 0);
6943 SDValue SubVec =
6944 DAG.getNode(GatherVVOpc, DL, M1VT, SubV1, SubIndex,
6945 DAG.getUNDEF(M1VT), InnerTrueMask, InnerVL);
6946 Gather = DAG.getInsertSubvector(DL, Gather, SubVec,
6947 M1VT.getVectorMinNumElements() * i);
6948 }
6949 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6950 }
6951
6952 // Fallback to generic vrgather if we can't find anything better.
6953 // On many machines, this will be O(LMUL^2)
6954 SDValue Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
6955 DAG.getUNDEF(ContainerVT), TrueMask, VL);
6956 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
6957 }
6958
6959 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
6960 // merged with a second vrgather.
6961 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
6962
6963 // Now construct the mask that will be used by the blended vrgather operation.
6964 // Construct the appropriate indices into each vector.
6965 for (int MaskIndex : Mask) {
6966 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
6967 ShuffleMaskLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
6968 ? MaskIndex : -1);
6969 ShuffleMaskRHS.push_back(IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
6970 }
6971
6972 // If the mask indices are disjoint between the two sources, we can lower it
6973 // as a vselect + a single source vrgather.vv. Don't do this if we think the
6974 // operands may end up being lowered to something cheaper than a vrgather.vv.
6975 if (!DAG.isSplatValue(V2) && !DAG.isSplatValue(V1) &&
6976 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskLHS) &&
6977 !ShuffleVectorSDNode::isSplatMask(ShuffleMaskRHS) &&
6978 !ShuffleVectorInst::isIdentityMask(ShuffleMaskLHS, NumElts) &&
6979 !ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts))
6980 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
6981 return V;
6982
6983 // Before hitting generic lowering fallbacks, try to widen the mask
6984 // to a wider SEW.
6985 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6986 return V;
6987
6988 // Try to pick a profitable operand order.
6989 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
6990 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(ShuffleMaskRHS, NumElts);
6991
6992 // Recursively invoke lowering for each operand if we had two
6993 // independent single source shuffles, and then combine the result via a
6994 // vselect. Note that the vselect will likely be folded back into the
6995 // second permute (vrgather, or other) by the post-isel combine.
6996 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), ShuffleMaskLHS);
6997 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), ShuffleMaskRHS);
6998
6999 SmallVector<SDValue> MaskVals;
7000 for (int MaskIndex : Mask) {
7001 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
7002 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
7003 }
7004
7005 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7006 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
7007 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
7008
7009 if (SwapOps)
7010 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
7011 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V2, V1);
7012}
7013
7014bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7015 // Only support legal VTs for other shuffles for now.
7016 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
7017 return false;
7018
7019 // Support splats for any type. These should type legalize well.
7020 if (ShuffleVectorSDNode::isSplatMask(M))
7021 return true;
7022
7023 const unsigned NumElts = M.size();
7024 MVT SVT = VT.getSimpleVT();
7025
7026 // Not for i1 vectors.
7027 if (SVT.getScalarType() == MVT::i1)
7028 return false;
7029
7030 std::array<std::pair<int, int>, 2> SrcInfo;
7031 int Dummy1, Dummy2;
7032 return ShuffleVectorInst::isReverseMask(M, NumElts) ||
7033 (::isMaskedSlidePair(M, SrcInfo) &&
7034 isElementRotate(SrcInfo, NumElts)) ||
7035 isInterleaveShuffle(M, SVT, Dummy1, Dummy2, Subtarget);
7036}
7037
7038// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
7039// the exponent.
7040SDValue
7041RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
7042 SelectionDAG &DAG) const {
7043 MVT VT = Op.getSimpleValueType();
7044 unsigned EltSize = VT.getScalarSizeInBits();
7045 SDValue Src = Op.getOperand(0);
7046 SDLoc DL(Op);
7047 MVT ContainerVT = VT;
7048
7049 // We choose FP type that can represent the value if possible. Otherwise, we
7050 // use rounding to zero conversion for correct exponent of the result.
7051 // TODO: Use f16 for i8 when possible?
7052 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
7053 if (!isTypeLegal(MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount())))
7054 FloatEltVT = MVT::f32;
7055 MVT FloatVT = MVT::getVectorVT(FloatEltVT, VT.getVectorElementCount());
7056
7057 // Legal types should have been checked in the RISCVTargetLowering
7058 // constructor.
7059 // TODO: Splitting may make sense in some cases.
7060 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
7061 "Expected legal float type!");
7062
7063 // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
7064 // The trailing zero count is equal to log2 of this single bit value.
7065 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
7066 SDValue Neg = DAG.getNegative(Src, DL, VT);
7067 Src = DAG.getNode(ISD::AND, DL, VT, Src, Neg);
7068 }
7069
7070 // We have a legal FP type, convert to it.
7071 SDValue FloatVal;
7072 if (FloatVT.bitsGT(VT)) {
7073 FloatVal = DAG.getNode(ISD::UINT_TO_FP, DL, FloatVT, Src);
7074 } else {
7075 // Use RTZ to avoid rounding influencing exponent of FloatVal.
7076 if (VT.isFixedLengthVector()) {
7077 ContainerVT = getContainerForFixedLengthVector(VT);
7078 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
7079 }
7080 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7081 SDValue RTZRM =
7082 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, Subtarget.getXLenVT());
7083 MVT ContainerFloatVT =
7084 MVT::getVectorVT(FloatEltVT, ContainerVT.getVectorElementCount());
7085 FloatVal = DAG.getNode(RISCVISD::VFCVT_RM_F_XU_VL, DL, ContainerFloatVT,
7086 Src, Mask, RTZRM, VL);
7087 if (VT.isFixedLengthVector())
7088 FloatVal = convertFromScalableVector(FloatVT, FloatVal, DAG, Subtarget);
7089 }
7090 // Bitcast to integer and shift the exponent to the LSB.
7091 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
7092 SDValue Bitcast = DAG.getBitcast(IntVT, FloatVal);
7093 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
7094
7095 // Restore back to original type. Truncation after SRL is to generate vnsrl.
7096 SDValue Exp = DAG.getNode(ISD::SRL, DL, IntVT, Bitcast,
7097 DAG.getConstant(ShiftAmt, DL, IntVT));
7098 if (IntVT.bitsLT(VT))
7099 Exp = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Exp);
7100 else if (IntVT.bitsGT(VT))
7101 Exp = DAG.getNode(ISD::TRUNCATE, DL, VT, Exp);
7102
7103 // The exponent contains log2 of the value in biased form.
7104 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
7105 // For trailing zeros, we just need to subtract the bias.
7106 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
7107 return DAG.getNode(ISD::SUB, DL, VT, Exp,
7108 DAG.getConstant(ExponentBias, DL, VT));
7109
7110 // For leading zeros, we need to remove the bias and convert from log2 to
7111 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
7112 unsigned Adjust = ExponentBias + (EltSize - 1);
7113 SDValue Res =
7114 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(Adjust, DL, VT), Exp);
7115
7116 // The above result with zero input equals to Adjust which is greater than
7117 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
7118 if (Op.getOpcode() == ISD::CTLZ)
7119 Res = DAG.getNode(ISD::UMIN, DL, VT, Res, DAG.getConstant(EltSize, DL, VT));
7120
7121 return Res;
7122}
7123
7124SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7125 SelectionDAG &DAG) const {
7126 SDLoc DL(Op);
7127 MVT XLenVT = Subtarget.getXLenVT();
7128 SDValue Source = Op->getOperand(0);
7129 MVT SrcVT = Source.getSimpleValueType();
7130 SDValue Mask = Op->getOperand(1);
7131 SDValue EVL = Op->getOperand(2);
7132
7133 if (SrcVT.isFixedLengthVector()) {
7134 MVT ContainerVT = getContainerForFixedLengthVector(SrcVT);
7135 Source = convertToScalableVector(ContainerVT, Source, DAG, Subtarget);
7136 Mask = convertToScalableVector(getMaskTypeFor(ContainerVT), Mask, DAG,
7137 Subtarget);
7138 SrcVT = ContainerVT;
7139 }
7140
7141 // Convert to boolean vector.
7142 if (SrcVT.getScalarType() != MVT::i1) {
7143 SDValue AllZero = DAG.getConstant(0, DL, SrcVT);
7144 SrcVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorElementCount());
7145 Source = DAG.getNode(RISCVISD::SETCC_VL, DL, SrcVT,
7146 {Source, AllZero, DAG.getCondCode(ISD::SETNE),
7147 DAG.getUNDEF(SrcVT), Mask, EVL});
7148 }
7149
7150 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Source, Mask, EVL);
7151 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
7152 // In this case, we can interpret poison as -1, so nothing to do further.
7153 return Res;
7154
7155 // Convert -1 to VL.
7156 SDValue SetCC =
7157 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
7158 Res = DAG.getSelect(DL, XLenVT, SetCC, EVL, Res);
7159 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
7160}
7161
7162// While RVV has alignment restrictions, we should always be able to load as a
7163// legal equivalently-sized byte-typed vector instead. This method is
7164// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7165// the load is already correctly-aligned, it returns SDValue().
7166SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7167 SelectionDAG &DAG) const {
7168 auto *Load = cast<LoadSDNode>(Op);
7169 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7170
7171 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7172 Load->getMemoryVT(),
7173 *Load->getMemOperand()))
7174 return SDValue();
7175
7176 SDLoc DL(Op);
7177 MVT VT = Op.getSimpleValueType();
7178 unsigned EltSizeBits = VT.getScalarSizeInBits();
7179 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7180 "Unexpected unaligned RVV load type");
7181 MVT NewVT =
7182 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7183 assert(NewVT.isValid() &&
7184 "Expecting equally-sized RVV vector types to be legal");
7185 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7186 Load->getPointerInfo(), Load->getBaseAlign(),
7187 Load->getMemOperand()->getFlags());
7188 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7189}
7190
7191// While RVV has alignment restrictions, we should always be able to store as a
7192// legal equivalently-sized byte-typed vector instead. This method is
7193// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7194// returns SDValue() if the store is already correctly aligned.
7195SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7196 SelectionDAG &DAG) const {
7197 auto *Store = cast<StoreSDNode>(Op);
7198 assert(Store && Store->getValue().getValueType().isVector() &&
7199 "Expected vector store");
7200
7201 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7202 Store->getMemoryVT(),
7203 *Store->getMemOperand()))
7204 return SDValue();
7205
7206 SDLoc DL(Op);
7207 SDValue StoredVal = Store->getValue();
7208 MVT VT = StoredVal.getSimpleValueType();
7209 unsigned EltSizeBits = VT.getScalarSizeInBits();
7210 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7211 "Unexpected unaligned RVV store type");
7212 MVT NewVT =
7213 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7214 assert(NewVT.isValid() &&
7215 "Expecting equally-sized RVV vector types to be legal");
7216 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7217 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7218 Store->getPointerInfo(), Store->getBaseAlign(),
7219 Store->getMemOperand()->getFlags());
7220}
7221
7222// While RVV has alignment restrictions, we should always be able to load as a
7223// legal equivalently-sized byte-typed vector instead. This method is
7224// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7225// the load is already correctly-aligned, it returns SDValue().
7226SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7227 SelectionDAG &DAG) const {
7228 auto *Load = cast<VPLoadSDNode>(Op);
7229 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7230
7231 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7232 Load->getMemoryVT(),
7233 *Load->getMemOperand()))
7234 return SDValue();
7235
7236 SDValue Mask = Load->getMask();
7237
7238 // FIXME: Handled masked loads somehow.
7239 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7240 return SDValue();
7241
7242 SDLoc DL(Op);
7243 MVT VT = Op.getSimpleValueType();
7244 unsigned EltSizeBits = VT.getScalarSizeInBits();
7245 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7246 "Unexpected unaligned RVV load type");
7247 MVT NewVT =
7248 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7249 assert(NewVT.isValid() &&
7250 "Expecting equally-sized RVV vector types to be legal");
7251
7252 SDValue VL = Load->getVectorLength();
7253 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7254 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7255
7256 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7257 SDValue L = DAG.getLoadVP(NewVT, DL, Load->getChain(), Load->getBasePtr(),
7258 DAG.getAllOnesConstant(DL, MaskVT), VL,
7259 Load->getPointerInfo(), Load->getBaseAlign(),
7260 Load->getMemOperand()->getFlags(), AAMDNodes());
7261 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
7262}
7263
7264// While RVV has alignment restrictions, we should always be able to store as a
7265// legal equivalently-sized byte-typed vector instead. This method is
7266// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7267// It returns SDValue() if the store is already correctly aligned.
7268SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7269 SelectionDAG &DAG) const {
7270 auto *Store = cast<VPStoreSDNode>(Op);
7271 assert(Store && Store->getValue().getValueType().isVector() &&
7272 "Expected vector store");
7273
7274 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
7275 Store->getMemoryVT(),
7276 *Store->getMemOperand()))
7277 return SDValue();
7278
7279 SDValue Mask = Store->getMask();
7280
7281 // FIXME: Handled masked stores somehow.
7282 if (!ISD::isConstantSplatVectorAllOnes(Mask.getNode()))
7283 return SDValue();
7284
7285 SDLoc DL(Op);
7286 SDValue StoredVal = Store->getValue();
7287 MVT VT = StoredVal.getSimpleValueType();
7288 unsigned EltSizeBits = VT.getScalarSizeInBits();
7289 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7290 "Unexpected unaligned RVV store type");
7291 MVT NewVT =
7292 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
7293 assert(NewVT.isValid() &&
7294 "Expecting equally-sized RVV vector types to be legal");
7295
7296 SDValue VL = Store->getVectorLength();
7297 VL = DAG.getNode(ISD::MUL, DL, VL.getValueType(), VL,
7298 DAG.getConstant((EltSizeBits / 8), DL, VL.getValueType()));
7299
7300 StoredVal = DAG.getBitcast(NewVT, StoredVal);
7301
7302 LocationSize Size = LocationSize::precise(NewVT.getStoreSize());
7303 MachineFunction &MF = DAG.getMachineFunction();
7304 MachineMemOperand *MMO = MF.getMachineMemOperand(
7305 Store->getPointerInfo(), Store->getMemOperand()->getFlags(), Size,
7306 Store->getBaseAlign());
7307
7308 MVT MaskVT = MVT::getVectorVT(MVT::i1, NewVT.getVectorElementCount());
7309 return DAG.getStoreVP(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
7310 DAG.getUNDEF(Store->getBasePtr().getValueType()),
7311 DAG.getAllOnesConstant(DL, MaskVT), VL, NewVT, MMO,
7312 ISD::UNINDEXED);
7313}
7314
7315static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7316 const RISCVSubtarget &Subtarget) {
7317 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7318
7319 int64_t Imm = cast<ConstantSDNode>(Op)->getSExtValue();
7320
7321 // All simm32 constants should be handled by isel.
7322 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7323 // this check redundant, but small immediates are common so this check
7324 // should have better compile time.
7325 if (isInt<32>(Imm))
7326 return Op;
7327
7328 // We only need to cost the immediate, if constant pool lowering is enabled.
7329 if (!Subtarget.useConstantPoolForLargeInts())
7330 return Op;
7331
7332 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, Subtarget);
7333 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7334 return Op;
7335
7336 // Optimizations below are disabled for opt size. If we're optimizing for
7337 // size, use a constant pool.
7338 if (DAG.shouldOptForSize())
7339 return SDValue();
7340
7341 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7342 // that if it will avoid a constant pool.
7343 // It will require an extra temporary register though.
7344 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7345 // low and high 32 bits are the same and bit 31 and 63 are set.
7346 unsigned ShiftAmt, AddOpc;
7347 RISCVMatInt::InstSeq SeqLo =
7348 RISCVMatInt::generateTwoRegInstSeq(Imm, Subtarget, ShiftAmt, AddOpc);
7349 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7350 return Op;
7351
7352 return SDValue();
7353}
7354
7355SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7356 SelectionDAG &DAG) const {
7357 MVT VT = Op.getSimpleValueType();
7358 const APFloat &Imm = cast<ConstantFPSDNode>(Op)->getValueAPF();
7359
7360 // Can this constant be selected by a Zfa FLI instruction?
7361 bool Negate = false;
7362 int Index = getLegalZfaFPImm(Imm, VT);
7363
7364 // If the constant is negative, try negating.
7365 if (Index < 0 && Imm.isNegative()) {
7366 Index = getLegalZfaFPImm(-Imm, VT);
7367 Negate = true;
7368 }
7369
7370 // If we couldn't find a FLI lowering, fall back to generic code.
7371 if (Index < 0)
7372 return SDValue();
7373
7374 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7375 SDLoc DL(Op);
7376 SDValue Const =
7377 DAG.getNode(RISCVISD::FLI, DL, VT,
7378 DAG.getTargetConstant(Index, DL, Subtarget.getXLenVT()));
7379 if (!Negate)
7380 return Const;
7381
7382 return DAG.getNode(ISD::FNEG, DL, VT, Const);
7383}
7384
7385static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7386 SelectionDAG &DAG) {
7387
7388 unsigned IsData = Op.getConstantOperandVal(4);
7389
7390 // mips-p8700 we support data prefetch for now.
7391 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7392 return Op.getOperand(0);
7393 return Op;
7394}
7395
7396static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7397 const RISCVSubtarget &Subtarget) {
7398 SDLoc dl(Op);
7399 AtomicOrdering FenceOrdering =
7400 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
7401 SyncScope::ID FenceSSID =
7402 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
7403
7404 if (Subtarget.hasStdExtZtso()) {
7405 // The only fence that needs an instruction is a sequentially-consistent
7406 // cross-thread fence.
7407 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7408 FenceSSID == SyncScope::System)
7409 return Op;
7410
7411 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7412 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7413 }
7414
7415 // singlethread fences only synchronize with signal handlers on the same
7416 // thread and thus only need to preserve instruction order, not actually
7417 // enforce memory ordering.
7418 if (FenceSSID == SyncScope::SingleThread)
7419 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7420 return DAG.getNode(ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
7421
7422 return Op;
7423}
7424
7425SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7426 SelectionDAG &DAG) const {
7427 SDLoc DL(Op);
7428 MVT VT = Op.getSimpleValueType();
7429 MVT XLenVT = Subtarget.getXLenVT();
7430 unsigned Check = Op.getConstantOperandVal(1);
7431 unsigned TDCMask = 0;
7432 if (Check & fcSNan)
7433 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7434 if (Check & fcQNan)
7435 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7436 if (Check & fcPosInf)
7437 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7438 if (Check & fcNegInf)
7439 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7440 if (Check & fcPosNormal)
7441 TDCMask |= RISCV::FPMASK_Positive_Normal;
7442 if (Check & fcNegNormal)
7443 TDCMask |= RISCV::FPMASK_Negative_Normal;
7444 if (Check & fcPosSubnormal)
7445 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7446 if (Check & fcNegSubnormal)
7447 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7448 if (Check & fcPosZero)
7449 TDCMask |= RISCV::FPMASK_Positive_Zero;
7450 if (Check & fcNegZero)
7451 TDCMask |= RISCV::FPMASK_Negative_Zero;
7452
7453 bool IsOneBitMask = isPowerOf2_32(TDCMask);
7454
7455 SDValue TDCMaskV = DAG.getConstant(TDCMask, DL, XLenVT);
7456
7457 if (VT.isVector()) {
7458 SDValue Op0 = Op.getOperand(0);
7459 MVT VT0 = Op.getOperand(0).getSimpleValueType();
7460
7461 if (VT.isScalableVector()) {
7462 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7463 auto [Mask, VL] = getDefaultScalableVLOps(VT0, DL, DAG, Subtarget);
7464 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, DstVT, Op0, Mask,
7465 VL, Op->getFlags());
7466 if (IsOneBitMask)
7467 return DAG.getSetCC(DL, VT, FPCLASS,
7468 DAG.getConstant(TDCMask, DL, DstVT),
7469 ISD::CondCode::SETEQ);
7470 SDValue AND = DAG.getNode(ISD::AND, DL, DstVT, FPCLASS,
7471 DAG.getConstant(TDCMask, DL, DstVT));
7472 return DAG.getSetCC(DL, VT, AND, DAG.getConstant(0, DL, DstVT),
7473 ISD::SETNE);
7474 }
7475
7476 MVT ContainerVT0 = getContainerForFixedLengthVector(VT0);
7477 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7478 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7479 auto [Mask, VL] = getDefaultVLOps(VT0, ContainerVT0, DL, DAG, Subtarget);
7480 Op0 = convertToScalableVector(ContainerVT0, Op0, DAG, Subtarget);
7481
7482 SDValue FPCLASS = DAG.getNode(RISCVISD::FCLASS_VL, DL, ContainerDstVT, Op0,
7483 Mask, VL, Op->getFlags());
7484
7485 TDCMaskV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7486 DAG.getUNDEF(ContainerDstVT), TDCMaskV, VL);
7487 if (IsOneBitMask) {
7488 SDValue VMSEQ =
7489 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7490 {FPCLASS, TDCMaskV, DAG.getCondCode(ISD::SETEQ),
7491 DAG.getUNDEF(ContainerVT), Mask, VL});
7492 return convertFromScalableVector(VT, VMSEQ, DAG, Subtarget);
7493 }
7494 SDValue AND = DAG.getNode(RISCVISD::AND_VL, DL, ContainerDstVT, FPCLASS,
7495 TDCMaskV, DAG.getUNDEF(ContainerDstVT), Mask, VL);
7496
7497 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
7498 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerDstVT,
7499 DAG.getUNDEF(ContainerDstVT), SplatZero, VL);
7500
7501 SDValue VMSNE = DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
7502 {AND, SplatZero, DAG.getCondCode(ISD::SETNE),
7503 DAG.getUNDEF(ContainerVT), Mask, VL});
7504 return convertFromScalableVector(VT, VMSNE, DAG, Subtarget);
7505 }
7506
7507 SDValue FCLASS = DAG.getNode(RISCVISD::FCLASS, DL, XLenVT, Op.getOperand(0));
7508 SDValue AND = DAG.getNode(ISD::AND, DL, XLenVT, FCLASS, TDCMaskV);
7509 SDValue Res = DAG.getSetCC(DL, XLenVT, AND, DAG.getConstant(0, DL, XLenVT),
7510 ISD::CondCode::SETNE);
7511 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
7512}
7513
7514// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7515// operations propagate nans.
7516static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7517 const RISCVSubtarget &Subtarget) {
7518 SDLoc DL(Op);
7519 MVT VT = Op.getSimpleValueType();
7520
7521 SDValue X = Op.getOperand(0);
7522 SDValue Y = Op.getOperand(1);
7523
7524 if (!VT.isVector()) {
7525 MVT XLenVT = Subtarget.getXLenVT();
7526
7527 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7528 // ensures that when one input is a nan, the other will also be a nan
7529 // allowing the nan to propagate. If both inputs are nan, this will swap the
7530 // inputs which is harmless.
7531
7532 SDValue NewY = Y;
7533 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(X)) {
7534 SDValue XIsNonNan = DAG.getSetCC(DL, XLenVT, X, X, ISD::SETOEQ);
7535 NewY = DAG.getSelect(DL, VT, XIsNonNan, Y, X);
7536 }
7537
7538 SDValue NewX = X;
7539 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Y)) {
7540 SDValue YIsNonNan = DAG.getSetCC(DL, XLenVT, Y, Y, ISD::SETOEQ);
7541 NewX = DAG.getSelect(DL, VT, YIsNonNan, X, Y);
7542 }
7543
7544 unsigned Opc =
7545 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7546 return DAG.getNode(Opc, DL, VT, NewX, NewY);
7547 }
7548
7549 // Check no NaNs before converting to fixed vector scalable.
7550 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(X);
7551 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Y);
7552
7553 MVT ContainerVT = VT;
7554 if (VT.isFixedLengthVector()) {
7555 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
7556 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
7557 Y = convertToScalableVector(ContainerVT, Y, DAG, Subtarget);
7558 }
7559
7560 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
7561
7562 SDValue NewY = Y;
7563 if (!XIsNeverNan) {
7564 SDValue XIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7565 {X, X, DAG.getCondCode(ISD::SETOEQ),
7566 DAG.getUNDEF(ContainerVT), Mask, VL});
7567 NewY = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, XIsNonNan, Y, X,
7568 DAG.getUNDEF(ContainerVT), VL);
7569 }
7570
7571 SDValue NewX = X;
7572 if (!YIsNeverNan) {
7573 SDValue YIsNonNan = DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
7574 {Y, Y, DAG.getCondCode(ISD::SETOEQ),
7575 DAG.getUNDEF(ContainerVT), Mask, VL});
7576 NewX = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, YIsNonNan, X, Y,
7577 DAG.getUNDEF(ContainerVT), VL);
7578 }
7579
7580 unsigned Opc =
7581 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
7582 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, NewX, NewY,
7583 DAG.getUNDEF(ContainerVT), Mask, VL);
7584 if (VT.isFixedLengthVector())
7585 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
7586 return Res;
7587}
7588
7589static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7590 const RISCVSubtarget &Subtarget) {
7591 bool IsFABS = Op.getOpcode() == ISD::FABS;
7592 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7593 "Wrong opcode for lowering FABS or FNEG.");
7594
7595 MVT XLenVT = Subtarget.getXLenVT();
7596 MVT VT = Op.getSimpleValueType();
7597 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7598
7599 SDLoc DL(Op);
7600 SDValue Fmv =
7601 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op.getOperand(0));
7602
7603 APInt Mask = IsFABS ? APInt::getSignedMaxValue(16) : APInt::getSignMask(16);
7604 Mask = Mask.sext(Subtarget.getXLen());
7605
7606 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7607 SDValue Logic =
7608 DAG.getNode(LogicOpc, DL, XLenVT, Fmv, DAG.getConstant(Mask, DL, XLenVT));
7609 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, Logic);
7610}
7611
7612static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7613 const RISCVSubtarget &Subtarget) {
7614 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7615
7616 MVT XLenVT = Subtarget.getXLenVT();
7617 MVT VT = Op.getSimpleValueType();
7618 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7619
7620 SDValue Mag = Op.getOperand(0);
7621 SDValue Sign = Op.getOperand(1);
7622
7623 SDLoc DL(Op);
7624
7625 // Get sign bit into an integer value.
7626 unsigned SignSize = Sign.getValueSizeInBits();
7627 SDValue SignAsInt = [&]() {
7628 if (SignSize == Subtarget.getXLen())
7629 return DAG.getNode(ISD::BITCAST, DL, XLenVT, Sign);
7630 switch (SignSize) {
7631 case 16:
7632 return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Sign);
7633 case 32:
7634 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, Sign);
7635 case 64: {
7636 assert(XLenVT == MVT::i32 && "Unexpected type");
7637 // Copy the upper word to integer.
7638 SignSize = 32;
7639 return DAG.getNode(RISCVISD::SplitF64, DL, {MVT::i32, MVT::i32}, Sign)
7640 .getValue(1);
7641 }
7642 default:
7643 llvm_unreachable("Unexpected sign size");
7644 }
7645 }();
7646
7647 // Get the signbit at the right position for MagAsInt.
7648 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
7649 SignAsInt = DAG.getNode(ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, XLenVT,
7650 SignAsInt,
7651 DAG.getConstant(std::abs(ShiftAmount), DL, XLenVT));
7652
7653 // Mask the sign bit and any bits above it. The extra bits will be dropped
7654 // when we convert back to FP.
7655 SDValue SignMask = DAG.getConstant(
7656 APInt::getSignMask(16).sext(Subtarget.getXLen()), DL, XLenVT);
7657 SDValue SignBit = DAG.getNode(ISD::AND, DL, XLenVT, SignAsInt, SignMask);
7658
7659 // Transform Mag value to integer, and clear the sign bit.
7660 SDValue MagAsInt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Mag);
7661 SDValue ClearSignMask = DAG.getConstant(
7662 APInt::getSignedMaxValue(16).sext(Subtarget.getXLen()), DL, XLenVT);
7663 SDValue ClearedSign =
7664 DAG.getNode(ISD::AND, DL, XLenVT, MagAsInt, ClearSignMask);
7665
7666 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, XLenVT, ClearedSign, SignBit,
7667 SDNodeFlags::Disjoint);
7668
7669 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, CopiedSign);
7670}
7671
7672/// Get a RISC-V target specified VL op for a given SDNode.
7673static unsigned getRISCVVLOp(SDValue Op) {
7674#define OP_CASE(NODE) \
7675 case ISD::NODE: \
7676 return RISCVISD::NODE##_VL;
7677#define VP_CASE(NODE) \
7678 case ISD::VP_##NODE: \
7679 return RISCVISD::NODE##_VL;
7680 // clang-format off
7681 switch (Op.getOpcode()) {
7682 default:
7683 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
7684 OP_CASE(ADD)
7685 OP_CASE(SUB)
7686 OP_CASE(MUL)
7687 OP_CASE(MULHS)
7688 OP_CASE(MULHU)
7689 OP_CASE(SDIV)
7690 OP_CASE(SREM)
7691 OP_CASE(UDIV)
7692 OP_CASE(UREM)
7693 OP_CASE(SHL)
7694 OP_CASE(SRA)
7695 OP_CASE(SRL)
7696 OP_CASE(ROTL)
7697 OP_CASE(ROTR)
7698 OP_CASE(BSWAP)
7699 OP_CASE(CTTZ)
7700 OP_CASE(CTLZ)
7701 OP_CASE(CTPOP)
7702 OP_CASE(BITREVERSE)
7703 OP_CASE(SADDSAT)
7704 OP_CASE(UADDSAT)
7705 OP_CASE(SSUBSAT)
7706 OP_CASE(USUBSAT)
7707 OP_CASE(AVGFLOORS)
7708 OP_CASE(AVGFLOORU)
7709 OP_CASE(AVGCEILS)
7710 OP_CASE(AVGCEILU)
7711 OP_CASE(FADD)
7712 OP_CASE(FSUB)
7713 OP_CASE(FMUL)
7714 OP_CASE(FDIV)
7715 OP_CASE(FNEG)
7716 OP_CASE(FABS)
7717 OP_CASE(FCOPYSIGN)
7718 OP_CASE(FSQRT)
7719 OP_CASE(SMIN)
7720 OP_CASE(SMAX)
7721 OP_CASE(UMIN)
7722 OP_CASE(UMAX)
7723 OP_CASE(ABDS)
7724 OP_CASE(ABDU)
7725 OP_CASE(STRICT_FADD)
7726 OP_CASE(STRICT_FSUB)
7727 OP_CASE(STRICT_FMUL)
7728 OP_CASE(STRICT_FDIV)
7729 OP_CASE(STRICT_FSQRT)
7730 VP_CASE(SDIV) // VP_SDIV
7731 VP_CASE(SREM) // VP_SREM
7732 VP_CASE(UDIV) // VP_UDIV
7733 VP_CASE(UREM) // VP_UREM
7734 case ISD::CTLZ_ZERO_POISON:
7735 return RISCVISD::CTLZ_VL;
7736 case ISD::CTTZ_ZERO_POISON:
7737 return RISCVISD::CTTZ_VL;
7738 case ISD::FMA:
7739 return RISCVISD::VFMADD_VL;
7740 case ISD::STRICT_FMA:
7741 return RISCVISD::STRICT_VFMADD_VL;
7742 case ISD::AND:
7743 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7744 return RISCVISD::VMAND_VL;
7745 return RISCVISD::AND_VL;
7746 case ISD::OR:
7747 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7748 return RISCVISD::VMOR_VL;
7749 return RISCVISD::OR_VL;
7750 case ISD::XOR:
7751 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7752 return RISCVISD::VMXOR_VL;
7753 return RISCVISD::XOR_VL;
7754 case ISD::ANY_EXTEND:
7755 case ISD::ZERO_EXTEND:
7756 return RISCVISD::VZEXT_VL;
7757 case ISD::SIGN_EXTEND:
7758 return RISCVISD::VSEXT_VL;
7759 case ISD::SETCC:
7760 return RISCVISD::SETCC_VL;
7761 case ISD::VSELECT:
7762 return RISCVISD::VMERGE_VL;
7763 case ISD::VP_MERGE:
7764 return RISCVISD::VMERGE_VL;
7765 case ISD::FMINNUM:
7766 case ISD::FMINIMUMNUM:
7767 return RISCVISD::VFMIN_VL;
7768 case ISD::FMAXNUM:
7769 case ISD::FMAXIMUMNUM:
7770 return RISCVISD::VFMAX_VL;
7771 case ISD::LRINT:
7772 case ISD::LLRINT:
7773 return RISCVISD::VFCVT_RM_X_F_VL;
7774 case ISD::MASKED_UDIV:
7775 return RISCVISD::UDIV_VL;
7776 case ISD::MASKED_UREM:
7777 return RISCVISD::UREM_VL;
7778 case ISD::MASKED_SDIV:
7779 return RISCVISD::SDIV_VL;
7780 case ISD::MASKED_SREM:
7781 return RISCVISD::SREM_VL;
7782 }
7783 // clang-format on
7784#undef OP_CASE
7785#undef VP_CASE
7786}
7787
7788static bool isPromotedOpNeedingSplit(SDValue Op,
7789 const RISCVSubtarget &Subtarget,
7790 const TargetLowering &TLI) {
7791 MVT OpVT = Op.getSimpleValueType();
7792 if (!OpVT.isVector())
7793 return false;
7794 MVT EltVT = OpVT.getVectorElementType();
7795 if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
7796 !Subtarget.hasVInstructionsF16()) &&
7797 !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
7798 (!Subtarget.hasVInstructionsBF16() ||
7799 !llvm::is_contained(ZvfbfaOps, Op.getOpcode()))))
7800 return false;
7801 // Need to split when the same width f32 vector type isn't legal.
7802 return !TLI.isTypeLegal(
7803 MVT::getVectorVT(MVT::f32, OpVT.getVectorElementCount()));
7804}
7805
7806static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
7807 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op.getValueType());
7808 SDLoc DL(Op);
7809
7810 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
7811 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
7812
7813 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
7814 if (!Op.getOperand(j).getValueType().isVector()) {
7815 LoOperands[j] = Op.getOperand(j);
7816 HiOperands[j] = Op.getOperand(j);
7817 continue;
7818 }
7819 std::tie(LoOperands[j], HiOperands[j]) =
7820 DAG.SplitVector(Op.getOperand(j), DL);
7821 }
7822
7823 SDValue LoRes =
7824 DAG.getNode(Op.getOpcode(), DL, LoVT, LoOperands, Op->getFlags());
7825 SDValue HiRes =
7826 DAG.getNode(Op.getOpcode(), DL, HiVT, HiOperands, Op->getFlags());
7827
7828 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Op.getValueType(), LoRes, HiRes);
7829}
7830
7831static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
7832 bool IsVP) {
7833 SDLoc DL(Op);
7834
7835 if (IsVP) {
7836 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(1), DL);
7837 auto [MaskLo, MaskHi] = DAG.SplitVector(Op.getOperand(2), DL);
7838 auto [EVLLo, EVLHi] =
7839 DAG.SplitEVL(Op.getOperand(3), Op.getOperand(1).getValueType(), DL);
7840
7841 SDValue ResLo =
7842 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
7843 {Op.getOperand(0), Lo, MaskLo, EVLLo}, Op->getFlags());
7844 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
7845 {ResLo, Hi, MaskHi, EVLHi}, Op->getFlags());
7846 }
7847
7848 unsigned Opcode = Op.getOpcode();
7849 unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
7850
7851 auto [Lo, Hi] = DAG.SplitVector(Op.getOperand(OpNo), DL);
7852 if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
7853 SDValue ResLo = DAG.getNode(Op.getOpcode(), DL, Op.getValueType(),
7854 Op.getOperand(0), Lo, Op->getFlags());
7855 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), ResLo, Hi,
7856 Op->getFlags());
7857 }
7858
7859 SDValue ResLo =
7860 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Lo, Op->getFlags());
7861 SDValue ResHi =
7862 DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Hi, Op->getFlags());
7863 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
7864 return DAG.getNode(BaseOpc, DL, Op.getValueType(), ResLo, ResHi,
7865 Op->getFlags());
7866}
7867
7868static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
7869
7870 assert(Op->isStrictFPOpcode());
7871
7872 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(Op->getValueType(0));
7873
7874 SDVTList LoVTs = DAG.getVTList(LoVT, Op->getValueType(1));
7875 SDVTList HiVTs = DAG.getVTList(HiVT, Op->getValueType(1));
7876
7877 SDLoc DL(Op);
7878
7879 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
7880 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
7881
7882 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
7883 if (!Op.getOperand(j).getValueType().isVector()) {
7884 LoOperands[j] = Op.getOperand(j);
7885 HiOperands[j] = Op.getOperand(j);
7886 continue;
7887 }
7888 std::tie(LoOperands[j], HiOperands[j]) =
7889 DAG.SplitVector(Op.getOperand(j), DL);
7890 }
7891
7892 SDValue LoRes =
7893 DAG.getNode(Op.getOpcode(), DL, LoVTs, LoOperands, Op->getFlags());
7894 HiOperands[0] = LoRes.getValue(1);
7895 SDValue HiRes =
7896 DAG.getNode(Op.getOpcode(), DL, HiVTs, HiOperands, Op->getFlags());
7897
7898 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, Op->getValueType(0),
7899 LoRes.getValue(0), HiRes.getValue(0));
7900 return DAG.getMergeValues({V, HiRes.getValue(1)}, DL);
7901}
7902
7903SDValue
7904RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
7905 SelectionDAG &DAG) const {
7906 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
7907 "Unexpected bfloat16 load lowering");
7908
7909 SDLoc DL(Op);
7910 LoadSDNode *LD = cast<LoadSDNode>(Op.getNode());
7911 EVT MemVT = LD->getMemoryVT();
7912 SDValue Load = DAG.getExtLoad(
7913 ISD::ZEXTLOAD, DL, Subtarget.getXLenVT(), LD->getChain(),
7914 LD->getBasePtr(),
7915 EVT::getIntegerVT(*DAG.getContext(), MemVT.getSizeInBits()),
7916 LD->getMemOperand());
7917 // Using mask to make bf16 nan-boxing valid when we don't have flh
7918 // instruction. -65536 would be treat as a small number and thus it can be
7919 // directly used lui to get the constant.
7920 SDValue mask = DAG.getSignedConstant(-65536, DL, Subtarget.getXLenVT());
7921 SDValue OrSixteenOne =
7922 DAG.getNode(ISD::OR, DL, Load.getValueType(), {Load, mask});
7923 SDValue ConvertedResult =
7924 DAG.getNode(RISCVISD::NDS_FMV_BF16_X, DL, MVT::bf16, OrSixteenOne);
7925 return DAG.getMergeValues({ConvertedResult, Load.getValue(1)}, DL);
7926}
7927
7928SDValue
7929RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
7930 SelectionDAG &DAG) const {
7931 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
7932 "Unexpected bfloat16 store lowering");
7933
7934 StoreSDNode *ST = cast<StoreSDNode>(Op.getNode());
7935 SDLoc DL(Op);
7936 SDValue FMV = DAG.getNode(RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
7937 Subtarget.getXLenVT(), ST->getValue());
7938 return DAG.getTruncStore(
7939 ST->getChain(), DL, FMV, ST->getBasePtr(),
7940 EVT::getIntegerVT(*DAG.getContext(), ST->getMemoryVT().getSizeInBits()),
7941 ST->getMemOperand());
7942}
7943
7944static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
7945 const RISCVSubtarget &Subtarget);
7946
7947SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
7948 SelectionDAG &DAG) const {
7949 switch (Op.getOpcode()) {
7950 default:
7951 reportFatalInternalError(
7952 "Unimplemented RISCVTargetLowering::LowerOperation Case");
7953 case ISD::PREFETCH:
7954 return LowerPREFETCH(Op, Subtarget, DAG);
7955 case ISD::ATOMIC_FENCE:
7956 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
7957 case ISD::GlobalAddress:
7958 return lowerGlobalAddress(Op, DAG);
7959 case ISD::BlockAddress:
7960 return lowerBlockAddress(Op, DAG);
7961 case ISD::ConstantPool:
7962 return lowerConstantPool(Op, DAG);
7963 case ISD::JumpTable:
7964 return lowerJumpTable(Op, DAG);
7965 case ISD::GlobalTLSAddress:
7966 return lowerGlobalTLSAddress(Op, DAG);
7967 case ISD::Constant:
7968 return lowerConstant(Op, DAG, Subtarget);
7969 case ISD::ConstantFP:
7970 return lowerConstantFP(Op, DAG);
7971 case ISD::SELECT:
7972 return lowerSELECT(Op, DAG);
7973 case ISD::BRCOND:
7974 return lowerBRCOND(Op, DAG);
7975 case ISD::VASTART:
7976 return lowerVASTART(Op, DAG);
7977 case ISD::FRAMEADDR:
7978 return lowerFRAMEADDR(Op, DAG);
7979 case ISD::RETURNADDR:
7980 return lowerRETURNADDR(Op, DAG);
7981 case ISD::SHL_PARTS:
7982 return lowerShiftLeftParts(Op, DAG);
7983 case ISD::SRA_PARTS:
7984 return lowerShiftRightParts(Op, DAG, true);
7985 case ISD::SRL_PARTS:
7986 return lowerShiftRightParts(Op, DAG, false);
7987 case ISD::ROTL:
7988 case ISD::ROTR:
7989 if (Op.getValueType().isFixedLengthVector()) {
7990 assert(Subtarget.hasStdExtZvkb());
7991 return lowerToScalableOp(Op, DAG);
7992 }
7993 assert(Subtarget.hasVendorXTHeadBb() &&
7994 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
7995 "Unexpected custom legalization");
7996 // XTHeadBb only supports rotate by constant.
7997 if (!isa<ConstantSDNode>(Op.getOperand(1)))
7998 return SDValue();
7999 return Op;
8000 case ISD::BITCAST: {
8001 SDLoc DL(Op);
8002 EVT VT = Op.getValueType();
8003 SDValue Op0 = Op.getOperand(0);
8004 EVT Op0VT = Op0.getValueType();
8005 MVT XLenVT = Subtarget.getXLenVT();
8006 if (Op0VT == MVT::i16 &&
8007 ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
8008 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
8009 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
8010 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, NewOp0);
8011 }
8012 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
8013 Subtarget.hasStdExtFOrZfinx()) {
8014 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
8015 return DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
8016 }
8017 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
8018 Subtarget.hasStdExtDOrZdinx()) {
8019 SDValue Lo, Hi;
8020 std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
8021 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
8022 }
8023
8024 if (Subtarget.hasStdExtP() && VT.isSimple() && Op0VT.isSimple()) {
8025 if (VT.getSimpleVT() == Subtarget.getXLenVT() &&
8026 Subtarget.isPExtPackedType(Op0VT.getSimpleVT()))
8027 return Op;
8028 if (Op0VT.getSimpleVT() == Subtarget.getXLenVT() &&
8029 Subtarget.isPExtPackedType(VT.getSimpleVT()))
8030 return Op;
8031 }
8032
8033 // Consider other scalar<->scalar casts as legal if the types are legal.
8034 // Otherwise expand them.
8035 if (!VT.isVector() && !Op0VT.isVector()) {
8036 if (isTypeLegal(VT) && isTypeLegal(Op0VT))
8037 return Op;
8038 return SDValue();
8039 }
8040
8041 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
8042 "Unexpected types");
8043
8044 if (VT.isFixedLengthVector()) {
8045 // We can handle fixed length vector bitcasts with a simple replacement
8046 // in isel.
8047 if (Op0VT.isFixedLengthVector())
8048 return Op;
8049 // When bitcasting from scalar to fixed-length vector, insert the scalar
8050 // into a one-element vector of the result type, and perform a vector
8051 // bitcast.
8052 if (!Op0VT.isVector()) {
8053 EVT BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
8054 if (!isTypeLegal(BVT))
8055 return SDValue();
8056 return DAG.getBitcast(
8057 VT, DAG.getInsertVectorElt(DL, DAG.getUNDEF(BVT), Op0, 0));
8058 }
8059 return SDValue();
8060 }
8061 // Custom-legalize bitcasts from fixed-length vector types to scalar types
8062 // thus: bitcast the vector to a one-element vector type whose element type
8063 // is the same as the result type, and extract the first element.
8064 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
8065 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
8066 if (!isTypeLegal(BVT))
8067 return SDValue();
8068 SDValue BVec = DAG.getBitcast(BVT, Op0);
8069 return DAG.getExtractVectorElt(DL, VT, BVec, 0);
8070 }
8071 return SDValue();
8072 }
8073 case ISD::INTRINSIC_WO_CHAIN:
8074 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
8075 case ISD::INTRINSIC_W_CHAIN:
8076 return LowerINTRINSIC_W_CHAIN(Op, DAG);
8077 case ISD::INTRINSIC_VOID:
8078 return LowerINTRINSIC_VOID(Op, DAG);
8079 case ISD::IS_FPCLASS:
8080 return LowerIS_FPCLASS(Op, DAG);
8081 case ISD::BITREVERSE: {
8082 MVT VT = Op.getSimpleValueType();
8083 if (VT.isFixedLengthVector()) {
8084 assert(Subtarget.hasStdExtZvbb());
8085 return lowerToScalableOp(Op, DAG);
8086 }
8087 SDLoc DL(Op);
8088 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
8089 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
8090 // Expand bitreverse to a bswap(rev8) followed by brev8.
8091 SDValue BSwap = DAG.getNode(ISD::BSWAP, DL, VT, Op.getOperand(0));
8092 return DAG.getNode(RISCVISD::BREV8, DL, VT, BSwap);
8093 }
8094 case ISD::TRUNCATE:
8095 case ISD::TRUNCATE_SSAT_S:
8096 case ISD::TRUNCATE_USAT_U:
8097 // Only custom-lower vector truncates
8098 if (!Op.getSimpleValueType().isVector())
8099 return Op;
8100 return lowerVectorTrunc(Op, DAG);
8101 case ISD::ANY_EXTEND:
8102 case ISD::ZERO_EXTEND:
8103 if (Op.getOperand(0).getValueType().isVector() &&
8104 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
8105 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
8106 if (Op.getValueType().isScalableVector())
8107 return Op;
8108 return lowerToScalableOp(Op, DAG);
8109 case ISD::SIGN_EXTEND:
8110 if (Op.getOperand(0).getValueType().isVector() &&
8111 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
8112 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
8113 if (Op.getValueType().isScalableVector())
8114 return Op;
8115 return lowerToScalableOp(Op, DAG);
8116 case ISD::SPLAT_VECTOR_PARTS:
8117 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
8118 case ISD::INSERT_VECTOR_ELT:
8119 return lowerINSERT_VECTOR_ELT(Op, DAG);
8120 case ISD::EXTRACT_VECTOR_ELT:
8121 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
8122 case ISD::SCALAR_TO_VECTOR: {
8123 MVT VT = Op.getSimpleValueType();
8124 SDLoc DL(Op);
8125 SDValue Scalar = Op.getOperand(0);
8126 if (VT.getVectorElementType() == MVT::i1) {
8127 MVT WideVT = VT.changeVectorElementType(MVT::i8);
8128 SDValue V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, WideVT, Scalar);
8129 return DAG.getNode(ISD::TRUNCATE, DL, VT, V);
8130 }
8131 MVT ContainerVT = VT;
8132 if (VT.isFixedLengthVector())
8133 ContainerVT = getContainerForFixedLengthVector(VT);
8134 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
8135
8136 SDValue V;
8137 if (VT.isFloatingPoint()) {
8138 V = DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, ContainerVT,
8139 DAG.getUNDEF(ContainerVT), Scalar, VL);
8140 } else {
8141 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getXLenVT(), Scalar);
8142 V = DAG.getNode(RISCVISD::VMV_S_X_VL, DL, ContainerVT,
8143 DAG.getUNDEF(ContainerVT), Scalar, VL);
8144 }
8145 if (VT.isFixedLengthVector())
8146 V = convertFromScalableVector(VT, V, DAG, Subtarget);
8147 return V;
8148 }
8149 case ISD::VSCALE: {
8150 MVT XLenVT = Subtarget.getXLenVT();
8151 MVT VT = Op.getSimpleValueType();
8152 SDLoc DL(Op);
8153 SDValue Res = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
8154 // We define our scalable vector types for lmul=1 to use a 64 bit known
8155 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
8156 // vscale as VLENB / 8.
8157 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
8158 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
8159 reportFatalInternalError("Support for VLEN==32 is incomplete.");
8160 // We assume VLENB is a multiple of 8. We manually choose the best shift
8161 // here because SimplifyDemandedBits isn't always able to simplify it.
8162 uint64_t Val = Op.getConstantOperandVal(0);
8163 if (isPowerOf2_64(Val)) {
8164 uint64_t Log2 = Log2_64(Val);
8165 if (Log2 < 3) {
8166 SDNodeFlags Flags;
8167 Flags.setExact(true);
8168 Res = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
8169 DAG.getConstant(3 - Log2, DL, XLenVT), Flags);
8170 } else if (Log2 > 3) {
8171 Res = DAG.getNode(ISD::SHL, DL, XLenVT, Res,
8172 DAG.getConstant(Log2 - 3, DL, XLenVT));
8173 }
8174 } else if ((Val % 8) == 0) {
8175 // If the multiplier is a multiple of 8, scale it down to avoid needing
8176 // to shift the VLENB value.
8177 Res = DAG.getNode(ISD::MUL, DL, XLenVT, Res,
8178 DAG.getConstant(Val / 8, DL, XLenVT));
8179 } else {
8180 SDNodeFlags Flags;
8181 Flags.setExact(true);
8182 SDValue VScale = DAG.getNode(ISD::SRL, DL, XLenVT, Res,
8183 DAG.getConstant(3, DL, XLenVT), Flags);
8184 Res = DAG.getNode(ISD::MUL, DL, XLenVT, VScale,
8185 DAG.getConstant(Val, DL, XLenVT));
8186 }
8187 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
8188 }
8189 case ISD::FPOWI: {
8190 // Custom promote f16 powi with illegal i32 integer type on RV64. Once
8191 // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
8192 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
8193 Op.getOperand(1).getValueType() == MVT::i32) {
8194 SDLoc DL(Op);
8195 SDValue Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
8196 SDValue Powi =
8197 DAG.getNode(ISD::FPOWI, DL, MVT::f32, Op0, Op.getOperand(1));
8198 return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Powi,
8199 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
8200 }
8201 return SDValue();
8202 }
8203 case ISD::FMAXIMUM:
8204 case ISD::FMINIMUM:
8205 if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
8206 return SplitVectorOp(Op, DAG);
8207 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
8208 case ISD::FP_EXTEND:
8209 case ISD::FP_ROUND:
8210 return lowerVectorFPExtendOrRound(Op, DAG);
8211 case ISD::STRICT_FP_ROUND:
8212 case ISD::STRICT_FP_EXTEND:
8213 return lowerStrictFPExtendOrRoundLike(Op, DAG);
8214 case ISD::SINT_TO_FP:
8215 case ISD::UINT_TO_FP:
8216 // Fall back to zvfbfmin for bf16 case if source type is wider than 8 bits.
8217 if (SDValue Op1 = Op.getOperand(0);
8218 Op.getValueType().isVector() &&
8219 ((Op.getValueType().getScalarType() == MVT::f16 &&
8220 (Subtarget.hasVInstructionsF16Minimal() &&
8221 !Subtarget.hasVInstructionsF16())) ||
8222 (Op.getValueType().getScalarType() == MVT::bf16 &&
8223 (Subtarget.hasVInstructionsBF16Minimal() &&
8224 (!Subtarget.hasVInstructionsBF16() ||
8225 Op1.getValueType().getScalarSizeInBits() > 8))))) {
8226 MVT NVT =
8227 MVT::getVectorVT(MVT::f32, Op.getValueType().getVectorElementCount());
8228 if (!isTypeLegal(NVT))
8229 return SplitVectorOp(Op, DAG);
8230 // int -> f32
8231 SDLoc DL(Op);
8232 SDValue NC = DAG.getNode(Op.getOpcode(), DL, NVT, Op->ops());
8233 // f32 -> [b]f16
8234 return DAG.getNode(ISD::FP_ROUND, DL, Op.getValueType(), NC,
8235 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true));
8236 }
8237 [[fallthrough]];
8238 case ISD::FP_TO_SINT:
8239 case ISD::FP_TO_UINT:
8240 // Fall back to zvfbfmin for bf16 case if destination type is wider than 8
8241 // bits.
8242 if (SDValue Op1 = Op.getOperand(0);
8243 Op1.getValueType().isVector() &&
8244 ((Op1.getValueType().getScalarType() == MVT::f16 &&
8245 (Subtarget.hasVInstructionsF16Minimal() &&
8246 !Subtarget.hasVInstructionsF16())) ||
8247 (Op1.getValueType().getScalarType() == MVT::bf16 &&
8248 (Subtarget.hasVInstructionsBF16Minimal() &&
8249 (!Subtarget.hasVInstructionsBF16() ||
8250 Op.getValueType().getScalarSizeInBits() > 8))))) {
8251 MVT NVT = MVT::getVectorVT(MVT::f32,
8252 Op1.getValueType().getVectorElementCount());
8253 if (!isTypeLegal(NVT))
8254 return SplitVectorOp(Op, DAG);
8255 // [b]f16 -> f32
8256 SDLoc DL(Op);
8257 SDValue WidenVec = DAG.getNode(ISD::FP_EXTEND, DL, NVT, Op1);
8258 // f32 -> int
8259 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), WidenVec);
8260 }
8261 [[fallthrough]];
8262 case ISD::STRICT_FP_TO_SINT:
8263 case ISD::STRICT_FP_TO_UINT:
8264 case ISD::STRICT_SINT_TO_FP:
8265 case ISD::STRICT_UINT_TO_FP: {
8266 // RVV can only do fp<->int conversions to types half/double the size as
8267 // the source. We custom-lower any conversions that do two hops into
8268 // sequences.
8269 MVT VT = Op.getSimpleValueType();
8270 if (VT.isScalarInteger())
8271 return lowerFP_TO_INT(Op, DAG, Subtarget);
8272 bool IsStrict = Op->isStrictFPOpcode();
8273 SDValue Src = Op.getOperand(0 + IsStrict);
8274 MVT SrcVT = Src.getSimpleValueType();
8275 if (SrcVT.isScalarInteger())
8276 return lowerINT_TO_FP(Op, DAG, Subtarget);
8277 if (!VT.isVector())
8278 return Op;
8279 SDLoc DL(Op);
8280 MVT EltVT = VT.getVectorElementType();
8281 MVT SrcEltVT = SrcVT.getVectorElementType();
8282 unsigned EltSize = EltVT.getSizeInBits();
8283 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
8284 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
8285 "Unexpected vector element types");
8286
8287 bool IsInt2FP = SrcEltVT.isInteger();
8288 // Widening conversions
8289 if (EltSize > (2 * SrcEltSize)) {
8290 if (IsInt2FP) {
8291 // Do a regular integer sign/zero extension then convert to float.
8292 MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltSize / 2),
8293 VT.getVectorElementCount());
8294 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
8295 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
8296 ? ISD::ZERO_EXTEND
8297 : ISD::SIGN_EXTEND;
8298 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
8299 if (IsStrict)
8300 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(),
8301 Op.getOperand(0), Ext);
8302 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
8303 }
8304 // FP2Int
8305 assert((SrcEltVT == MVT::f16 || SrcEltVT == MVT::bf16) &&
8306 "Unexpected FP_TO_[US]INT lowering");
8307 // Do one doubling fp_extend then complete the operation by converting
8308 // to int.
8309 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
8310 if (IsStrict) {
8311 auto [FExt, Chain] =
8312 DAG.getStrictFPExtendOrRound(Src, Op.getOperand(0), DL, InterimFVT);
8313 return DAG.getNode(Op.getOpcode(), DL, Op->getVTList(), Chain, FExt);
8314 }
8315 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
8316 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
8317 }
8318
8319 // Narrowing conversions
8320 if (SrcEltSize > (2 * EltSize)) {
8321 if (IsInt2FP) {
8322 // One narrowing int_to_fp, then an fp_round.
8323 assert((EltVT == MVT::f16 || EltVT == MVT::bf16) &&
8324 "Unexpected [US]_TO_FP lowering");
8325 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
8326 if (IsStrict) {
8327 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL,
8328 DAG.getVTList(InterimFVT, MVT::Other),
8329 Op.getOperand(0), Src);
8330 SDValue Chain = Int2FP.getValue(1);
8331 return DAG.getStrictFPExtendOrRound(Int2FP, Chain, DL, VT).first;
8332 }
8333 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
8334 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
8335 }
8336 // FP2Int
8337 // One narrowing fp_to_int, then truncate the integer. If the float isn't
8338 // representable by the integer, the result is poison.
8339 MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize / 2),
8340 VT.getVectorElementCount());
8341 if (IsStrict) {
8342 SDValue FP2Int =
8343 DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(IVecVT, MVT::Other),
8344 Op.getOperand(0), Src);
8345 SDValue Res = DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
8346 return DAG.getMergeValues({Res, FP2Int.getValue(1)}, DL);
8347 }
8348 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
8349 if (EltSize == 1)
8350 // The integer should be 0 or 1/-1, so compare the integer result to 0.
8351 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, IVecVT), FP2Int,
8352 ISD::SETNE);
8353 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
8354 }
8355
8356 // Scalable vectors can exit here. Patterns will handle equally-sized
8357 // conversions halving/doubling ones.
8358 if (!VT.isFixedLengthVector())
8359 return Op;
8360
8361 // For fixed-length vectors we lower to a custom "VL" node.
8362 unsigned RVVOpc = 0;
8363 switch (Op.getOpcode()) {
8364 default:
8365 llvm_unreachable("Impossible opcode");
8366 case ISD::FP_TO_SINT:
8367 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
8368 break;
8369 case ISD::FP_TO_UINT:
8370 RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
8371 break;
8372 case ISD::SINT_TO_FP:
8373 RVVOpc = RISCVISD::SINT_TO_FP_VL;
8374 break;
8375 case ISD::UINT_TO_FP:
8376 RVVOpc = RISCVISD::UINT_TO_FP_VL;
8377 break;
8378 case ISD::STRICT_FP_TO_SINT:
8379 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
8380 break;
8381 case ISD::STRICT_FP_TO_UINT:
8382 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
8383 break;
8384 case ISD::STRICT_SINT_TO_FP:
8385 RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
8386 break;
8387 case ISD::STRICT_UINT_TO_FP:
8388 RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
8389 break;
8390 }
8391
8392 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8393 MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
8394 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
8395 "Expected same element count");
8396
8397 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
8398
8399 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
8400 if (IsStrict) {
8401 Src = DAG.getNode(RVVOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
8402 Op.getOperand(0), Src, Mask, VL);
8403 SDValue SubVec = convertFromScalableVector(VT, Src, DAG, Subtarget);
8404 return DAG.getMergeValues({SubVec, Src.getValue(1)}, DL);
8405 }
8406 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
8407 return convertFromScalableVector(VT, Src, DAG, Subtarget);
8408 }
8409 case ISD::FP_TO_SINT_SAT:
8410 case ISD::FP_TO_UINT_SAT:
8411 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
8412 case ISD::FP_TO_BF16: {
8413 // Custom lower to ensure the libcall return is passed in an FPR on hard
8414 // float ABIs.
8415 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization");
8416 SDLoc DL(Op);
8417 MakeLibCallOptions CallOptions;
8418 RTLIB::Libcall LC =
8419 RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);
8420 SDValue Res =
8421 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
8422 if (Subtarget.is64Bit())
8423 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
8424 return DAG.getBitcast(MVT::i32, Res);
8425 }
8426 case ISD::BF16_TO_FP: {
8427 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization");
8428 MVT VT = Op.getSimpleValueType();
8429 SDLoc DL(Op);
8430 Op = DAG.getNode(
8431 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
8432 DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
8433 SDValue Res = Subtarget.is64Bit()
8434 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op)
8435 : DAG.getBitcast(MVT::f32, Op);
8436 // fp_extend if the target VT is bigger than f32.
8437 if (VT != MVT::f32)
8438 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
8439 return Res;
8440 }
8441 case ISD::STRICT_FP_TO_FP16:
8442 case ISD::FP_TO_FP16: {
8443 // Custom lower to ensure the libcall return is passed in an FPR on hard
8444 // float ABIs.
8445 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8446 SDLoc DL(Op);
8447 MakeLibCallOptions CallOptions;
8448 bool IsStrict = Op->isStrictFPOpcode();
8449 SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
8450 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
8451 RTLIB::Libcall LC = RTLIB::getFPROUND(Op0.getValueType(), MVT::f16);
8452 SDValue Res;
8453 std::tie(Res, Chain) =
8454 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions, DL, Chain);
8455 if (Subtarget.is64Bit())
8456 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Res);
8457 SDValue Result = DAG.getBitcast(MVT::i32, IsStrict ? Res.getValue(0) : Res);
8458 if (IsStrict)
8459 return DAG.getMergeValues({Result, Chain}, DL);
8460 return Result;
8461 }
8462 case ISD::STRICT_FP16_TO_FP:
8463 case ISD::FP16_TO_FP: {
8464 // Custom lower to ensure the libcall argument is passed in an FPR on hard
8465 // float ABIs.
8466 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8467 SDLoc DL(Op);
8468 MakeLibCallOptions CallOptions;
8469 bool IsStrict = Op->isStrictFPOpcode();
8470 SDValue Op0 = IsStrict ? Op.getOperand(1) : Op.getOperand(0);
8471 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
8472 SDValue Arg = Subtarget.is64Bit()
8473 ? DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Op0)
8474 : DAG.getBitcast(MVT::f32, Op0);
8475 SDValue Res;
8476 std::tie(Res, Chain) = makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
8477 CallOptions, DL, Chain);
8478 if (IsStrict)
8479 return DAG.getMergeValues({Res, Chain}, DL);
8480 return Res;
8481 }
8482 case ISD::FTRUNC:
8483 case ISD::FCEIL:
8484 case ISD::FFLOOR:
8485 case ISD::FNEARBYINT:
8486 case ISD::FRINT:
8487 case ISD::FROUND:
8488 case ISD::FROUNDEVEN:
8489 if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
8490 return SplitVectorOp(Op, DAG);
8491 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
8492 case ISD::FCANONICALIZE: {
8493 MVT VT = Op.getSimpleValueType();
8494 assert(VT.isFixedLengthVector() && "Unexpected type");
8495 SDLoc DL(Op);
8496 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8497 SDValue Src =
8498 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
8499 SDValue Res = DAG.getNode(ISD::FCANONICALIZE, DL, ContainerVT, Src);
8500 return convertFromScalableVector(VT, Res, DAG, Subtarget);
8501 }
8502 case ISD::LRINT:
8503 case ISD::LLRINT:
8504 case ISD::LROUND:
8505 case ISD::LLROUND: {
8506 if (Op.getValueType().isVector())
8507 return lowerVectorXRINT_XROUND(Op, DAG, Subtarget);
8508 assert(Op.getOperand(0).getValueType() == MVT::f16 &&
8509 "Unexpected custom legalisation");
8510 SDLoc DL(Op);
8511 SDValue Ext = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op.getOperand(0));
8512 return DAG.getNode(Op.getOpcode(), DL, Op.getValueType(), Ext);
8513 }
8514 case ISD::STRICT_LRINT:
8515 case ISD::STRICT_LLRINT:
8516 case ISD::STRICT_LROUND:
8517 case ISD::STRICT_LLROUND: {
8518 assert(Op.getOperand(1).getValueType() == MVT::f16 &&
8519 "Unexpected custom legalisation");
8520 SDLoc DL(Op);
8521 SDValue Ext = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
8522 {Op.getOperand(0), Op.getOperand(1)});
8523 return DAG.getNode(Op.getOpcode(), DL, {Op.getValueType(), MVT::Other},
8524 {Ext.getValue(1), Ext.getValue(0)});
8525 }
8526 case ISD::VECREDUCE_ADD:
8527 case ISD::VECREDUCE_UMAX:
8528 case ISD::VECREDUCE_SMAX:
8529 case ISD::VECREDUCE_UMIN:
8530 case ISD::VECREDUCE_SMIN:
8531 return lowerVECREDUCE(Op, DAG);
8532 case ISD::VECREDUCE_AND:
8533 case ISD::VECREDUCE_OR:
8534 case ISD::VECREDUCE_XOR:
8535 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
8536 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
8537 return lowerVECREDUCE(Op, DAG);
8538 case ISD::VECREDUCE_SEQ_FADD:
8539 if (isPromotedOpNeedingSplit(Op.getOperand(1), Subtarget, *this))
8540 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8541 return lowerFPVECREDUCE(Op, DAG);
8542 case ISD::VECREDUCE_FADD:
8543 case ISD::VECREDUCE_FMIN:
8544 case ISD::VECREDUCE_FMAX:
8545 case ISD::VECREDUCE_FMAXIMUM:
8546 case ISD::VECREDUCE_FMINIMUM:
8547 if (isPromotedOpNeedingSplit(Op.getOperand(0), Subtarget, *this))
8548 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8549 return lowerFPVECREDUCE(Op, DAG);
8550 case ISD::VP_REDUCE_ADD:
8551 case ISD::VP_REDUCE_UMAX:
8552 case ISD::VP_REDUCE_SMAX:
8553 case ISD::VP_REDUCE_UMIN:
8554 case ISD::VP_REDUCE_SMIN:
8555 case ISD::VP_REDUCE_FADD:
8556 case ISD::VP_REDUCE_SEQ_FADD:
8557 case ISD::VP_REDUCE_FMIN:
8558 case ISD::VP_REDUCE_FMAX:
8559 case ISD::VP_REDUCE_FMINIMUM:
8560 case ISD::VP_REDUCE_FMAXIMUM:
8561 if (isPromotedOpNeedingSplit(Op.getOperand(1), Subtarget, *this))
8562 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ true);
8563 return lowerVPREDUCE(Op, DAG);
8564 case ISD::VP_REDUCE_AND:
8565 case ISD::VP_REDUCE_OR:
8566 case ISD::VP_REDUCE_XOR:
8567 if (Op.getOperand(1).getValueType().getVectorElementType() == MVT::i1)
8568 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
8569 return lowerVPREDUCE(Op, DAG);
8570 case ISD::VP_CTTZ_ELTS:
8571 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8572 return lowerVPCttzElements(Op, DAG);
8573 case ISD::UNDEF: {
8574 MVT ContainerVT = getContainerForFixedLengthVector(Op.getSimpleValueType());
8575 return convertFromScalableVector(Op.getSimpleValueType(),
8576 DAG.getUNDEF(ContainerVT), DAG, Subtarget);
8577 }
8578 case ISD::INSERT_SUBVECTOR:
8579 return lowerINSERT_SUBVECTOR(Op, DAG);
8580 case ISD::EXTRACT_SUBVECTOR:
8581 return lowerEXTRACT_SUBVECTOR(Op, DAG);
8582 case ISD::VECTOR_DEINTERLEAVE:
8583 return lowerVECTOR_DEINTERLEAVE(Op, DAG);
8584 case ISD::VECTOR_INTERLEAVE:
8585 return lowerVECTOR_INTERLEAVE(Op, DAG);
8586 case ISD::STEP_VECTOR:
8587 return lowerSTEP_VECTOR(Op, DAG);
8588 case ISD::VECTOR_REVERSE:
8589 return lowerVECTOR_REVERSE(Op, DAG);
8590 case ISD::VECTOR_SPLICE_LEFT:
8591 case ISD::VECTOR_SPLICE_RIGHT:
8592 return lowerVECTOR_SPLICE(Op, DAG);
8593 case ISD::BUILD_VECTOR: {
8594 MVT VT = Op.getSimpleValueType();
8595 MVT EltVT = VT.getVectorElementType();
8596 if (!Subtarget.is64Bit() && EltVT == MVT::i64)
8597 return lowerBuildVectorViaVID(Op, DAG, Subtarget);
8598 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
8599 }
8600 case ISD::SPLAT_VECTOR: {
8601 MVT VT = Op.getSimpleValueType();
8602 MVT EltVT = VT.getVectorElementType();
8603 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
8604 EltVT == MVT::bf16) {
8605 SDLoc DL(Op);
8606 SDValue Elt;
8607 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
8608 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
8609 Elt = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, Subtarget.getXLenVT(),
8610 Op.getOperand(0));
8611 else
8612 Elt = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op.getOperand(0));
8613 MVT IVT = VT.changeVectorElementType(MVT::i16);
8614 return DAG.getNode(ISD::BITCAST, DL, VT,
8615 DAG.getNode(ISD::SPLAT_VECTOR, DL, IVT, Elt));
8616 }
8617
8618 if (EltVT == MVT::i1)
8619 return lowerVectorMaskSplat(Op, DAG);
8620 return SDValue();
8621 }
8622 case ISD::VECTOR_SHUFFLE:
8623 return lowerVECTOR_SHUFFLE(Op, DAG);
8624 case ISD::CONCAT_VECTORS: {
8625 // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
8626 // better than going through the stack, as the default expansion does.
8627 SDLoc DL(Op);
8628 MVT VT = Op.getSimpleValueType();
8629 MVT ContainerVT = VT;
8630 if (VT.isFixedLengthVector())
8631 ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
8632
8633 // Recursively split concat_vectors with more than 2 operands:
8634 //
8635 // concat_vector op1, op2, op3, op4
8636 // ->
8637 // concat_vector (concat_vector op1, op2), (concat_vector op3, op4)
8638 //
8639 // This reduces the length of the chain of vslideups and allows us to
8640 // perform the vslideups at a smaller LMUL, limited to MF2.
8641 if (Op.getNumOperands() > 2 &&
8642 ContainerVT.bitsGE(RISCVTargetLowering::getM1VT(ContainerVT))) {
8643 MVT HalfVT = VT.getHalfNumVectorElementsVT();
8644 assert(isPowerOf2_32(Op.getNumOperands()));
8645 size_t HalfNumOps = Op.getNumOperands() / 2;
8646 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
8647 Op->ops().take_front(HalfNumOps));
8648 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
8649 Op->ops().drop_front(HalfNumOps));
8650 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
8651 }
8652
8653 unsigned NumOpElts =
8654 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
8655 SDValue Vec = DAG.getUNDEF(VT);
8656 for (const auto &OpIdx : enumerate(Op->ops())) {
8657 SDValue SubVec = OpIdx.value();
8658 // Don't insert undef subvectors.
8659 if (SubVec.isUndef())
8660 continue;
8661 Vec = DAG.getInsertSubvector(DL, Vec, SubVec, OpIdx.index() * NumOpElts);
8662 }
8663 return Vec;
8664 }
8665 case ISD::LOAD: {
8666 auto *Load = cast<LoadSDNode>(Op);
8667 EVT VT = Load->getValueType(0);
8668 if (VT == MVT::f64) {
8669 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8670 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8671
8672 // Replace a double precision load with two i32 loads and a BuildPairF64.
8673 SDLoc DL(Op);
8674 SDValue BasePtr = Load->getBasePtr();
8675 SDValue Chain = Load->getChain();
8676
8677 SDValue Lo =
8678 DAG.getLoad(MVT::i32, DL, Chain, BasePtr, Load->getPointerInfo(),
8679 Load->getBaseAlign(), Load->getMemOperand()->getFlags());
8680 BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
8681 SDValue Hi = DAG.getLoad(
8682 MVT::i32, DL, Chain, BasePtr, Load->getPointerInfo().getWithOffset(4),
8683 Load->getBaseAlign(), Load->getMemOperand()->getFlags());
8684 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
8685 Hi.getValue(1));
8686
8687 // For big-endian, swap the order of Lo and Hi.
8688 if (!Subtarget.isLittleEndian())
8689 std::swap(Lo, Hi);
8690
8691 SDValue Pair = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
8692 return DAG.getMergeValues({Pair, Chain}, DL);
8693 }
8694
8695 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
8696 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
8697 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
8698
8699 // Determine the half-size type
8700 MVT HalfVT;
8701 if (VT == MVT::v2i32)
8702 HalfVT = MVT::i32;
8703 else if (VT == MVT::v4i16)
8704 HalfVT = MVT::v2i16;
8705 else // VT == MVT::v8i8
8706 HalfVT = MVT::v4i8;
8707
8708 SDLoc DL(Op);
8709 SDValue BasePtr = Load->getBasePtr();
8710 SDValue Chain = Load->getChain();
8711
8712 // Create two loads for the lower and upper halves
8713 SDValue Lo =
8714 DAG.getLoad(HalfVT, DL, Chain, BasePtr, Load->getPointerInfo(),
8715 Load->getBaseAlign(), Load->getMemOperand()->getFlags());
8716 unsigned HalfSize = HalfVT.getStoreSize();
8717 BasePtr =
8718 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(HalfSize));
8719 SDValue Hi =
8720 DAG.getLoad(HalfVT, DL, Chain, BasePtr,
8721 Load->getPointerInfo().getWithOffset(HalfSize),
8722 Load->getBaseAlign(), Load->getMemOperand()->getFlags());
8723 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo.getValue(1),
8724 Hi.getValue(1));
8725
8726 // Combine the two halves into the result vector
8727 SDValue Result;
8728 if (VT == MVT::v2i32) {
8729 // For v2i32, build vector from two i32 scalars
8730 Result = DAG.getNode(ISD::BUILD_VECTOR, DL, VT, Lo, Hi);
8731 } else {
8732 // For v4i16 and v8i8, use CONCAT_VECTORS
8733 Result = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
8734 }
8735
8736 return DAG.getMergeValues({Result, Chain}, DL);
8737 }
8738
8739 if (VT == MVT::bf16)
8740 return lowerXAndesBfHCvtBFloat16Load(Op, DAG);
8741
8742 // Handle normal vector tuple load.
8743 if (VT.isRISCVVectorTuple()) {
8744 SDLoc DL(Op);
8745 MVT XLenVT = Subtarget.getXLenVT();
8746 unsigned NF = VT.getRISCVVectorTupleNumFields();
8747 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
8748 unsigned NumElts = Sz / (NF * 8);
8749 int Log2LMUL = Log2_64(NumElts) - 3;
8750
8751 auto Flag = SDNodeFlags();
8752 Flag.setNoUnsignedWrap(true);
8753 SDValue Ret = DAG.getUNDEF(VT);
8754 SDValue BasePtr = Load->getBasePtr();
8755 SDValue VROffset = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
8756 VROffset =
8757 DAG.getNode(ISD::SHL, DL, XLenVT, VROffset,
8758 DAG.getConstant(std::max(Log2LMUL, 0), DL, XLenVT));
8759 SmallVector<SDValue, 8> OutChains;
8760
8761 // Load NF vector registers and combine them to a vector tuple.
8762 for (unsigned i = 0; i < NF; ++i) {
8763 SDValue LoadVal = DAG.getLoad(
8764 MVT::getScalableVectorVT(MVT::i8, NumElts), DL, Load->getChain(),
8765 BasePtr, MachinePointerInfo(Load->getAddressSpace()), Align(8));
8766 OutChains.push_back(LoadVal.getValue(1));
8767 Ret = DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VT, Ret, LoadVal,
8768 DAG.getTargetConstant(i, DL, MVT::i32));
8769 BasePtr = DAG.getNode(ISD::ADD, DL, XLenVT, BasePtr, VROffset, Flag);
8770 }
8771 return DAG.getMergeValues(
8772 {Ret, DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains)}, DL);
8773 }
8774
8775 if (auto V = expandUnalignedRVVLoad(Op, DAG))
8776 return V;
8777 if (Op.getValueType().isFixedLengthVector())
8778 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
8779 return Op;
8780 }
8781 case ISD::STORE: {
8782 auto *Store = cast<StoreSDNode>(Op);
8783 SDValue StoredVal = Store->getValue();
8784 EVT VT = StoredVal.getValueType();
8785
8786 if (VT == MVT::f64) {
8787 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8788 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8789
8790 // Replace a double precision store with a SplitF64 and i32 stores.
8791 SDValue DL(Op);
8792 SDValue BasePtr = Store->getBasePtr();
8793 SDValue Chain = Store->getChain();
8794 SDValue Split = DAG.getNode(RISCVISD::SplitF64, DL,
8795 DAG.getVTList(MVT::i32, MVT::i32), StoredVal);
8796
8797 SDValue Lo = Split.getValue(0);
8798 SDValue Hi = Split.getValue(1);
8799
8800 // For big-endian, swap the order of Lo and Hi before storing.
8801 if (!Subtarget.isLittleEndian())
8802 std::swap(Lo, Hi);
8803
8804 SDValue LoStore = DAG.getStore(
8805 Chain, DL, Lo, BasePtr, Store->getPointerInfo(),
8806 Store->getBaseAlign(), Store->getMemOperand()->getFlags());
8807 BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(4));
8808 SDValue HiStore = DAG.getStore(
8809 Chain, DL, Hi, BasePtr, Store->getPointerInfo().getWithOffset(4),
8810 Store->getBaseAlign(), Store->getMemOperand()->getFlags());
8811 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoStore, HiStore);
8812 }
8813 if (VT == MVT::i64) {
8814 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
8815 "Unexpected custom legalisation");
8816 if (Store->isTruncatingStore())
8817 return SDValue();
8818
8819 if (Store->getAlign() < Subtarget.getZilsdAlign())
8820 return SDValue();
8821
8822 SDLoc DL(Op);
8823 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, StoredVal,
8824 DAG.getTargetConstant(0, DL, MVT::i32));
8825 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, StoredVal,
8826 DAG.getTargetConstant(1, DL, MVT::i32));
8827
8828 return DAG.getMemIntrinsicNode(
8829 RISCVISD::SD_RV32, DL, DAG.getVTList(MVT::Other),
8830 {Store->getChain(), Lo, Hi, Store->getBasePtr()}, MVT::i64,
8831 Store->getMemOperand());
8832 }
8833
8834 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
8835 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
8836 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
8837
8838 auto *Store = cast<StoreSDNode>(Op);
8839 SDValue Val = Store->getValue();
8840
8841 // Determine the half-size type
8842 MVT HalfVT;
8843 if (VT == MVT::v2i32)
8844 HalfVT = MVT::i32;
8845 else if (VT == MVT::v4i16)
8846 HalfVT = MVT::v2i16;
8847 else // VT == MVT::v8i8
8848 HalfVT = MVT::v4i8;
8849
8850 SDLoc DL(Op);
8851 SDValue BasePtr = Store->getBasePtr();
8852 SDValue Chain = Store->getChain();
8853
8854 // Extract the two halves from the vector
8855 SDValue Lo, Hi;
8856 if (VT == MVT::v2i32) {
8857 // For v2i32, extract two i32 scalars
8858 Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Val,
8859 DAG.getVectorIdxConstant(0, DL));
8860 Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, Val,
8861 DAG.getVectorIdxConstant(1, DL));
8862 } else {
8863 // For v4i16 and v8i8, extract two vector halves
8864 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Val,
8865 DAG.getVectorIdxConstant(0, DL));
8866 unsigned HalfNumElts = HalfVT.getVectorNumElements();
8867 Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Val,
8868 DAG.getVectorIdxConstant(HalfNumElts, DL));
8869 }
8870
8871 // Create two stores for the lower and upper halves
8872 SDValue LoStore = DAG.getStore(
8873 Chain, DL, Lo, BasePtr, Store->getPointerInfo(),
8874 Store->getBaseAlign(), Store->getMemOperand()->getFlags());
8875 unsigned HalfSize = HalfVT.getStoreSize();
8876 BasePtr =
8877 DAG.getObjectPtrOffset(DL, BasePtr, TypeSize::getFixed(HalfSize));
8878 SDValue HiStore = DAG.getStore(
8879 Chain, DL, Hi, BasePtr,
8880 Store->getPointerInfo().getWithOffset(HalfSize),
8881 Store->getBaseAlign(), Store->getMemOperand()->getFlags());
8882
8883 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoStore, HiStore);
8884 }
8885
8886 if (VT == MVT::bf16)
8887 return lowerXAndesBfHCvtBFloat16Store(Op, DAG);
8888
8889 // Handle normal vector tuple store.
8890 if (VT.isRISCVVectorTuple()) {
8891 SDLoc DL(Op);
8892 MVT XLenVT = Subtarget.getXLenVT();
8893 unsigned NF = VT.getRISCVVectorTupleNumFields();
8894 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
8895 unsigned NumElts = Sz / (NF * 8);
8896 int Log2LMUL = Log2_64(NumElts) - 3;
8897
8898 auto Flag = SDNodeFlags();
8899 Flag.setNoUnsignedWrap(true);
8900 SDValue Ret;
8901 SDValue Chain = Store->getChain();
8902 SDValue BasePtr = Store->getBasePtr();
8903 SDValue VROffset = DAG.getNode(RISCVISD::READ_VLENB, DL, XLenVT);
8904 VROffset =
8905 DAG.getNode(ISD::SHL, DL, XLenVT, VROffset,
8906 DAG.getConstant(std::max(Log2LMUL, 0), DL, XLenVT));
8907
8908 // Extract subregisters in a vector tuple and store them individually.
8909 for (unsigned i = 0; i < NF; ++i) {
8910 auto Extract =
8911 DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL,
8912 MVT::getScalableVectorVT(MVT::i8, NumElts), StoredVal,
8913 DAG.getTargetConstant(i, DL, MVT::i32));
8914 Ret = DAG.getStore(Chain, DL, Extract, BasePtr,
8915 MachinePointerInfo(Store->getAddressSpace()),
8916 Store->getBaseAlign(),
8917 Store->getMemOperand()->getFlags());
8918 Chain = Ret.getValue(0);
8919 BasePtr = DAG.getNode(ISD::ADD, DL, XLenVT, BasePtr, VROffset, Flag);
8920 }
8921 return Ret;
8922 }
8923
8924 if (auto V = expandUnalignedRVVStore(Op, DAG))
8925 return V;
8926 if (Op.getOperand(1).getValueType().isFixedLengthVector())
8927 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
8928 return Op;
8929 }
8930 case ISD::VP_LOAD:
8931 if (SDValue V = expandUnalignedVPLoad(Op, DAG))
8932 return V;
8933 [[fallthrough]];
8934 case ISD::MLOAD:
8935 return lowerMaskedLoad(Op, DAG);
8936 case ISD::VP_LOAD_FF:
8937 return lowerLoadFF(Op, DAG);
8938 case ISD::VP_STORE:
8939 if (SDValue V = expandUnalignedVPStore(Op, DAG))
8940 return V;
8941 [[fallthrough]];
8942 case ISD::MSTORE:
8943 return lowerMaskedStore(Op, DAG);
8944 case ISD::VECTOR_COMPRESS:
8945 return lowerVectorCompress(Op, DAG);
8946 case ISD::SELECT_CC: {
8947 // This occurs because we custom legalize SETGT and SETUGT for setcc. That
8948 // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
8949 // into separate SETCC+SELECT just like LegalizeDAG.
8950 SDValue Tmp1 = Op.getOperand(0);
8951 SDValue Tmp2 = Op.getOperand(1);
8952 SDValue True = Op.getOperand(2);
8953 SDValue False = Op.getOperand(3);
8954 EVT VT = Op.getValueType();
8955 SDValue CC = Op.getOperand(4);
8956 EVT CmpVT = Tmp1.getValueType();
8957 EVT CCVT =
8958 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
8959 SDLoc DL(Op);
8960 SDValue Cond =
8961 DAG.getNode(ISD::SETCC, DL, CCVT, Tmp1, Tmp2, CC, Op->getFlags());
8962 return DAG.getSelect(DL, VT, Cond, True, False);
8963 }
8964 case ISD::SETCC: {
8965 MVT OpVT = Op.getOperand(0).getSimpleValueType();
8966 if (OpVT.isScalarInteger()) {
8967 MVT VT = Op.getSimpleValueType();
8968 SDValue LHS = Op.getOperand(0);
8969 SDValue RHS = Op.getOperand(1);
8970 ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
8971 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
8972 "Unexpected CondCode");
8973
8974 SDLoc DL(Op);
8975
8976 // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
8977 // convert this to the equivalent of (set(u)ge X, C+1) by using
8978 // (xori (slti(u) X, C+1), 1). This avoids materializing a small constant
8979 // in a register.
8980 if (isa<ConstantSDNode>(RHS)) {
8981 int64_t Imm = cast<ConstantSDNode>(RHS)->getSExtValue();
8982 if (Imm != 0 && isInt<12>((uint64_t)Imm + 1)) {
8983 // If this is an unsigned compare and the constant is -1, incrementing
8984 // the constant would change behavior. The result should be false.
8985 if (CCVal == ISD::SETUGT && Imm == -1)
8986 return DAG.getConstant(0, DL, VT);
8987 // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
8988 CCVal = ISD::getSetCCSwappedOperands(CCVal);
8989 SDValue SetCC = DAG.getSetCC(
8990 DL, VT, LHS, DAG.getSignedConstant(Imm + 1, DL, OpVT), CCVal);
8991 return DAG.getLogicalNOT(DL, SetCC, VT);
8992 }
8993 // Lower (setugt X, 2047) as (setne (srl X, 11), 0).
8994 if (CCVal == ISD::SETUGT && Imm == 2047) {
8995 SDValue Shift = DAG.getNode(ISD::SRL, DL, OpVT, LHS,
8996 DAG.getShiftAmountConstant(11, OpVT, DL));
8997 return DAG.getSetCC(DL, VT, Shift, DAG.getConstant(0, DL, OpVT),
8998 ISD::SETNE);
8999 }
9000 }
9001
9002 // Not a constant we could handle, swap the operands and condition code to
9003 // SETLT/SETULT.
9004 CCVal = ISD::getSetCCSwappedOperands(CCVal);
9005 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
9006 }
9007
9008 MVT VT = Op.getSimpleValueType();
9009 if (Subtarget.hasStdExtP() && VT.isFixedLengthVector()) {
9010 ISD::CondCode CCVal = cast<CondCodeSDNode>(Op.getOperand(2))->get();
9011 SDValue LHS = Op.getOperand(0);
9012 SDValue RHS = Op.getOperand(1);
9013 SDLoc DL(Op);
9014 if (CCVal == ISD::SETNE) {
9015 // Convert setne X, 0 to setult 0, X.
9016 if (ISD::isConstantSplatVectorAllZeros(RHS.getNode())) {
9017 return DAG.getSetCC(DL, VT, RHS, LHS, ISD::SETULT);
9018 }
9019
9020 // Not a constant we could handle, convert to SETEQ+Invert
9021 SDValue SetCC = DAG.getSetCC(DL, VT, LHS, RHS, ISD::SETEQ);
9022 return DAG.getLogicalNOT(DL, SetCC, VT);
9023 }
9024
9025 if (CCVal == ISD::SETGT) {
9026 if (ISD::isConstantSplatVectorAllOnes(RHS.getNode())) {
9027 SDValue SetCC =
9028 DAG.getSetCC(DL, VT, LHS, DAG.getConstant(0, DL, VT), ISD::SETLT);
9029 return DAG.getLogicalNOT(DL, SetCC, VT);
9030 }
9031
9032 // Not a constant we could handle, swap the operands and condition code
9033 // to SETLT.
9034 CCVal = ISD::getSetCCSwappedOperands(CCVal);
9035 return DAG.getSetCC(DL, VT, RHS, LHS, CCVal);
9036 }
9037
9038 return SDValue();
9039 }
9040
9041 if (isPromotedOpNeedingSplit(Op.getOperand(0), Subtarget, *this))
9042 return SplitVectorOp(Op, DAG);
9043
9044 return lowerToScalableOp(Op, DAG);
9045 }
9046 case ISD::ADD:
9047 case ISD::SUB:
9048 case ISD::SDIV:
9049 case ISD::SREM:
9050 case ISD::UDIV:
9051 case ISD::UREM:
9052 case ISD::BSWAP:
9053 case ISD::CTPOP:
9054 return lowerToScalableOp(Op, DAG);
9055 case ISD::VSELECT: {
9056 EVT VT = Op.getValueType();
9057 // Split 64-bit vector VSELECT on RV32 with P extension for v4i16 and v8i8
9058 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9059 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9060 SDLoc DL(Op);
9061 SDValue Mask = Op.getOperand(0);
9062 SDValue TrueVal = Op.getOperand(1);
9063 SDValue FalseVal = Op.getOperand(2);
9064
9065 // Split all three operands into two halves
9066 auto [MaskLo, MaskHi] = DAG.SplitVector(Mask, DL);
9067 auto [TrueLo, TrueHi] = DAG.SplitVector(TrueVal, DL);
9068 auto [FalseLo, FalseHi] = DAG.SplitVector(FalseVal, DL);
9069
9070 // Perform VSELECT on each half
9071 SDValue ResLo = DAG.getNode(ISD::VSELECT, DL, TrueLo.getValueType(),
9072 MaskLo, TrueLo, FalseLo);
9073 SDValue ResHi = DAG.getNode(ISD::VSELECT, DL, TrueHi.getValueType(),
9074 MaskHi, TrueHi, FalseHi);
9075
9076 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
9077 }
9078 return lowerToScalableOp(Op, DAG);
9079 }
9080 case ISD::AND:
9081 case ISD::OR:
9082 case ISD::XOR:
9083 case ISD::MUL:
9084 case ISD::MULHS:
9085 case ISD::MULHU: {
9086 EVT VT = Op.getValueType();
9087 unsigned Opc = Op.getOpcode();
9088 // Split 64-bit vector AND/OR/XOR/MUL/MULHS/MULHU on RV32 with P extension
9089 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9090 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9091 SDLoc DL(Op);
9092 SDValue LHS = Op.getOperand(0);
9093 SDValue RHS = Op.getOperand(1);
9094
9095 // Determine the half-size type
9096 MVT HalfVT = (VT == MVT::v4i16) ? MVT::v2i16 : MVT::v4i8;
9097
9098 // Extract the two halves from LHS
9099 auto [LHSLo, LHSHi] = DAG.SplitVector(LHS, DL, HalfVT, HalfVT);
9100
9101 // Extract the two halves from RHS
9102 auto [RHSLo, RHSHi] = DAG.SplitVector(RHS, DL, HalfVT, HalfVT);
9103
9104 // Perform the operation on each half
9105 SDValue ResLo = DAG.getNode(Opc, DL, HalfVT, LHSLo, RHSLo);
9106 SDValue ResHi = DAG.getNode(Opc, DL, HalfVT, LHSHi, RHSHi);
9107
9108 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
9109 }
9110 // Lower v4i8/v2i16 MUL/MULHS/MULHU via widening multiply + srl + truncate.
9111 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9112 (Opc == ISD::MUL || Opc == ISD::MULHS || Opc == ISD::MULHU) &&
9113 (VT == MVT::v4i8 || VT == MVT::v2i16)) {
9114 assert((VT == MVT::v4i8 || Opc == ISD::MUL) &&
9115 "Unexpected custom legalisation");
9116 SDLoc DL(Op);
9117 MVT WideVT = (VT == MVT::v4i8) ? MVT::v4i16 : MVT::v2i32;
9118 unsigned WMulOpc =
9119 (Opc == ISD::MULHU) ? RISCVISD::PWMULU : RISCVISD::PWMUL;
9120 SDValue Res =
9121 DAG.getNode(WMulOpc, DL, WideVT, Op.getOperand(0), Op.getOperand(1));
9122 if (Opc != ISD::MUL) {
9123 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
9124 Res = DAG.getNode(ISD::SRL, DL, WideVT, Res,
9125 DAG.getConstant(EltBits, DL, WideVT));
9126 }
9127 return DAG.getNode(ISD::TRUNCATE, DL, VT, Res);
9128 }
9129 return lowerToScalableOp(Op, DAG);
9130 }
9131 case ISD::ANY_EXTEND_VECTOR_INREG: {
9132 EVT VT = Op.getValueType();
9133 assert(Subtarget.hasStdExtP() && Subtarget.is64Bit() &&
9134 (VT == MVT::v2i32 || VT == MVT::v4i16) &&
9135 "Unexpected custom legalisation");
9136 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(Op), VT,
9137 Op.getOperand(0));
9138 }
9139 case ISD::SHL:
9140 case ISD::SRL:
9141 case ISD::SRA:
9142 case ISD::SSHLSAT:
9143 if (Op.getSimpleValueType().isFixedLengthVector()) {
9144 if (Subtarget.hasStdExtP()) {
9145 SDValue ShAmtVec = Op.getOperand(1);
9146 SDValue SplatVal;
9147 if (ShAmtVec.getOpcode() == ISD::SPLAT_VECTOR)
9148 SplatVal = ShAmtVec.getOperand(0);
9149 else if (ShAmtVec.getOpcode() == ISD::BUILD_VECTOR)
9150 SplatVal = cast<BuildVectorSDNode>(ShAmtVec)->getSplatValue();
9151
9152 if (!SplatVal)
9153 return SDValue();
9154
9155 unsigned Opc;
9156 switch (Op.getOpcode()) {
9157 default:
9158 llvm_unreachable("Unexpected opcode");
9159 case ISD::SHL:
9160 Opc = RISCVISD::PSHL;
9161 break;
9162 case ISD::SRL:
9163 Opc = RISCVISD::PSRL;
9164 break;
9165 case ISD::SRA:
9166 Opc = RISCVISD::PSRA;
9167 break;
9168 case ISD::SSHLSAT:
9169 Opc = RISCVISD::PSSHA;
9170 break;
9171 }
9172 return DAG.getNode(Opc, SDLoc(Op), Op.getValueType(), Op.getOperand(0),
9173 SplatVal);
9174 }
9175 return lowerToScalableOp(Op, DAG);
9176 }
9177 assert(Op.getOpcode() != ISD::SSHLSAT);
9178 // This can be called for an i32 shift amount that needs to be promoted.
9179 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
9180 "Unexpected custom legalisation");
9181 return SDValue();
9182 case ISD::MASKED_UDIV:
9183 case ISD::MASKED_SDIV:
9184 case ISD::MASKED_UREM:
9185 case ISD::MASKED_SREM: {
9186 SDLoc DL(Op);
9187 MVT VT = Op.getSimpleValueType();
9188 MVT ContainerVT = getContainerForFixedLengthVector(VT);
9189 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
9190 SDValue Res = DAG.getNode(
9191 getRISCVVLOp(Op), DL, ContainerVT,
9192 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget),
9193 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget),
9194 DAG.getUNDEF(ContainerVT),
9195 convertToScalableVector(getMaskTypeFor(ContainerVT), Op.getOperand(2),
9196 DAG, Subtarget),
9197 VL);
9198 return convertFromScalableVector(VT, Res, DAG, Subtarget);
9199 }
9200 case ISD::FABS:
9201 case ISD::FNEG:
9202 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9203 return lowerFABSorFNEG(Op, DAG, Subtarget);
9204 [[fallthrough]];
9205 case ISD::FADD:
9206 case ISD::FSUB:
9207 case ISD::FMUL:
9208 case ISD::FDIV:
9209 case ISD::FSQRT:
9210 case ISD::FMA:
9211 case ISD::FMINNUM:
9212 case ISD::FMAXNUM:
9213 case ISD::FMINIMUMNUM:
9214 case ISD::FMAXIMUMNUM:
9215 if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
9216 return SplitVectorOp(Op, DAG);
9217 [[fallthrough]];
9218 case ISD::AVGFLOORS:
9219 case ISD::AVGFLOORU:
9220 case ISD::AVGCEILS:
9221 case ISD::AVGCEILU:
9222 case ISD::SMIN:
9223 case ISD::SMAX:
9224 case ISD::UMIN:
9225 case ISD::UMAX:
9226 case ISD::UADDSAT:
9227 case ISD::USUBSAT:
9228 case ISD::SADDSAT:
9229 case ISD::SSUBSAT:
9230 return lowerToScalableOp(Op, DAG);
9231 case ISD::ABDS:
9232 case ISD::ABDU: {
9233 EVT VT = Op->getValueType(0);
9234 // Only SEW=8/16 are supported in Zvabd.
9235 if (Subtarget.hasStdExtZvabd() && VT.isVector() &&
9236 (VT.getVectorElementType() == MVT::i8 ||
9237 VT.getVectorElementType() == MVT::i16))
9238 return lowerToScalableOp(Op, DAG);
9239
9240 SDLoc dl(Op);
9241 SDValue LHS = DAG.getFreeze(Op->getOperand(0));
9242 SDValue RHS = DAG.getFreeze(Op->getOperand(1));
9243 bool IsSigned = Op->getOpcode() == ISD::ABDS;
9244
9245 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
9246 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
9247 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
9248 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
9249 SDValue Max = DAG.getNode(MaxOpc, dl, VT, LHS, RHS);
9250 SDValue Min = DAG.getNode(MinOpc, dl, VT, LHS, RHS);
9251 return DAG.getNode(ISD::SUB, dl, VT, Max, Min);
9252 }
9253 case ISD::ABS:
9254 case ISD::ABS_MIN_POISON:
9255 return lowerABS(Op, DAG);
9256 case ISD::CTLZ:
9257 case ISD::CTLZ_ZERO_POISON:
9258 case ISD::CTTZ:
9259 case ISD::CTTZ_ZERO_POISON:
9260 if (Subtarget.hasStdExtZvbb())
9261 return lowerToScalableOp(Op, DAG);
9262 assert(Op.getOpcode() != ISD::CTTZ);
9263 return lowerCTLZ_CTTZ_ZERO_POISON(Op, DAG);
9264 case ISD::CLMUL: {
9265 MVT VT = Op.getSimpleValueType();
9266 assert(VT.isScalableVector() && Subtarget.hasStdExtZvbc() &&
9267 "Unexpected custom legalisation");
9268 // Promote to i64 vector.
9269 MVT I64VecVT = VT.changeVectorElementType(MVT::i64);
9270 SDLoc DL(Op);
9271 SDValue Op0 = DAG.getNode(ISD::ZERO_EXTEND, DL, I64VecVT, Op.getOperand(0));
9272 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, I64VecVT, Op.getOperand(1));
9273 SDValue CLMUL = DAG.getNode(ISD::CLMUL, DL, I64VecVT, Op0, Op1);
9274 return DAG.getNode(ISD::TRUNCATE, DL, VT, CLMUL);
9275 }
9276 case ISD::FCOPYSIGN:
9277 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9278 return lowerFCOPYSIGN(Op, DAG, Subtarget);
9279 if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
9280 return SplitVectorOp(Op, DAG);
9281 return lowerToScalableOp(Op, DAG);
9282 case ISD::STRICT_FADD:
9283 case ISD::STRICT_FSUB:
9284 case ISD::STRICT_FMUL:
9285 case ISD::STRICT_FDIV:
9286 case ISD::STRICT_FSQRT:
9287 case ISD::STRICT_FMA:
9288 if (isPromotedOpNeedingSplit(Op, Subtarget, *this))
9289 return SplitStrictFPVectorOp(Op, DAG);
9290 return lowerToScalableOp(Op, DAG);
9291 case ISD::STRICT_FSETCC:
9292 case ISD::STRICT_FSETCCS:
9293 return lowerVectorStrictFSetcc(Op, DAG);
9294 case ISD::STRICT_FCEIL:
9295 case ISD::STRICT_FRINT:
9296 case ISD::STRICT_FFLOOR:
9297 case ISD::STRICT_FTRUNC:
9298 case ISD::STRICT_FNEARBYINT:
9299 case ISD::STRICT_FROUND:
9300 case ISD::STRICT_FROUNDEVEN:
9301 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
9302 case ISD::MGATHER:
9303 case ISD::VP_GATHER:
9304 return lowerMaskedGather(Op, DAG);
9305 case ISD::MSCATTER:
9306 case ISD::VP_SCATTER:
9307 return lowerMaskedScatter(Op, DAG);
9308 case ISD::GET_ROUNDING:
9309 return lowerGET_ROUNDING(Op, DAG);
9310 case ISD::SET_ROUNDING:
9311 return lowerSET_ROUNDING(Op, DAG);
9312 case ISD::GET_FPENV:
9313 return lowerGET_FPENV(Op, DAG);
9314 case ISD::SET_FPENV:
9315 return lowerSET_FPENV(Op, DAG);
9316 case ISD::RESET_FPENV:
9317 return lowerRESET_FPENV(Op, DAG);
9318 case ISD::GET_FPMODE:
9319 return lowerGET_FPMODE(Op, DAG);
9320 case ISD::SET_FPMODE:
9321 return lowerSET_FPMODE(Op, DAG);
9322 case ISD::RESET_FPMODE:
9323 return lowerRESET_FPMODE(Op, DAG);
9324 case ISD::EH_DWARF_CFA:
9325 return lowerEH_DWARF_CFA(Op, DAG);
9326 case ISD::VP_MERGE:
9327 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
9328 return lowerVPMergeMask(Op, DAG);
9329 [[fallthrough]];
9330 case ISD::VP_SDIV:
9331 case ISD::VP_UDIV:
9332 case ISD::VP_SREM:
9333 case ISD::VP_UREM:
9334 return lowerVPOp(Op, DAG);
9335 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9336 return lowerVPStridedLoad(Op, DAG);
9337 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9338 return lowerVPStridedStore(Op, DAG);
9339 case ISD::EXPERIMENTAL_VP_SPLICE:
9340 return lowerVPSpliceExperimental(Op, DAG);
9341 case ISD::EXPERIMENTAL_VP_REVERSE:
9342 return lowerVPReverseExperimental(Op, DAG);
9343 case ISD::CLEAR_CACHE: {
9344 assert(getTargetMachine().getTargetTriple().isOSLinux() &&
9345 "llvm.clear_cache only needs custom lower on Linux targets");
9346 SDLoc DL(Op);
9347 SDValue Flags = DAG.getConstant(0, DL, Subtarget.getXLenVT());
9348 return emitFlushICache(DAG, Op.getOperand(0), Op.getOperand(1),
9349 Op.getOperand(2), Flags, DL);
9350 }
9351 case ISD::DYNAMIC_STACKALLOC:
9352 return lowerDYNAMIC_STACKALLOC(Op, DAG);
9353 case ISD::INIT_TRAMPOLINE:
9354 return lowerINIT_TRAMPOLINE(Op, DAG);
9355 case ISD::ADJUST_TRAMPOLINE:
9356 return lowerADJUST_TRAMPOLINE(Op, DAG);
9357 case ISD::PARTIAL_REDUCE_UMLA:
9358 case ISD::PARTIAL_REDUCE_SMLA:
9359 case ISD::PARTIAL_REDUCE_SUMLA:
9360 return lowerPARTIAL_REDUCE_MLA(Op, DAG);
9361 case ISD::CTTZ_ELTS:
9362 case ISD::CTTZ_ELTS_ZERO_POISON:
9363 return lowerCttzElts(Op, DAG, Subtarget);
9364 }
9365}
9366
9367SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
9368 SDValue Start, SDValue End,
9369 SDValue Flags, SDLoc DL) const {
9370 MakeLibCallOptions CallOptions;
9371 std::pair<SDValue, SDValue> CallResult =
9372 makeLibCall(DAG, RTLIB::RISCV_FLUSH_ICACHE, MVT::isVoid,
9373 {Start, End, Flags}, CallOptions, DL, InChain);
9374
9375 // This function returns void so only the out chain matters.
9376 return CallResult.second;
9377}
9378
9379SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
9380 SelectionDAG &DAG) const {
9381 if (!Subtarget.is64Bit())
9382 llvm::reportFatalUsageError("Trampolines only implemented for RV64");
9383
9384 // Create an MCCodeEmitter to encode instructions.
9385 TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
9386 assert(TLO);
9387 MCContext &MCCtx = TLO->getContext();
9388
9389 std::unique_ptr<MCCodeEmitter> CodeEmitter(
9390 createRISCVMCCodeEmitter(*getTargetMachine().getMCInstrInfo(), MCCtx));
9391
9392 SDValue Root = Op.getOperand(0);
9393 SDValue Trmp = Op.getOperand(1); // trampoline
9394 SDLoc dl(Op);
9395
9396 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
9397
9398 // We store in the trampoline buffer the following instructions and data.
9399 // Offset:
9400 // 0: auipc t2, 0
9401 // 4: ld t0, 24(t2)
9402 // 8: ld t2, 16(t2)
9403 // 12: jalr t0
9404 // 16: <StaticChainOffset>
9405 // 24: <FunctionAddressOffset>
9406 // 32:
9407 // Offset with branch control flow protection enabled:
9408 // 0: lpad <imm20>
9409 // 4: auipc t3, 0
9410 // 8: ld t2, 28(t3)
9411 // 12: ld t3, 20(t3)
9412 // 16: jalr t2
9413 // 20: <StaticChainOffset>
9414 // 28: <FunctionAddressOffset>
9415 // 36:
9416
9417 const MachineFunction &MF = DAG.getMachineFunction();
9418 const bool HasCFBranch =
9419 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
9420 const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
9421 const unsigned StaticChainOffset = StaticChainIdx * 4;
9422 const unsigned FunctionAddressOffset = StaticChainOffset + 8;
9423
9424 const MCSubtargetInfo &STI = getTargetMachine().getMCSubtargetInfo();
9425 auto GetEncoding = [&](const MCInst &MC) {
9426 SmallVector<char, 4> CB;
9427 SmallVector<MCFixup> Fixups;
9428 CodeEmitter->encodeInstruction(MC, CB, Fixups, STI);
9429 uint32_t Encoding = support::endian::read32le(CB.data());
9430 return Encoding;
9431 };
9432
9433 SmallVector<SDValue> OutChains;
9434
9435 SmallVector<uint32_t> Encodings;
9436 if (!HasCFBranch) {
9437 Encodings.append(
9438 {// auipc t2, 0
9439 // Loads the current PC into t2.
9440 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X7).addImm(0)),
9441 // ld t0, 24(t2)
9442 // Loads the function address into t0. Note that we are using offsets
9443 // pc-relative to the first instruction of the trampoline.
9444 GetEncoding(MCInstBuilder(RISCV::LD)
9445 .addReg(RISCV::X5)
9446 .addReg(RISCV::X7)
9447 .addImm(FunctionAddressOffset)),
9448 // ld t2, 16(t2)
9449 // Load the value of the static chain.
9450 GetEncoding(MCInstBuilder(RISCV::LD)
9451 .addReg(RISCV::X7)
9452 .addReg(RISCV::X7)
9453 .addImm(StaticChainOffset)),
9454 // jalr t0
9455 // Jump to the function.
9456 GetEncoding(MCInstBuilder(RISCV::JALR)
9457 .addReg(RISCV::X0)
9458 .addReg(RISCV::X5)
9459 .addImm(0))});
9460 } else {
9461 Encodings.append(
9462 {// auipc x0, <imm20> (lpad <imm20>)
9463 // Landing pad.
9464 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X0).addImm(0)),
9465 // auipc t3, 0
9466 // Loads the current PC into t3.
9467 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(RISCV::X28).addImm(0)),
9468 // ld t2, (FunctionAddressOffset - 4)(t3)
9469 // Loads the function address into t2. Note that we are using offsets
9470 // pc-relative to the SECOND instruction of the trampoline.
9471 GetEncoding(MCInstBuilder(RISCV::LD)
9472 .addReg(RISCV::X7)
9473 .addReg(RISCV::X28)
9474 .addImm(FunctionAddressOffset - 4)),
9475 // ld t3, (StaticChainOffset - 4)(t3)
9476 // Load the value of the static chain.
9477 GetEncoding(MCInstBuilder(RISCV::LD)
9478 .addReg(RISCV::X28)
9479 .addReg(RISCV::X28)
9480 .addImm(StaticChainOffset - 4)),
9481 // jalr t2
9482 // Software-guarded jump to the function.
9483 GetEncoding(MCInstBuilder(RISCV::JALR)
9484 .addReg(RISCV::X0)
9485 .addReg(RISCV::X7)
9486 .addImm(0))});
9487 }
9488
9489 // Store encoded instructions.
9490 for (auto [Idx, Encoding] : llvm::enumerate(Encodings)) {
9491 SDValue Addr = Idx > 0 ? DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
9492 DAG.getConstant(Idx * 4, dl, MVT::i64))
9493 : Trmp;
9494 OutChains.push_back(DAG.getTruncStore(
9495 Root, dl, DAG.getConstant(Encoding, dl, MVT::i64), Addr,
9496 MachinePointerInfo(TrmpAddr, Idx * 4), MVT::i32));
9497 }
9498
9499 // Now store the variable part of the trampoline.
9500 SDValue FunctionAddress = Op.getOperand(2);
9501 SDValue StaticChain = Op.getOperand(3);
9502
9503 // Store the given static chain and function pointer in the trampoline buffer.
9504 struct OffsetValuePair {
9505 const unsigned Offset;
9506 const SDValue Value;
9507 SDValue Addr = SDValue(); // Used to cache the address.
9508 } OffsetValues[] = {
9509 {StaticChainOffset, StaticChain},
9510 {FunctionAddressOffset, FunctionAddress},
9511 };
9512 for (auto &OffsetValue : OffsetValues) {
9513 SDValue Addr =
9514 DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
9515 DAG.getConstant(OffsetValue.Offset, dl, MVT::i64));
9516 OffsetValue.Addr = Addr;
9517 OutChains.push_back(
9518 DAG.getStore(Root, dl, OffsetValue.Value, Addr,
9519 MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
9520 }
9521
9522 assert(OutChains.size() == StaticChainIdx + 2 &&
9523 "Size of OutChains mismatch");
9524 SDValue StoreToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
9525
9526 // The end of instructions of trampoline is the same as the static chain
9527 // address that we computed earlier.
9528 SDValue EndOfTrmp = OffsetValues[0].Addr;
9529
9530 // Call clear cache on the trampoline instructions.
9531 SDValue Chain = DAG.getNode(ISD::CLEAR_CACHE, dl, MVT::Other, StoreToken,
9532 Trmp, EndOfTrmp);
9533
9534 return Chain;
9535}
9536
9537SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
9538 SelectionDAG &DAG) const {
9539 if (!Subtarget.is64Bit())
9540 llvm::reportFatalUsageError("Trampolines only implemented for RV64");
9541
9542 return Op.getOperand(0);
9543}
9544
9545SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
9546 SelectionDAG &DAG) const {
9547 // Currently, only the vdot4a and vdot4au case (from zvdot4a8i) should be
9548 // legal.
9549 // TODO: There are many other sub-cases we could potentially lower, are
9550 // any of them worthwhile? Ex: via vredsum, vwredsum, vwwmaccu, etc..
9551 SDLoc DL(Op);
9552 MVT VT = Op.getSimpleValueType();
9553 SDValue Accum = Op.getOperand(0);
9554 assert(Accum.getSimpleValueType() == VT &&
9555 VT.getVectorElementType() == MVT::i32);
9556 SDValue A = Op.getOperand(1);
9557 SDValue B = Op.getOperand(2);
9558 MVT ArgVT = A.getSimpleValueType();
9559 assert(ArgVT == B.getSimpleValueType() &&
9560 ArgVT.getVectorElementType() == MVT::i8);
9561 (void)ArgVT;
9562
9563 // The zvdot4a8i pseudos are defined with sources and destination both
9564 // being i32. This cast is needed for correctness to avoid incorrect
9565 // .vx matching of i8 splats.
9566 A = DAG.getBitcast(VT, A);
9567 B = DAG.getBitcast(VT, B);
9568
9569 MVT ContainerVT = VT;
9570 if (VT.isFixedLengthVector()) {
9571 ContainerVT = getContainerForFixedLengthVector(VT);
9572 Accum = convertToScalableVector(ContainerVT, Accum, DAG, Subtarget);
9573 A = convertToScalableVector(ContainerVT, A, DAG, Subtarget);
9574 B = convertToScalableVector(ContainerVT, B, DAG, Subtarget);
9575 }
9576
9577 unsigned Opc;
9578 switch (Op.getOpcode()) {
9579 case ISD::PARTIAL_REDUCE_SMLA:
9580 Opc = RISCVISD::VDOT4A_VL;
9581 break;
9582 case ISD::PARTIAL_REDUCE_UMLA:
9583 Opc = RISCVISD::VDOT4AU_VL;
9584 break;
9585 case ISD::PARTIAL_REDUCE_SUMLA:
9586 Opc = RISCVISD::VDOT4ASU_VL;
9587 break;
9588 default:
9589 llvm_unreachable("Unexpected opcode");
9590 }
9591 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
9592 SDValue Res = DAG.getNode(Opc, DL, ContainerVT, {A, B, Accum, Mask, VL});
9593 if (VT.isFixedLengthVector())
9594 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
9595 return Res;
9596}
9597
9598static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
9599 SelectionDAG &DAG, unsigned Flags) {
9600 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
9601}
9602
9603static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
9604 SelectionDAG &DAG, unsigned Flags) {
9605 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
9606 Flags);
9607}
9608
9609static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
9610 SelectionDAG &DAG, unsigned Flags) {
9611 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
9612 N->getOffset(), Flags);
9613}
9614
9615static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
9616 SelectionDAG &DAG, unsigned Flags) {
9617 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
9618}
9619
9620static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL,
9621 EVT Ty, SelectionDAG &DAG) {
9622 RISCVConstantPoolValue *CPV = RISCVConstantPoolValue::Create(N->getGlobal());
9623 SDValue CPAddr = DAG.getTargetConstantPool(CPV, Ty, Align(8));
9624 SDValue LC = DAG.getNode(RISCVISD::LLA, DL, Ty, CPAddr);
9625 return DAG.getLoad(
9626 Ty, DL, DAG.getEntryNode(), LC,
9627 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
9628}
9629
9630static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL,
9631 EVT Ty, SelectionDAG &DAG) {
9632 RISCVConstantPoolValue *CPV =
9633 RISCVConstantPoolValue::Create(*DAG.getContext(), N->getSymbol());
9634 SDValue CPAddr = DAG.getTargetConstantPool(CPV, Ty, Align(8));
9635 SDValue LC = DAG.getNode(RISCVISD::LLA, DL, Ty, CPAddr);
9636 return DAG.getLoad(
9637 Ty, DL, DAG.getEntryNode(), LC,
9638 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()));
9639}
9640
9641template <class NodeTy>
9642SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
9643 bool IsLocal, bool IsExternWeak) const {
9644 SDLoc DL(N);
9645 EVT Ty = getPointerTy(DAG.getDataLayout());
9646
9647 // When HWASAN is used and tagging of global variables is enabled
9648 // they should be accessed via the GOT, since the tagged address of a global
9649 // is incompatible with existing code models. This also applies to non-pic
9650 // mode.
9651 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) {
9652 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9653 if (IsLocal && !Subtarget.allowTaggedGlobals())
9654 // Use PC-relative addressing to access the symbol. This generates the
9655 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
9656 // %pcrel_lo(auipc)).
9657 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
9658
9659 // Use PC-relative addressing to access the GOT for this symbol, then load
9660 // the address from the GOT. This generates the pattern (PseudoLGA sym),
9661 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9662 SDValue Load =
9663 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
9664 MachineFunction &MF = DAG.getMachineFunction();
9665 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9666 MachinePointerInfo::getGOT(MF),
9667 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9668 MachineMemOperand::MOInvariant,
9669 LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
9670 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
9671 return Load;
9672 }
9673
9674 switch (getTargetMachine().getCodeModel()) {
9675 default:
9676 reportFatalUsageError("Unsupported code model for lowering");
9677 case CodeModel::Small: {
9678 // Generate a sequence for accessing addresses within the first 2 GiB of
9679 // address space.
9680 if (Subtarget.hasVendorXqcili()) {
9681 // Use QC.E.LI to generate the address, as this is easier to relax than
9682 // LUI/ADDI.
9683 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9684 return DAG.getNode(RISCVISD::QC_E_LI, DL, Ty, Addr);
9685 }
9686
9687 // This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
9688 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
9689 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
9690 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
9691 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNHi, AddrLo);
9692 }
9693 case CodeModel::Medium: {
9694 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9695 if (IsExternWeak) {
9696 // An extern weak symbol may be undefined, i.e. have value 0, which may
9697 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
9698 // symbol. This generates the pattern (PseudoLGA sym), which expands to
9699 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9700 SDValue Load =
9701 SDValue(DAG.getMachineNode(RISCV::PseudoLGA, DL, Ty, Addr), 0);
9702 MachineFunction &MF = DAG.getMachineFunction();
9703 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9704 MachinePointerInfo::getGOT(MF),
9705 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9706 MachineMemOperand::MOInvariant,
9707 LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
9708 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
9709 return Load;
9710 }
9711
9712 // Generate a sequence for accessing addresses within any 2GiB range within
9713 // the address space. This generates the pattern (PseudoLLA sym), which
9714 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
9715 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
9716 }
9717 case CodeModel::Large: {
9718 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N))
9719 return getLargeGlobalAddress(G, DL, Ty, DAG);
9720
9721 // Using pc-relative mode for other node type.
9722 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9723 return DAG.getNode(RISCVISD::LLA, DL, Ty, Addr);
9724 }
9725 }
9726}
9727
9728SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
9729 SelectionDAG &DAG) const {
9730 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
9731 assert(N->getOffset() == 0 && "unexpected offset in global node");
9732 const GlobalValue *GV = N->getGlobal();
9733 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(GV);
9734 return getAddr(N, DAG, IsLocal, GV->hasExternalWeakLinkage());
9735}
9736
9737SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
9738 SelectionDAG &DAG) const {
9739 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
9740
9741 return getAddr(N, DAG);
9742}
9743
9744SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
9745 SelectionDAG &DAG) const {
9746 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
9747
9748 return getAddr(N, DAG);
9749}
9750
9751SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
9752 SelectionDAG &DAG) const {
9753 JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
9754
9755 return getAddr(N, DAG);
9756}
9757
9758SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
9759 SelectionDAG &DAG,
9760 bool UseGOT) const {
9761 SDLoc DL(N);
9762 EVT Ty = getPointerTy(DAG.getDataLayout());
9763 const GlobalValue *GV = N->getGlobal();
9764 MVT XLenVT = Subtarget.getXLenVT();
9765
9766 if (UseGOT) {
9767 // Use PC-relative addressing to access the GOT for this TLS symbol, then
9768 // load the address from the GOT and add the thread pointer. This generates
9769 // the pattern (PseudoLA_TLS_IE sym), which expands to
9770 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
9771 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
9772 SDValue Load =
9773 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
9774 MachineFunction &MF = DAG.getMachineFunction();
9775 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9776 MachinePointerInfo::getGOT(MF),
9777 MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9778 MachineMemOperand::MOInvariant,
9779 LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
9780 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
9781
9782 // Add the thread pointer.
9783 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
9784 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
9785 }
9786
9787 // Generate a sequence for accessing the address relative to the thread
9788 // pointer, with the appropriate adjustment for the thread pointer offset.
9789 // This generates the pattern
9790 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
9791 SDValue AddrHi =
9792 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
9793 SDValue AddrAdd =
9794 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
9795 SDValue AddrLo =
9796 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
9797
9798 SDValue MNHi = DAG.getNode(RISCVISD::HI, DL, Ty, AddrHi);
9799 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
9800 SDValue MNAdd =
9801 DAG.getNode(RISCVISD::ADD_TPREL, DL, Ty, MNHi, TPReg, AddrAdd);
9802 return DAG.getNode(RISCVISD::ADD_LO, DL, Ty, MNAdd, AddrLo);
9803}
9804
9805SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
9806 SelectionDAG &DAG) const {
9807 SDLoc DL(N);
9808 EVT Ty = getPointerTy(DAG.getDataLayout());
9809 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
9810 const GlobalValue *GV = N->getGlobal();
9811
9812 // Use a PC-relative addressing mode to access the global dynamic GOT address.
9813 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
9814 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
9815 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
9816 SDValue Load =
9817 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
9818
9819 // Prepare argument list to generate call.
9820 ArgListTy Args;
9821 Args.emplace_back(Load, CallTy);
9822
9823 // Setup call to __tls_get_addr.
9824 TargetLowering::CallLoweringInfo CLI(DAG);
9825 CLI.setDebugLoc(DL)
9826 .setChain(DAG.getEntryNode())
9827 .setLibCallee(CallingConv::C, CallTy,
9828 DAG.getExternalSymbol("__tls_get_addr", Ty),
9829 std::move(Args));
9830
9831 return LowerCallTo(CLI).first;
9832}
9833
9834SDValue RISCVTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
9835 SelectionDAG &DAG) const {
9836 SDLoc DL(N);
9837 EVT Ty = getPointerTy(DAG.getDataLayout());
9838 const GlobalValue *GV = N->getGlobal();
9839
9840 // Use a PC-relative addressing mode to access the global dynamic GOT address.
9841 // This generates the pattern (PseudoLA_TLSDESC sym), which expands to
9842 //
9843 // auipc tX, %tlsdesc_hi(symbol) // R_RISCV_TLSDESC_HI20(symbol)
9844 // lw tY, tX, %tlsdesc_load_lo(label) // R_RISCV_TLSDESC_LOAD_LO12(label)
9845 // addi a0, tX, %tlsdesc_add_lo(label) // R_RISCV_TLSDESC_ADD_LO12(label)
9846 // jalr t0, tY // R_RISCV_TLSDESC_CALL(label)
9847 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
9848 return SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLSDESC, DL, Ty, Addr), 0);
9849}
9850
9851SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
9852 SelectionDAG &DAG) const {
9853 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
9854 assert(N->getOffset() == 0 && "unexpected offset in global node");
9855
9856 if (DAG.getTarget().useEmulatedTLS())
9857 return LowerToTLSEmulatedModel(N, DAG);
9858
9859 TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
9860
9861 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
9862 CallingConv::GHC)
9863 reportFatalUsageError("In GHC calling convention TLS is not supported");
9864
9865 SDValue Addr;
9866 switch (Model) {
9867 case TLSModel::LocalExec:
9868 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
9869 break;
9870 case TLSModel::InitialExec:
9871 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
9872 break;
9873 case TLSModel::LocalDynamic:
9874 case TLSModel::GeneralDynamic:
9875 Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
9876 : getDynamicTLSAddr(N, DAG);
9877 break;
9878 }
9879
9880 return Addr;
9881}
9882
9883// Return true if Val is equal to (setcc LHS, RHS, CC).
9884// Return false if Val is the inverse of (setcc LHS, RHS, CC).
9885// Otherwise, return std::nullopt.
9886static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
9887 ISD::CondCode CC, SDValue Val) {
9888 assert(Val->getOpcode() == ISD::SETCC);
9889 SDValue LHS2 = Val.getOperand(0);
9890 SDValue RHS2 = Val.getOperand(1);
9891 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val.getOperand(2))->get();
9892
9893 if (LHS == LHS2 && RHS == RHS2) {
9894 if (CC == CC2)
9895 return true;
9896 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
9897 return false;
9898 } else if (LHS == RHS2 && RHS == LHS2) {
9899 CC2 = ISD::getSetCCSwappedOperands(CC2);
9900 if (CC == CC2)
9901 return true;
9902 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
9903 return false;
9904 }
9905
9906 return std::nullopt;
9907}
9908
9909static bool isSimm12Constant(SDValue V) {
9910 return isa<ConstantSDNode>(V) && V->getAsAPIntVal().isSignedIntN(12);
9911}
9912
9913static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG,
9914 const RISCVSubtarget &Subtarget) {
9915 SDValue CondV = N->getOperand(0);
9916 SDValue TrueV = N->getOperand(1);
9917 SDValue FalseV = N->getOperand(2);
9918 MVT VT = N->getSimpleValueType(0);
9919 SDLoc DL(N);
9920
9921 if (!Subtarget.hasConditionalMoveFusion()) {
9922 // (select c, -1, y) -> -c | y
9923 if (isAllOnesConstant(TrueV)) {
9924 SDValue Neg = DAG.getNegative(CondV, DL, VT);
9925 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
9926 }
9927 // (select c, y, -1) -> (c-1) | y
9928 if (isAllOnesConstant(FalseV)) {
9929 SDValue Neg = DAG.getNode(ISD::ADD, DL, VT, CondV,
9930 DAG.getAllOnesConstant(DL, VT));
9931 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
9932 }
9933
9934 const bool HasCZero = VT.isScalarInteger() && Subtarget.hasCZEROLike();
9935
9936 // (select c, 0, y) -> (c-1) & y
9937 if (isNullConstant(TrueV) && (!HasCZero || isSimm12Constant(FalseV))) {
9938 SDValue Neg =
9939 DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT));
9940 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
9941 }
9942 if (isNullConstant(FalseV)) {
9943 if (auto *TrueC = dyn_cast<ConstantSDNode>(TrueV)) {
9944 // (select c, y, 0) -> (c * (y - 1)) + c
9945 int64_t MulImm = TrueC->getSExtValue();
9946 if (MulImm != INT64_MIN && isInt<12>(MulImm - 1) &&
9947 Subtarget.hasVendorXqciac())
9948 return DAG.getNode(RISCVISD::QC_MULIADD, DL, VT, CondV, CondV,
9949 DAG.getSignedTargetConstant(MulImm - 1, DL, VT));
9950
9951 // (select c, (1 << ShAmount) + 1, 0) -> (c << ShAmount) + c
9952 uint64_t TrueM1 = TrueC->getZExtValue() - 1;
9953 if (isPowerOf2_64(TrueM1)) {
9954 unsigned ShAmount = Log2_64(TrueM1);
9955 if (Subtarget.hasShlAdd(ShAmount))
9956 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, CondV,
9957 DAG.getTargetConstant(ShAmount, DL, VT), CondV);
9958 }
9959 }
9960 // (select c, y, 0) -> -c & y
9961 if (!HasCZero || isSimm12Constant(TrueV)) {
9962 SDValue Neg = DAG.getNegative(CondV, DL, VT);
9963 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
9964 }
9965 }
9966 }
9967
9968 // select c, ~x, x --> xor -c, x
9969 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
9970 const APInt &TrueVal = TrueV->getAsAPIntVal();
9971 const APInt &FalseVal = FalseV->getAsAPIntVal();
9972 if (~TrueVal == FalseVal) {
9973 SDValue Neg = DAG.getNegative(CondV, DL, VT);
9974 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
9975 }
9976 }
9977
9978 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
9979 // when both truev and falsev are also setcc.
9980 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
9981 FalseV.getOpcode() == ISD::SETCC) {
9982 SDValue LHS = CondV.getOperand(0);
9983 SDValue RHS = CondV.getOperand(1);
9984 ISD::CondCode CC = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
9985
9986 // (select x, x, y) -> x | y
9987 // (select !x, x, y) -> x & y
9988 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, TrueV)) {
9989 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
9990 DAG.getFreeze(FalseV));
9991 }
9992 // (select x, y, x) -> x & y
9993 // (select !x, y, x) -> x | y
9994 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, FalseV)) {
9995 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
9996 DAG.getFreeze(TrueV), FalseV);
9997 }
9998 }
9999
10000 return SDValue();
10001}
10002
10003// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
10004// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
10005// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
10006// being `0` or `-1`. In such cases we can replace `select` with `and`.
10007// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
10008// than `c0`?
10009static SDValue
10010foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
10011 const RISCVSubtarget &Subtarget) {
10012 if (Subtarget.hasShortForwardBranchIALU())
10013 return SDValue();
10014
10015 unsigned SelOpNo = 0;
10016 SDValue Sel = BO->getOperand(0);
10017 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
10018 SelOpNo = 1;
10019 Sel = BO->getOperand(1);
10020 }
10021
10022 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
10023 return SDValue();
10024
10025 unsigned ConstSelOpNo = 1;
10026 unsigned OtherSelOpNo = 2;
10027 if (!isa<ConstantSDNode>(Sel->getOperand(ConstSelOpNo))) {
10028 ConstSelOpNo = 2;
10029 OtherSelOpNo = 1;
10030 }
10031 SDValue ConstSelOp = Sel->getOperand(ConstSelOpNo);
10032 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
10033 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
10034 return SDValue();
10035
10036 SDValue ConstBinOp = BO->getOperand(SelOpNo ^ 1);
10037 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
10038 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
10039 return SDValue();
10040
10041 SDLoc DL(Sel);
10042 EVT VT = BO->getValueType(0);
10043
10044 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
10045 if (SelOpNo == 1)
10046 std::swap(NewConstOps[0], NewConstOps[1]);
10047
10048 SDValue NewConstOp =
10049 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
10050 if (!NewConstOp)
10051 return SDValue();
10052
10053 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
10054 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
10055 return SDValue();
10056
10057 SDValue OtherSelOp = Sel->getOperand(OtherSelOpNo);
10058 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
10059 if (SelOpNo == 1)
10060 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
10061 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
10062
10063 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
10064 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
10065 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
10066}
10067
10068SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
10069 SDValue CondV = Op.getOperand(0);
10070 SDValue TrueV = Op.getOperand(1);
10071 SDValue FalseV = Op.getOperand(2);
10072 SDLoc DL(Op);
10073 MVT VT = Op.getSimpleValueType();
10074 MVT XLenVT = Subtarget.getXLenVT();
10075
10076 // Handle P extension packed types by bitcasting to an integer of
10077 // matching width and reusing the scalar selection mechanism.
10078 // Reachable cases:
10079 // RV32: v4i8/v2i16 -> select on i32
10080 // RV32: v8i8/v4i16 -> select on i64 (legalizes to two i32 selects)
10081 // RV64: v8i8/v4i16/v2i32 -> select on i64
10082 if (Subtarget.isPExtPackedType(VT)) {
10083 MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits());
10084 SDValue TrueVInt = DAG.getBitcast(IntVT, TrueV);
10085 SDValue FalseVInt = DAG.getBitcast(IntVT, FalseV);
10086 SDValue ResultInt =
10087 DAG.getNode(ISD::SELECT, DL, IntVT, CondV, TrueVInt, FalseVInt);
10088 return DAG.getBitcast(VT, ResultInt);
10089 }
10090
10091 // Lower vector SELECTs to VSELECTs by splatting the condition.
10092 if (VT.isVector()) {
10093 MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
10094 SDValue CondSplat = DAG.getSplat(SplatCondVT, DL, CondV);
10095 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
10096 }
10097
10098 // Try some other optimizations before falling back to generic lowering.
10099 if (SDValue V = lowerSelectToBinOp(Op.getNode(), DAG, Subtarget))
10100 return V;
10101
10102 // When there is no cost for GPR <-> FPR, we can use zicond select for
10103 // floating value when CondV is int type
10104 bool FPinGPR = Subtarget.hasStdExtZfinx();
10105
10106 // We can handle FGPR without spliting into hi/lo parts
10107 bool FitsInGPR = TypeSize::isKnownLE(VT.getSizeInBits(),
10108 Subtarget.getXLenVT().getSizeInBits());
10109
10110 bool UseZicondForFPSel = Subtarget.hasStdExtZicond() && FPinGPR &&
10111 VT.isFloatingPoint() && FitsInGPR;
10112
10113 if (UseZicondForFPSel) {
10114
10115 auto CastToInt = [&](SDValue V) -> SDValue {
10116 // Treat +0.0 as int 0 to enable single 'czero' instruction generation.
10117 if (isNullFPConstant(V))
10118 return DAG.getConstant(0, DL, XLenVT);
10119
10120 if (VT == MVT::f16)
10121 return DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, V);
10122
10123 if (VT == MVT::f32 && Subtarget.is64Bit())
10124 return DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, XLenVT, V);
10125
10126 return DAG.getBitcast(XLenVT, V);
10127 };
10128
10129 SDValue TrueVInt = CastToInt(TrueV);
10130 SDValue FalseVInt = CastToInt(FalseV);
10131
10132 // Emit integer SELECT (lowers to Zicond)
10133 SDValue ResultInt =
10134 DAG.getNode(ISD::SELECT, DL, XLenVT, CondV, TrueVInt, FalseVInt);
10135
10136 // Convert back to floating VT
10137 if (VT == MVT::f32 && Subtarget.is64Bit())
10138 return DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, VT, ResultInt);
10139
10140 if (VT == MVT::f16)
10141 return DAG.getNode(RISCVISD::FMV_H_X, DL, VT, ResultInt);
10142
10143 return DAG.getBitcast(VT, ResultInt);
10144 }
10145
10146 // When Zicond or XVentanaCondOps is present, emit CZERO_EQZ and CZERO_NEZ
10147 // nodes to implement the SELECT. Performing the lowering here allows for
10148 // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless
10149 // sequence or RISCVISD::SELECT_CC node (branch-based select).
10150 if (Subtarget.hasCZEROLike() && VT.isScalarInteger()) {
10151
10152 // (select c, t, 0) -> (czero_eqz t, c)
10153 if (isNullConstant(FalseV))
10154 return DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV);
10155 // (select c, 0, f) -> (czero_nez f, c)
10156 if (isNullConstant(TrueV))
10157 return DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV);
10158
10159 // Check to see if a given operation is a 'NOT', if so return the negated
10160 // operand
10161 auto getNotOperand = [](const SDValue &Op) -> std::optional<const SDValue> {
10162 using namespace llvm::SDPatternMatch;
10163 SDValue Xor;
10164 if (sd_match(Op, m_OneUse(m_Not(m_Value(Xor))))) {
10165 return Xor;
10166 }
10167 return std::nullopt;
10168 };
10169 // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
10170 // (select c, (and f, ~x), f) -> (andn f, (czero_eqz x, c))
10171 if (TrueV.getOpcode() == ISD::AND &&
10172 (TrueV.getOperand(0) == FalseV || TrueV.getOperand(1) == FalseV)) {
10173 auto NotOperand = (TrueV.getOperand(0) == FalseV)
10174 ? getNotOperand(TrueV.getOperand(1))
10175 : getNotOperand(TrueV.getOperand(0));
10176 if (NotOperand) {
10177 SDValue CMOV =
10178 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, *NotOperand, CondV);
10179 SDValue NOT = DAG.getNOT(DL, CMOV, VT);
10180 return DAG.getNode(ISD::AND, DL, VT, FalseV, NOT);
10181 }
10182 return DAG.getNode(
10183 ISD::OR, DL, VT, TrueV,
10184 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV));
10185 }
10186
10187 // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
10188 // (select c, t, (and t, ~x)) -> (andn t, (czero_nez x, c))
10189 if (FalseV.getOpcode() == ISD::AND &&
10190 (FalseV.getOperand(0) == TrueV || FalseV.getOperand(1) == TrueV)) {
10191 auto NotOperand = (FalseV.getOperand(0) == TrueV)
10192 ? getNotOperand(FalseV.getOperand(1))
10193 : getNotOperand(FalseV.getOperand(0));
10194 if (NotOperand) {
10195 SDValue CMOV =
10196 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, *NotOperand, CondV);
10197 SDValue NOT = DAG.getNOT(DL, CMOV, VT);
10198 return DAG.getNode(ISD::AND, DL, VT, TrueV, NOT);
10199 }
10200 return DAG.getNode(
10201 ISD::OR, DL, VT, FalseV,
10202 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV));
10203 }
10204
10205 // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
10206 // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
10207 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
10208 const APInt &TrueVal = TrueV->getAsAPIntVal();
10209 const APInt &FalseVal = FalseV->getAsAPIntVal();
10210
10211 // Prefer these over Zicond to avoid materializing an immediate:
10212 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
10213 // (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
10214 if (CondV.getOpcode() == ISD::SETCC &&
10215 CondV.getOperand(0).getValueType() == VT && CondV.hasOneUse()) {
10216 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
10217 if ((CCVal == ISD::SETLT && isNullConstant(CondV.getOperand(1))) ||
10218 (CCVal == ISD::SETGT && isAllOnesConstant(CondV.getOperand(1)))) {
10219 int64_t TrueImm = TrueVal.getSExtValue();
10220 int64_t FalseImm = FalseVal.getSExtValue();
10221 if (CCVal == ISD::SETGT)
10222 std::swap(TrueImm, FalseImm);
10223 if (isInt<12>(TrueImm) && isInt<12>(FalseImm) &&
10224 isInt<12>(TrueImm - FalseImm)) {
10225 SDValue SRA =
10226 DAG.getNode(ISD::SRA, DL, VT, CondV.getOperand(0),
10227 DAG.getConstant(Subtarget.getXLen() - 1, DL, VT));
10228 SDValue AND =
10229 DAG.getNode(ISD::AND, DL, VT, SRA,
10230 DAG.getSignedConstant(TrueImm - FalseImm, DL, VT));
10231 return DAG.getNode(ISD::ADD, DL, VT, AND,
10232 DAG.getSignedConstant(FalseImm, DL, VT));
10233 }
10234 }
10235 }
10236
10237 // Use SHL/ADDI (and possible XORI) to avoid having to materialize
10238 // a constant in register
10239 if ((TrueVal - FalseVal).isPowerOf2() && FalseVal.isSignedIntN(12)) {
10240 SDValue Log2 = DAG.getConstant((TrueVal - FalseVal).logBase2(), DL, VT);
10241 SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
10242 return DAG.getNode(ISD::ADD, DL, VT, FalseV, BitDiff);
10243 }
10244 if ((FalseVal - TrueVal).isPowerOf2() && TrueVal.isSignedIntN(12)) {
10245 SDValue Log2 = DAG.getConstant((FalseVal - TrueVal).logBase2(), DL, VT);
10246 CondV = DAG.getLogicalNOT(DL, CondV, CondV->getValueType(0));
10247 SDValue BitDiff = DAG.getNode(ISD::SHL, DL, VT, CondV, Log2);
10248 return DAG.getNode(ISD::ADD, DL, VT, TrueV, BitDiff);
10249 }
10250
10251 auto getCost = [&](const APInt &Delta, const APInt &Addend) {
10252 const int DeltaCost = RISCVMatInt::getIntMatCost(
10253 Delta, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true);
10254 // Does the addend fold into an ADDI
10255 if (Addend.isSignedIntN(12))
10256 return DeltaCost;
10257 const int AddendCost = RISCVMatInt::getIntMatCost(
10258 Addend, Subtarget.getXLen(), Subtarget, /*CompressionCost=*/true);
10259 return AddendCost + DeltaCost;
10260 };
10261 bool IsCZERO_NEZ = getCost(FalseVal - TrueVal, TrueVal) <=
10262 getCost(TrueVal - FalseVal, FalseVal);
10263 SDValue LHSVal = DAG.getConstant(
10264 IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
10265 SDValue CMOV =
10266 DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10267 DL, VT, LHSVal, CondV);
10268 return DAG.getNode(ISD::ADD, DL, VT, CMOV, IsCZERO_NEZ ? TrueV : FalseV);
10269 }
10270
10271 // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
10272 // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
10273 if (isa<ConstantSDNode>(TrueV) != isa<ConstantSDNode>(FalseV)) {
10274 bool IsCZERO_NEZ = isa<ConstantSDNode>(TrueV);
10275 SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
10276 SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
10277 int64_t RawConstVal = cast<ConstantSDNode>(ConstVal)->getSExtValue();
10278 // Efficient only if the constant and its negation fit into `ADDI`
10279 // Prefer Add/Sub over Xor since can be compressed for small immediates
10280 if (isInt<12>(RawConstVal)) {
10281 // Fall back to XORI if Const == -0x800 since we don't have SUBI.
10282 unsigned SubOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::SUB;
10283 unsigned AddOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::ADD;
10284 SDValue SubOp = DAG.getNode(SubOpc, DL, VT, RegV, ConstVal);
10285 SDValue CZERO =
10286 DAG.getNode(IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10287 DL, VT, SubOp, CondV);
10288 return DAG.getNode(AddOpc, DL, VT, CZERO, ConstVal);
10289 }
10290 }
10291
10292 // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
10293 // Unless we have the short forward branch optimization.
10294 if (!Subtarget.hasConditionalMoveFusion())
10295 return DAG.getNode(
10296 ISD::OR, DL, VT,
10297 DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV, CondV),
10298 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV, CondV),
10299 SDNodeFlags::Disjoint);
10300 }
10301
10302 if (Op.hasOneUse()) {
10303 unsigned UseOpc = Op->user_begin()->getOpcode();
10304 if (isBinOp(UseOpc) && DAG.isSafeToSpeculativelyExecute(UseOpc)) {
10305 SDNode *BinOp = *Op->user_begin();
10306 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(*Op->user_begin(),
10307 DAG, Subtarget)) {
10308 DAG.ReplaceAllUsesWith(BinOp, &NewSel);
10309 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
10310 // may return a constant node and cause crash in lowerSELECT.
10311 if (NewSel.getOpcode() == ISD::SELECT)
10312 return lowerSELECT(NewSel, DAG);
10313 return NewSel;
10314 }
10315 }
10316 }
10317
10318 // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
10319 // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
10320 const ConstantFPSDNode *FPTV = dyn_cast<ConstantFPSDNode>(TrueV);
10321 const ConstantFPSDNode *FPFV = dyn_cast<ConstantFPSDNode>(FalseV);
10322 if (FPTV && FPFV) {
10323 if (FPTV->isOne() && FPFV->isPosZero())
10324 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, CondV);
10325 if (FPTV->isPosZero() && FPFV->isOne()) {
10326 SDValue XOR = DAG.getNode(ISD::XOR, DL, XLenVT, CondV,
10327 DAG.getConstant(1, DL, XLenVT));
10328 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, XOR);
10329 }
10330 }
10331
10332 // If the condition is not an integer SETCC which operates on XLenVT, we need
10333 // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
10334 // (select condv, truev, falsev)
10335 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
10336 if (CondV.getOpcode() != ISD::SETCC ||
10337 CondV.getOperand(0).getSimpleValueType() != XLenVT) {
10338 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
10339 SDValue SetNE = DAG.getCondCode(ISD::SETNE);
10340
10341 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
10342
10343 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
10344 }
10345
10346 // If the CondV is the output of a SETCC node which operates on XLenVT inputs,
10347 // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
10348 // advantage of the integer compare+branch instructions. i.e.:
10349 // (select (setcc lhs, rhs, cc), truev, falsev)
10350 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
10351 SDValue LHS = CondV.getOperand(0);
10352 SDValue RHS = CondV.getOperand(1);
10353 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
10354
10355 // Special case for a select of 2 constants that have a difference of 1.
10356 // Normally this is done by DAGCombine, but if the select is introduced by
10357 // type legalization or op legalization, we miss it. Restricting to SETLT
10358 // case for now because that is what signed saturating add/sub need.
10359 // FIXME: We don't need the condition to be SETLT or even a SETCC,
10360 // but we would probably want to swap the true/false values if the condition
10361 // is SETGE/SETLE to avoid an XORI.
10362 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
10363 CCVal == ISD::SETLT) {
10364 const APInt &TrueVal = TrueV->getAsAPIntVal();
10365 const APInt &FalseVal = FalseV->getAsAPIntVal();
10366 if (TrueVal - 1 == FalseVal)
10367 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
10368 if (TrueVal + 1 == FalseVal)
10369 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
10370 }
10371
10372 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
10373 // 1 < x ? x : 1 -> 0 < x ? x : 1
10374 if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
10375 RHS == TrueV && LHS == FalseV) {
10376 LHS = DAG.getConstant(0, DL, VT);
10377 // 0 <u x is the same as x != 0.
10378 if (CCVal == ISD::SETULT) {
10379 std::swap(LHS, RHS);
10380 CCVal = ISD::SETNE;
10381 }
10382 }
10383
10384 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
10385 if (isAllOnesConstant(RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
10386 RHS == FalseV) {
10387 RHS = DAG.getConstant(0, DL, VT);
10388 }
10389
10390 SDValue TargetCC = DAG.getCondCode(CCVal);
10391
10392 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
10393 // (select (setcc lhs, rhs, CC), constant, falsev)
10394 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
10395 std::swap(TrueV, FalseV);
10396 TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType()));
10397 }
10398
10399 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
10400 return DAG.getNode(RISCVISD::SELECT_CC, DL, VT, Ops);
10401}
10402
10403SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
10404 SDValue CondV = Op.getOperand(1);
10405 SDLoc DL(Op);
10406 MVT XLenVT = Subtarget.getXLenVT();
10407
10408 if (CondV.getOpcode() == ISD::SETCC &&
10409 CondV.getOperand(0).getValueType() == XLenVT) {
10410 SDValue LHS = CondV.getOperand(0);
10411 SDValue RHS = CondV.getOperand(1);
10412 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
10413
10414 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
10415
10416 SDValue TargetCC = DAG.getCondCode(CCVal);
10417 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
10418 LHS, RHS, TargetCC, Op.getOperand(2));
10419 }
10420
10421 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
10422 CondV, DAG.getConstant(0, DL, XLenVT),
10423 DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
10424}
10425
10426SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
10427 MachineFunction &MF = DAG.getMachineFunction();
10428 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
10429
10430 SDLoc DL(Op);
10431 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
10432 getPointerTy(MF.getDataLayout()));
10433
10434 // vastart just stores the address of the VarArgsFrameIndex slot into the
10435 // memory location argument.
10436 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
10437 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
10438 MachinePointerInfo(SV));
10439}
10440
10441SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
10442 SelectionDAG &DAG) const {
10443 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10444 MachineFunction &MF = DAG.getMachineFunction();
10445 MachineFrameInfo &MFI = MF.getFrameInfo();
10446 MFI.setFrameAddressIsTaken(true);
10447 Register FrameReg = RI.getFrameRegister(MF);
10448 int XLenInBytes = Subtarget.getXLen() / 8;
10449
10450 EVT VT = Op.getValueType();
10451 SDLoc DL(Op);
10452 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
10453 unsigned Depth = Op.getConstantOperandVal(0);
10454 while (Depth--) {
10455 int Offset = -(XLenInBytes * 2);
10456 SDValue Ptr = DAG.getNode(
10457 ISD::ADD, DL, VT, FrameAddr,
10458 DAG.getSignedConstant(Offset, DL, getPointerTy(DAG.getDataLayout())));
10459 FrameAddr =
10460 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
10461 }
10462 return FrameAddr;
10463}
10464
10465SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
10466 SelectionDAG &DAG) const {
10467 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10468 MachineFunction &MF = DAG.getMachineFunction();
10469 MachineFrameInfo &MFI = MF.getFrameInfo();
10470 MFI.setReturnAddressIsTaken(true);
10471 MVT XLenVT = Subtarget.getXLenVT();
10472 int XLenInBytes = Subtarget.getXLen() / 8;
10473
10474 EVT VT = Op.getValueType();
10475 SDLoc DL(Op);
10476 unsigned Depth = Op.getConstantOperandVal(0);
10477 if (Depth) {
10478 int Off = -XLenInBytes;
10479 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
10480 SDValue Offset = DAG.getSignedConstant(Off, DL, VT);
10481 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
10482 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
10483 MachinePointerInfo());
10484 }
10485
10486 // Return the value of the return address register, marking it an implicit
10487 // live-in.
10488 Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
10489 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
10490}
10491
10492SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
10493 SelectionDAG &DAG) const {
10494 SDLoc DL(Op);
10495 SDValue Lo = Op.getOperand(0);
10496 SDValue Hi = Op.getOperand(1);
10497 SDValue Shamt = Op.getOperand(2);
10498 EVT VT = Lo.getValueType();
10499 unsigned XLen = Subtarget.getXLen();
10500
10501 // With P extension, use SLX (FSHL) for the high part.
10502 if (Subtarget.hasStdExtP()) {
10503 // HiRes = fshl(Hi, Lo, Shamt) - correct when Shamt < XLen
10504 SDValue HiRes = DAG.getNode(ISD::FSHL, DL, VT, Hi, Lo, Shamt);
10505 // LoRes = Lo << Shamt - correct Lo when Shamt < XLen,
10506 // Mask shift amount to avoid UB when Shamt >= XLen.
10507 SDValue ShamtMasked =
10508 DAG.getNode(ISD::AND, DL, VT, Shamt, DAG.getConstant(XLen - 1, DL, VT));
10509 SDValue LoRes = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMasked);
10510
10511 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10512 // FIXME: We should use a select and let LowerSelect make the
10513 // optimizations.
10514 SDValue ShAmtExt =
10515 DAG.getNode(ISD::SHL, DL, VT, Shamt,
10516 DAG.getConstant(XLen - Log2_32(XLen) - 1, DL, VT));
10517 SDValue Mask = DAG.getNode(ISD::SRA, DL, VT, ShAmtExt,
10518 DAG.getConstant(XLen - 1, DL, VT));
10519
10520 // When Shamt >= XLen: HiRes = LoRes, LoRes = 0
10521 // HiRes = (HiRes & ~Mask) | (LoRes & Mask)
10522 SDValue HiMasked =
10523 DAG.getNode(ISD::AND, DL, VT, HiRes, DAG.getNOT(DL, Mask, VT));
10524 SDValue LoMasked = DAG.getNode(ISD::AND, DL, VT, LoRes, Mask);
10525 HiRes =
10526 DAG.getNode(ISD::OR, DL, VT, HiMasked, LoMasked, SDNodeFlags::Disjoint);
10527
10528 // LoRes = LoRes & ~Mask (clear when Shamt >= XLen)
10529 LoRes = DAG.getNode(ISD::AND, DL, VT, LoRes, DAG.getNOT(DL, Mask, VT));
10530
10531 return DAG.getMergeValues({LoRes, HiRes}, DL);
10532 }
10533
10534 // if Shamt-XLEN < 0: // Shamt < XLEN
10535 // Lo = Lo << Shamt
10536 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
10537 // else:
10538 // Lo = 0
10539 // Hi = Lo << (Shamt-XLEN)
10540
10541 SDValue Zero = DAG.getConstant(0, DL, VT);
10542 SDValue One = DAG.getConstant(1, DL, VT);
10543 SDValue MinusXLen = DAG.getSignedConstant(-(int)XLen, DL, VT);
10544 SDValue XLenMinus1 = DAG.getConstant(XLen - 1, DL, VT);
10545 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
10546 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
10547
10548 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
10549 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
10550 SDValue ShiftRightLo =
10551 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
10552 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
10553 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
10554 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
10555
10556 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
10557
10558 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
10559 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
10560
10561 SDValue Parts[2] = {Lo, Hi};
10562 return DAG.getMergeValues(Parts, DL);
10563}
10564
10565SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
10566 bool IsSRA) const {
10567 SDLoc DL(Op);
10568 SDValue Lo = Op.getOperand(0);
10569 SDValue Hi = Op.getOperand(1);
10570 SDValue Shamt = Op.getOperand(2);
10571 EVT VT = Lo.getValueType();
10572
10573 // With P extension, use NSRL/NSRA for RV32 or FSHR (SRX) for RV64.
10574 if (Subtarget.hasStdExtP()) {
10575 unsigned XLen = Subtarget.getXLen();
10576
10577 SDValue LoRes;
10578 if (Subtarget.is64Bit()) {
10579 // On RV64, use FSHR (SRX instruction) for the low part. We will need
10580 // to fix this later if ShAmt >= 64.
10581 LoRes = DAG.getNode(ISD::FSHR, DL, VT, Hi, Lo, Shamt);
10582 } else {
10583 // On RV32, use NSRL/NSRA for the low part.
10584 // NSRL/NSRA read 6 bits of shift amount, so they handle Shamt >= 32
10585 // correctly.
10586 LoRes = DAG.getNode(IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL, VT, Lo,
10587 Hi, Shamt);
10588 }
10589
10590 // Mask shift amount to avoid UB when Shamt >= XLen.
10591 SDValue ShamtMasked =
10592 DAG.getNode(ISD::AND, DL, VT, Shamt, DAG.getConstant(XLen - 1, DL, VT));
10593 SDValue HiRes =
10594 DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, DL, VT, Hi, ShamtMasked);
10595
10596 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10597 // FIXME: We should use a select and let LowerSelect make the
10598 // optimizations.
10599 SDValue ShAmtExt =
10600 DAG.getNode(ISD::SHL, DL, VT, Shamt,
10601 DAG.getConstant(XLen - Log2_32(XLen) - 1, DL, VT));
10602 SDValue Mask = DAG.getNode(ISD::SRA, DL, VT, ShAmtExt,
10603 DAG.getConstant(XLen - 1, DL, VT));
10604
10605 if (Subtarget.is64Bit()) {
10606 // On RV64, FSHR masks shift amount to 63. We need to replace LoRes
10607 // with HiRes when Shamt >= 64.
10608 // LoRes = (LoRes & ~Mask) | (HiRes & Mask)
10609 SDValue LoMasked =
10610 DAG.getNode(ISD::AND, DL, VT, LoRes, DAG.getNOT(DL, Mask, VT));
10611 SDValue HiMasked = DAG.getNode(ISD::AND, DL, VT, HiRes, Mask);
10612 LoRes = DAG.getNode(ISD::OR, DL, VT, LoMasked, HiMasked,
10613 SDNodeFlags::Disjoint);
10614 }
10615
10616 // If ShAmt >= XLen, we need to replace HiRes with 0 or sign bits.
10617 if (IsSRA) {
10618 // sra hi, hi, (mask & (XLen-1)) - shifts by XLen-1 when shamt >= XLen
10619 SDValue MaskAmt = DAG.getNode(ISD::AND, DL, VT, Mask,
10620 DAG.getConstant(XLen - 1, DL, VT));
10621 HiRes = DAG.getNode(ISD::SRA, DL, VT, HiRes, MaskAmt);
10622 } else {
10623 // andn hi, hi, mask - clears hi when shamt >= XLen
10624 HiRes = DAG.getNode(ISD::AND, DL, VT, HiRes, DAG.getNOT(DL, Mask, VT));
10625 }
10626
10627 return DAG.getMergeValues({LoRes, HiRes}, DL);
10628 }
10629
10630 // SRA expansion:
10631 // if Shamt-XLEN < 0: // Shamt < XLEN
10632 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10633 // Hi = Hi >>s Shamt
10634 // else:
10635 // Lo = Hi >>s (Shamt-XLEN);
10636 // Hi = Hi >>s (XLEN-1)
10637 //
10638 // SRL expansion:
10639 // if Shamt-XLEN < 0: // Shamt < XLEN
10640 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10641 // Hi = Hi >>u Shamt
10642 // else:
10643 // Lo = Hi >>u (Shamt-XLEN);
10644 // Hi = 0;
10645
10646 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
10647
10648 SDValue Zero = DAG.getConstant(0, DL, VT);
10649 SDValue One = DAG.getConstant(1, DL, VT);
10650 SDValue MinusXLen = DAG.getSignedConstant(-(int)Subtarget.getXLen(), DL, VT);
10651 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
10652 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
10653 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
10654
10655 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
10656 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
10657 SDValue ShiftLeftHi =
10658 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
10659 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
10660 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
10661 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
10662 SDValue HiFalse =
10663 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
10664
10665 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
10666
10667 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
10668 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
10669
10670 SDValue Parts[2] = {Lo, Hi};
10671 return DAG.getMergeValues(Parts, DL);
10672}
10673
10674// Lower splats of i1 types to SETCC. For each mask vector type, we have a
10675// legal equivalently-sized i8 type, so we can use that as a go-between.
10676SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
10677 SelectionDAG &DAG) const {
10678 SDLoc DL(Op);
10679 MVT VT = Op.getSimpleValueType();
10680 SDValue SplatVal = Op.getOperand(0);
10681 // All-zeros or all-ones splats are handled specially.
10682 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
10683 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
10684 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
10685 }
10686 if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
10687 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
10688 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
10689 }
10690 MVT InterVT = VT.changeVectorElementType(MVT::i8);
10691 SplatVal = DAG.getNode(ISD::AND, DL, SplatVal.getValueType(), SplatVal,
10692 DAG.getConstant(1, DL, SplatVal.getValueType()));
10693 SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
10694 SDValue Zero = DAG.getConstant(0, DL, InterVT);
10695 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
10696}
10697
10698// Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
10699// illegal (currently only vXi64 RV32).
10700// FIXME: We could also catch non-constant sign-extended i32 values and lower
10701// them to VMV_V_X_VL.
10702SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
10703 SelectionDAG &DAG) const {
10704 SDLoc DL(Op);
10705 MVT VecVT = Op.getSimpleValueType();
10706 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
10707 "Unexpected SPLAT_VECTOR_PARTS lowering");
10708
10709 assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
10710 SDValue Lo = Op.getOperand(0);
10711 SDValue Hi = Op.getOperand(1);
10712
10713 MVT ContainerVT = VecVT;
10714 if (VecVT.isFixedLengthVector())
10715 ContainerVT = getContainerForFixedLengthVector(VecVT);
10716
10717 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10718
10719 SDValue Res =
10720 splatPartsI64WithVL(DL, ContainerVT, SDValue(), Lo, Hi, VL, DAG);
10721
10722 if (VecVT.isFixedLengthVector())
10723 Res = convertFromScalableVector(VecVT, Res, DAG, Subtarget);
10724
10725 return Res;
10726}
10727
10728// Custom-lower extensions from mask vectors by using a vselect either with 1
10729// for zero/any-extension or -1 for sign-extension:
10730// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
10731// Note that any-extension is lowered identically to zero-extension.
10732SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
10733 int64_t ExtTrueVal) const {
10734 SDLoc DL(Op);
10735 MVT VecVT = Op.getSimpleValueType();
10736 SDValue Src = Op.getOperand(0);
10737 // Only custom-lower extensions from mask types
10738 assert(Src.getValueType().isVector() &&
10739 Src.getValueType().getVectorElementType() == MVT::i1);
10740
10741 if (VecVT.isScalableVector()) {
10742 SDValue SplatZero = DAG.getConstant(0, DL, VecVT);
10743 SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, VecVT);
10744 if (Src.getOpcode() == ISD::XOR &&
10745 ISD::isConstantSplatVectorAllOnes(Src.getOperand(1).getNode()))
10746 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src.getOperand(0), SplatZero,
10747 SplatTrueVal);
10748 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
10749 }
10750
10751 MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
10752 MVT I1ContainerVT =
10753 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
10754
10755 SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
10756
10757 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10758
10759 MVT XLenVT = Subtarget.getXLenVT();
10760 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
10761 SDValue SplatTrueVal = DAG.getSignedConstant(ExtTrueVal, DL, XLenVT);
10762
10763 if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
10764 SDValue Xor = Src.getOperand(0);
10765 if (Xor.getOpcode() == RISCVISD::VMXOR_VL) {
10766 SDValue ScalableOnes = Xor.getOperand(1);
10767 if (ScalableOnes.getOpcode() == ISD::INSERT_SUBVECTOR &&
10768 ScalableOnes.getOperand(0).isUndef() &&
10769 ISD::isConstantSplatVectorAllOnes(
10770 ScalableOnes.getOperand(1).getNode())) {
10771 CC = Xor.getOperand(0);
10772 std::swap(SplatZero, SplatTrueVal);
10773 }
10774 }
10775 }
10776
10777 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10778 DAG.getUNDEF(ContainerVT), SplatZero, VL);
10779 SplatTrueVal = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10780 DAG.getUNDEF(ContainerVT), SplatTrueVal, VL);
10781 SDValue Select =
10782 DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, CC, SplatTrueVal,
10783 SplatZero, DAG.getUNDEF(ContainerVT), VL);
10784
10785 return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
10786}
10787
10788// Custom-lower truncations from vectors to mask vectors by using a mask and a
10789// setcc operation:
10790// (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
10791SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
10792 SelectionDAG &DAG) const {
10793 SDLoc DL(Op);
10794 EVT MaskVT = Op.getValueType();
10795 // Only expect to custom-lower truncations to mask types
10796 assert(MaskVT.isVectorOf(MVT::i1) &&
10797 "Unexpected type for vector mask lowering");
10798 SDValue Src = Op.getOperand(0);
10799 MVT VecVT = Src.getSimpleValueType();
10800 // If this is a fixed vector, we need to convert it to a scalable vector.
10801 MVT ContainerVT = VecVT;
10802
10803 if (VecVT.isFixedLengthVector()) {
10804 ContainerVT = getContainerForFixedLengthVector(VecVT);
10805 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
10806 }
10807
10808 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
10809
10810 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
10811 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
10812
10813 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10814 DAG.getUNDEF(ContainerVT), SplatOne, VL);
10815 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
10816 DAG.getUNDEF(ContainerVT), SplatZero, VL);
10817
10818 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
10819 SDValue Trunc = DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne,
10820 DAG.getUNDEF(ContainerVT), Mask, VL);
10821 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT,
10822 {Trunc, SplatZero, DAG.getCondCode(ISD::SETNE),
10823 DAG.getUNDEF(MaskContainerVT), Mask, VL});
10824 if (MaskVT.isFixedLengthVector())
10825 Trunc = convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
10826 return Trunc;
10827}
10828
10829SDValue RISCVTargetLowering::lowerVectorTrunc(SDValue Op,
10830 SelectionDAG &DAG) const {
10831 unsigned Opc = Op.getOpcode();
10832 SDLoc DL(Op);
10833
10834 MVT VT = Op.getSimpleValueType();
10835 // Only custom-lower vector truncates
10836 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
10837
10838 // Truncates to mask types are handled differently
10839 if (VT.getVectorElementType() == MVT::i1)
10840 return lowerVectorMaskTrunc(Op, DAG);
10841
10842 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
10843 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
10844 // truncate by one power of two at a time.
10845 MVT DstEltVT = VT.getVectorElementType();
10846
10847 SDValue Src = Op.getOperand(0);
10848 MVT SrcVT = Src.getSimpleValueType();
10849 MVT SrcEltVT = SrcVT.getVectorElementType();
10850
10851 assert(DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) &&
10852 isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
10853 "Unexpected vector truncate lowering");
10854
10855 MVT ContainerVT = SrcVT;
10856 if (SrcVT.isFixedLengthVector()) {
10857 ContainerVT = getContainerForFixedLengthVector(SrcVT);
10858 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
10859 }
10860
10861 SDValue Result = Src;
10862 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
10863
10864 unsigned NewOpc;
10865 if (Opc == ISD::TRUNCATE_SSAT_S)
10866 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
10867 else if (Opc == ISD::TRUNCATE_USAT_U)
10868 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
10869 else
10870 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
10871
10872 do {
10873 SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
10874 MVT ResultVT = ContainerVT.changeVectorElementType(SrcEltVT);
10875 Result = DAG.getNode(NewOpc, DL, ResultVT, Result, Mask, VL);
10876 } while (SrcEltVT != DstEltVT);
10877
10878 if (SrcVT.isFixedLengthVector())
10879 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
10880
10881 return Result;
10882}
10883
10884SDValue
10885RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
10886 SelectionDAG &DAG) const {
10887 SDLoc DL(Op);
10888 SDValue Chain = Op.getOperand(0);
10889 SDValue Src = Op.getOperand(1);
10890 MVT VT = Op.getSimpleValueType();
10891 MVT SrcVT = Src.getSimpleValueType();
10892 MVT ContainerVT = VT;
10893 if (VT.isFixedLengthVector()) {
10894 MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
10895 ContainerVT =
10896 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
10897 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
10898 }
10899
10900 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
10901
10902 // RVV can only widen/truncate fp to types double/half the size as the source.
10903 if ((VT.getVectorElementType() == MVT::f64 &&
10904 (SrcVT.getVectorElementType() == MVT::f16 ||
10905 SrcVT.getVectorElementType() == MVT::bf16)) ||
10906 ((VT.getVectorElementType() == MVT::f16 ||
10907 VT.getVectorElementType() == MVT::bf16) &&
10908 SrcVT.getVectorElementType() == MVT::f64)) {
10909 // For double rounding, the intermediate rounding should be round-to-odd.
10910 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
10911 ? RISCVISD::STRICT_FP_EXTEND_VL
10912 : RISCVISD::STRICT_VFNCVT_ROD_VL;
10913 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
10914 Src = DAG.getNode(InterConvOpc, DL, DAG.getVTList(InterVT, MVT::Other),
10915 Chain, Src, Mask, VL);
10916 Chain = Src.getValue(1);
10917 }
10918
10919 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
10920 ? RISCVISD::STRICT_FP_EXTEND_VL
10921 : RISCVISD::STRICT_FP_ROUND_VL;
10922 SDValue Res = DAG.getNode(ConvOpc, DL, DAG.getVTList(ContainerVT, MVT::Other),
10923 Chain, Src, Mask, VL);
10924 if (VT.isFixedLengthVector()) {
10925 // StrictFP operations have two result values. Their lowered result should
10926 // have same result count.
10927 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
10928 Res = DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
10929 }
10930 return Res;
10931}
10932
10933SDValue
10934RISCVTargetLowering::lowerVectorFPExtendOrRound(SDValue Op,
10935 SelectionDAG &DAG) const {
10936 bool IsExtend = Op.getOpcode() == ISD::FP_EXTEND;
10937 // RVV can only do truncate fp to types half the size as the source. We
10938 // custom-lower f64->f16 rounds via RVV's round-to-odd float
10939 // conversion instruction.
10940 SDLoc DL(Op);
10941 MVT VT = Op.getSimpleValueType();
10942
10943 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
10944
10945 SDValue Src = Op.getOperand(0);
10946 MVT SrcVT = Src.getSimpleValueType();
10947
10948 bool IsDirectExtend =
10949 IsExtend && (VT.getVectorElementType() != MVT::f64 ||
10950 (SrcVT.getVectorElementType() != MVT::f16 &&
10951 SrcVT.getVectorElementType() != MVT::bf16));
10952 bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
10953 VT.getVectorElementType() != MVT::bf16) ||
10954 SrcVT.getVectorElementType() != MVT::f64);
10955
10956 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
10957
10958 // We have regular SD node patterns for direct non-VL extends.
10959 if (VT.isScalableVector() && IsDirectConv)
10960 return Op;
10961
10962 // Prepare any fixed-length vector operands.
10963 MVT ContainerVT = VT;
10964 if (VT.isFixedLengthVector()) {
10965 MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
10966 ContainerVT =
10967 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
10968 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
10969 }
10970
10971 auto [Mask, VL] = getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
10972
10973 unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
10974
10975 if (IsDirectConv) {
10976 Src = DAG.getNode(ConvOpc, DL, ContainerVT, Src, Mask, VL);
10977 if (VT.isFixedLengthVector())
10978 Src = convertFromScalableVector(VT, Src, DAG, Subtarget);
10979 return Src;
10980 }
10981
10982 unsigned InterConvOpc =
10983 IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
10984
10985 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
10986 SDValue IntermediateConv =
10987 DAG.getNode(InterConvOpc, DL, InterVT, Src, Mask, VL);
10988 SDValue Result =
10989 DAG.getNode(ConvOpc, DL, ContainerVT, IntermediateConv, Mask, VL);
10990 if (VT.isFixedLengthVector())
10991 return convertFromScalableVector(VT, Result, DAG, Subtarget);
10992 return Result;
10993}
10994
10995// Given a scalable vector type and an index into it, returns the type for the
10996// smallest subvector that the index fits in. This can be used to reduce LMUL
10997// for operations like vslidedown.
10998//
10999// E.g. With Zvl128b, index 3 in a nxv4i32 fits within the first nxv2i32.
11000static std::optional<MVT>
11001getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
11002 const RISCVSubtarget &Subtarget) {
11003 assert(VecVT.isScalableVector());
11004 const unsigned EltSize = VecVT.getScalarSizeInBits();
11005 const unsigned VectorBitsMin = Subtarget.getRealMinVLen();
11006 const unsigned MinVLMAX = VectorBitsMin / EltSize;
11007 MVT SmallerVT;
11008 if (MaxIdx < MinVLMAX)
11009 SmallerVT = RISCVTargetLowering::getM1VT(VecVT);
11010 else if (MaxIdx < MinVLMAX * 2)
11011 SmallerVT =
11012 RISCVTargetLowering::getM1VT(VecVT).getDoubleNumVectorElementsVT();
11013 else if (MaxIdx < MinVLMAX * 4)
11014 SmallerVT = RISCVTargetLowering::getM1VT(VecVT)
11015 .getDoubleNumVectorElementsVT()
11016 .getDoubleNumVectorElementsVT();
11017 if (!SmallerVT.isValid() || !VecVT.bitsGT(SmallerVT))
11018 return std::nullopt;
11019 return SmallerVT;
11020}
11021
11022// Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
11023// first position of a vector, and that vector is slid up to the insert index.
11024// By limiting the active vector length to index+1 and merging with the
11025// original vector (with an undisturbed tail policy for elements >= VL), we
11026// achieve the desired result of leaving all elements untouched except the one
11027// at VL-1, which is replaced with the desired value.
11028SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
11029 SelectionDAG &DAG) const {
11030 SDLoc DL(Op);
11031 MVT VecVT = Op.getSimpleValueType();
11032 MVT XLenVT = Subtarget.getXLenVT();
11033 SDValue Vec = Op.getOperand(0);
11034 SDValue Val = Op.getOperand(1);
11035 MVT ValVT = Val.getSimpleValueType();
11036 SDValue Idx = Op.getOperand(2);
11037
11038 if (VecVT.getVectorElementType() == MVT::i1) {
11039 // FIXME: For now we just promote to an i8 vector and insert into that,
11040 // but this is probably not optimal.
11041 MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
11042 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
11043 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
11044 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
11045 }
11046
11047 if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11048 (ValVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11049 // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
11050 MVT IntVT = VecVT.changeTypeToInteger();
11051 SDValue IntInsert = DAG.getNode(
11052 ISD::INSERT_VECTOR_ELT, DL, IntVT, DAG.getBitcast(IntVT, Vec),
11053 DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Val), Idx);
11054 return DAG.getBitcast(VecVT, IntInsert);
11055 }
11056
11057 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11058 auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
11059 if (!IdxC)
11060 return SDValue();
11061
11062 unsigned IdxVal = IdxC->getZExtValue();
11063 unsigned NumElts = VecVT.getVectorNumElements();
11064 MVT EltVT = VecVT.getVectorElementType();
11065
11066 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11067 unsigned HalfNumElts = NumElts / 2;
11068 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
11069 MVT HalfVT = Lo.getSimpleValueType();
11070 if (IdxVal < HalfNumElts) {
11071 SDValue NewLo =
11072 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, HalfVT, Lo, Val, Idx);
11073 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, NewLo, Hi);
11074 }
11075 SDValue NewHi =
11076 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, HalfVT, Hi, Val,
11077 DAG.getVectorIdxConstant(IdxVal - HalfNumElts, DL));
11078 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo, NewHi);
11079 }
11080
11081 Vec = DAG.getBitcast(XLenVT, Vec);
11082 SDValue ExtVal = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
11083
11084 // For 2-element vectors, BUILD_VECTOR is more efficient since it only needs
11085 // at most 2 instructions.
11086 if (NumElts == 2) {
11087 unsigned EltBits = EltVT.getSizeInBits();
11088 SDValue Elt0, Elt1;
11089 if (IdxVal == 0) {
11090 Elt0 = ExtVal;
11091 Elt1 = DAG.getNode(ISD::SRL, DL, XLenVT, Vec,
11092 DAG.getConstant(EltBits, DL, XLenVT));
11093 } else {
11094 Elt0 = Vec;
11095 Elt1 = ExtVal;
11096 }
11097 return DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, Elt0, Elt1);
11098 }
11099
11100 // For 4/8-element vectors, use MVM(or MERGE) instruction which does bitwise
11101 // select: rd = (~mask & rd) | (mask & rs1).
11102 // This generates: slli + lui/li + mvm
11103 if (NumElts == 4 || NumElts == 8) {
11104 unsigned EltBits = EltVT.getSizeInBits();
11105 unsigned ShiftAmt = IdxVal * EltBits;
11106 uint64_t PosMask = ((1ULL << EltBits) - 1) << ShiftAmt;
11107
11108 SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, XLenVT, ExtVal,
11109 DAG.getConstant(ShiftAmt, DL, XLenVT));
11110 SDValue Mask = DAG.getConstant(PosMask, DL, XLenVT);
11111 SDValue Result =
11112 DAG.getNode(RISCVISD::MERGE, DL, XLenVT, Mask, Vec, ShiftedVal);
11113 return DAG.getBitcast(VecVT, Result);
11114 }
11115
11116 return SDValue();
11117 }
11118
11119 MVT ContainerVT = VecVT;
11120 // If the operand is a fixed-length vector, convert to a scalable one.
11121 if (VecVT.isFixedLengthVector()) {
11122 ContainerVT = getContainerForFixedLengthVector(VecVT);
11123 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
11124 }
11125
11126 // If we know the index we're going to insert at, we can shrink Vec so that
11127 // we're performing the scalar inserts and slideup on a smaller LMUL.
11128 SDValue OrigVec = Vec;
11129 std::optional<unsigned> AlignedIdx;
11130 if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx)) {
11131 const unsigned OrigIdx = IdxC->getZExtValue();
11132 // Do we know an upper bound on LMUL?
11133 if (auto ShrunkVT = getSmallestVTForIndex(ContainerVT, OrigIdx,
11134 DL, DAG, Subtarget)) {
11135 ContainerVT = *ShrunkVT;
11136 AlignedIdx = 0;
11137 }
11138
11139 // If we're compiling for an exact VLEN value, we can always perform
11140 // the insert in m1 as we can determine the register corresponding to
11141 // the index in the register group.
11142 const MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
11143 if (auto VLEN = Subtarget.getRealVLen(); VLEN && ContainerVT.bitsGT(M1VT)) {
11144 EVT ElemVT = VecVT.getVectorElementType();
11145 unsigned ElemsPerVReg = *VLEN / ElemVT.getFixedSizeInBits();
11146 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11147 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11148 AlignedIdx = SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11149 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
11150 ContainerVT = M1VT;
11151 }
11152
11153 if (AlignedIdx)
11154 Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, *AlignedIdx);
11155 }
11156
11157 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
11158 // Even i64-element vectors on RV32 can be lowered without scalar
11159 // legalization if the most-significant 32 bits of the value are not affected
11160 // by the sign-extension of the lower 32 bits. This applies to i32 constants
11161 // and sign_extend of i32 values.
11162 if (!IsLegalInsert) {
11163 if (isa<ConstantSDNode>(Val)) {
11164 const auto *CVal = cast<ConstantSDNode>(Val);
11165 if (isInt<32>(CVal->getSExtValue())) {
11166 IsLegalInsert = true;
11167 Val = DAG.getSignedConstant(CVal->getSExtValue(), DL, MVT::i32);
11168 }
11169 } else if (Val.getOpcode() == ISD::SIGN_EXTEND &&
11170 Val.getOperand(0).getValueType() == MVT::i32) {
11171 IsLegalInsert = true;
11172 Val = Val.getOperand(0);
11173 }
11174 }
11175
11176 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11177
11178 SDValue ValInVec;
11179
11180 if (IsLegalInsert) {
11181 unsigned Opc =
11182 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
11183 if (isNullConstant(Idx)) {
11184 if (!VecVT.isFloatingPoint())
11185 Val = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Val);
11186 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
11187
11188 if (AlignedIdx)
11189 Vec = DAG.getInsertSubvector(DL, OrigVec, Vec, *AlignedIdx);
11190 if (!VecVT.isFixedLengthVector())
11191 return Vec;
11192 return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
11193 }
11194
11195 ValInVec = lowerScalarInsert(Val, VL, ContainerVT, DL, DAG, Subtarget);
11196 } else {
11197 // On RV32, i64-element vectors must be specially handled to place the
11198 // value at element 0, by using two vslide1down instructions in sequence on
11199 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
11200 // this.
11201 SDValue ValLo, ValHi;
11202 std::tie(ValLo, ValHi) = DAG.SplitScalar(Val, DL, MVT::i32, MVT::i32);
11203 MVT I32ContainerVT =
11204 MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
11205 SDValue I32Mask =
11206 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
11207 // Limit the active VL to two.
11208 SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
11209 // If the Idx is 0 we can insert directly into the vector.
11210 if (isNullConstant(Idx)) {
11211 // First slide in the lo value, then the hi in above it. We use slide1down
11212 // to avoid the register group overlap constraint of vslide1up.
11213 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
11214 Vec, Vec, ValLo, I32Mask, InsertI64VL);
11215 // If the source vector is undef don't pass along the tail elements from
11216 // the previous slide1down.
11217 SDValue Tail = Vec.isUndef() ? Vec : ValInVec;
11218 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
11219 Tail, ValInVec, ValHi, I32Mask, InsertI64VL);
11220 // Bitcast back to the right container type.
11221 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
11222
11223 if (AlignedIdx)
11224 ValInVec = DAG.getInsertSubvector(DL, OrigVec, ValInVec, *AlignedIdx);
11225 if (!VecVT.isFixedLengthVector())
11226 return ValInVec;
11227 return convertFromScalableVector(VecVT, ValInVec, DAG, Subtarget);
11228 }
11229
11230 // First slide in the lo value, then the hi in above it. We use slide1down
11231 // to avoid the register group overlap constraint of vslide1up.
11232 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
11233 DAG.getUNDEF(I32ContainerVT),
11234 DAG.getUNDEF(I32ContainerVT), ValLo,
11235 I32Mask, InsertI64VL);
11236 ValInVec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32ContainerVT,
11237 DAG.getUNDEF(I32ContainerVT), ValInVec, ValHi,
11238 I32Mask, InsertI64VL);
11239 // Bitcast back to the right container type.
11240 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
11241 }
11242
11243 // Now that the value is in a vector, slide it into position.
11244 SDValue InsertVL =
11245 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
11246
11247 // Use tail agnostic policy if Idx is the last index of Vec.
11248 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
11249 if (VecVT.isFixedLengthVector() && isa<ConstantSDNode>(Idx) &&
11250 Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
11251 Policy = RISCVVType::TAIL_AGNOSTIC;
11252 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, ValInVec,
11253 Idx, Mask, InsertVL, Policy);
11254
11255 if (AlignedIdx)
11256 Slideup = DAG.getInsertSubvector(DL, OrigVec, Slideup, *AlignedIdx);
11257 if (!VecVT.isFixedLengthVector())
11258 return Slideup;
11259 return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
11260}
11261
11262// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
11263// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
11264// types this is done using VMV_X_S to allow us to glean information about the
11265// sign bits of the result.
11266SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
11267 SelectionDAG &DAG) const {
11268 SDLoc DL(Op);
11269 SDValue Idx = Op.getOperand(1);
11270 SDValue Vec = Op.getOperand(0);
11271 EVT EltVT = Op.getValueType();
11272 MVT VecVT = Vec.getSimpleValueType();
11273 MVT XLenVT = Subtarget.getXLenVT();
11274
11275 if (VecVT.getVectorElementType() == MVT::i1) {
11276 // Use vfirst.m to extract the first bit.
11277 if (isNullConstant(Idx)) {
11278 MVT ContainerVT = VecVT;
11279 if (VecVT.isFixedLengthVector()) {
11280 ContainerVT = getContainerForFixedLengthVector(VecVT);
11281 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
11282 }
11283 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11284 SDValue Vfirst =
11285 DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Vec, Mask, VL);
11286 SDValue Res = DAG.getSetCC(DL, XLenVT, Vfirst,
11287 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
11288 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
11289 }
11290 if (VecVT.isFixedLengthVector()) {
11291 unsigned NumElts = VecVT.getVectorNumElements();
11292 if (NumElts >= 8) {
11293 MVT WideEltVT;
11294 unsigned WidenVecLen;
11295 SDValue ExtractElementIdx;
11296 SDValue ExtractBitIdx;
11297 unsigned MaxEEW = Subtarget.getELen();
11298 MVT LargestEltVT = MVT::getIntegerVT(
11299 std::min(MaxEEW, unsigned(XLenVT.getSizeInBits())));
11300 if (NumElts <= LargestEltVT.getSizeInBits()) {
11301 assert(isPowerOf2_32(NumElts) &&
11302 "the number of elements should be power of 2");
11303 WideEltVT = MVT::getIntegerVT(NumElts);
11304 WidenVecLen = 1;
11305 ExtractElementIdx = DAG.getConstant(0, DL, XLenVT);
11306 ExtractBitIdx = Idx;
11307 } else {
11308 WideEltVT = LargestEltVT;
11309 WidenVecLen = NumElts / WideEltVT.getSizeInBits();
11310 // extract element index = index / element width
11311 ExtractElementIdx = DAG.getNode(
11312 ISD::SRL, DL, XLenVT, Idx,
11313 DAG.getConstant(Log2_64(WideEltVT.getSizeInBits()), DL, XLenVT));
11314 // mask bit index = index % element width
11315 ExtractBitIdx = DAG.getNode(
11316 ISD::AND, DL, XLenVT, Idx,
11317 DAG.getConstant(WideEltVT.getSizeInBits() - 1, DL, XLenVT));
11318 }
11319 MVT WideVT = MVT::getVectorVT(WideEltVT, WidenVecLen);
11320 Vec = DAG.getNode(ISD::BITCAST, DL, WideVT, Vec);
11321 SDValue ExtractElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT,
11322 Vec, ExtractElementIdx);
11323 // Extract the bit from GPR.
11324 SDValue ShiftRight =
11325 DAG.getNode(ISD::SRL, DL, XLenVT, ExtractElt, ExtractBitIdx);
11326 SDValue Res = DAG.getNode(ISD::AND, DL, XLenVT, ShiftRight,
11327 DAG.getConstant(1, DL, XLenVT));
11328 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Res);
11329 }
11330 }
11331 // Otherwise, promote to an i8 vector and extract from that.
11332 MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
11333 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
11334 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
11335 }
11336
11337 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11338 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11339 // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
11340 MVT IntVT = VecVT.changeTypeToInteger();
11341 SDValue IntVec = DAG.getBitcast(IntVT, Vec);
11342 SDValue IntExtract =
11343 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, XLenVT, IntVec, Idx);
11344 return DAG.getNode(RISCVISD::FMV_H_X, DL, EltVT, IntExtract);
11345 }
11346
11347 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11348 if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
11349 VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
11350 return SDValue();
11351
11352 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11353 auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
11354 if (!IdxC)
11355 return SDValue();
11356 unsigned IdxVal = IdxC->getZExtValue();
11357 unsigned HalfNumElts = VecVT.getVectorNumElements() / 2;
11358 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
11359 if (IdxVal < HalfNumElts)
11360 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Lo, Idx);
11361 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Hi,
11362 DAG.getVectorIdxConstant(IdxVal - HalfNumElts, DL));
11363 }
11364
11365 SDValue Extracted = DAG.getBitcast(XLenVT, Vec);
11366 unsigned ElemWidth = VecVT.getVectorElementType().getSizeInBits();
11367 SDValue Shamt = DAG.getNode(ISD::MUL, DL, XLenVT, Idx,
11368 DAG.getConstant(ElemWidth, DL, XLenVT));
11369 return DAG.getNode(ISD::SRL, DL, XLenVT, Extracted, Shamt);
11370 }
11371
11372 // If this is a fixed vector, we need to convert it to a scalable vector.
11373 MVT ContainerVT = VecVT;
11374 if (VecVT.isFixedLengthVector()) {
11375 ContainerVT = getContainerForFixedLengthVector(VecVT);
11376 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
11377 }
11378
11379 // If we're compiling for an exact VLEN value and we have a known
11380 // constant index, we can always perform the extract in m1 (or
11381 // smaller) as we can determine the register corresponding to
11382 // the index in the register group.
11383 const auto VLen = Subtarget.getRealVLen();
11384 if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx);
11385 IdxC && VLen && VecVT.getSizeInBits().getKnownMinValue() > *VLen) {
11386 MVT M1VT = RISCVTargetLowering::getM1VT(ContainerVT);
11387 unsigned OrigIdx = IdxC->getZExtValue();
11388 EVT ElemVT = VecVT.getVectorElementType();
11389 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
11390 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11391 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11392 unsigned ExtractIdx =
11393 SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11394 Vec = DAG.getExtractSubvector(DL, M1VT, Vec, ExtractIdx);
11395 Idx = DAG.getVectorIdxConstant(RemIdx, DL);
11396 ContainerVT = M1VT;
11397 }
11398
11399 // Reduce the LMUL of our slidedown and vmv.x.s to the smallest LMUL which
11400 // contains our index.
11401 std::optional<uint64_t> MaxIdx;
11402 if (VecVT.isFixedLengthVector())
11403 MaxIdx = VecVT.getVectorNumElements() - 1;
11404 if (auto *IdxC = dyn_cast<ConstantSDNode>(Idx))
11405 MaxIdx = IdxC->getZExtValue();
11406 if (MaxIdx) {
11407 if (auto SmallerVT =
11408 getSmallestVTForIndex(ContainerVT, *MaxIdx, DL, DAG, Subtarget)) {
11409 ContainerVT = *SmallerVT;
11410 Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, 0);
11411 }
11412 }
11413
11414 // If after narrowing, the required slide is still greater than LMUL2,
11415 // fallback to generic expansion and go through the stack. This is done
11416 // for a subtle reason: extracting *all* elements out of a vector is
11417 // widely expected to be linear in vector size, but because vslidedown
11418 // is linear in LMUL, performing N extracts using vslidedown becomes
11419 // O(n^2) / (VLEN/ETYPE) work. On the surface, going through the stack
11420 // seems to have the same problem (the store is linear in LMUL), but the
11421 // generic expansion *memoizes* the store, and thus for many extracts of
11422 // the same vector we end up with one store and a bunch of loads.
11423 // TODO: We don't have the same code for insert_vector_elt because we
11424 // have BUILD_VECTOR and handle the degenerate case there. Should we
11425 // consider adding an inverse BUILD_VECTOR node?
11426 MVT LMUL2VT =
11427 RISCVTargetLowering::getM1VT(ContainerVT).getDoubleNumVectorElementsVT();
11428 if (ContainerVT.bitsGT(LMUL2VT) && VecVT.isFixedLengthVector())
11429 return SDValue();
11430
11431 // If the index is 0, the vector is already in the right position.
11432 if (!isNullConstant(Idx)) {
11433 // Use a VL of 1 to avoid processing more elements than we need.
11434 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
11435 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
11436 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
11437 }
11438
11439 if (!EltVT.isInteger()) {
11440 // Floating-point extracts are handled in TableGen.
11441 return DAG.getExtractVectorElt(DL, EltVT, Vec, 0);
11442 }
11443
11444 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
11445 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
11446}
11447
11448// Some RVV intrinsics may claim that they want an integer operand to be
11449// promoted or expanded.
11450static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
11451 const RISCVSubtarget &Subtarget) {
11452 assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
11453 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
11454 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
11455 "Unexpected opcode");
11456
11457 if (!Subtarget.hasVInstructions())
11458 return SDValue();
11459
11460 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11461 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11462 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
11463
11464 SDLoc DL(Op);
11465
11466 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11467 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
11468 if (!II || !II->hasScalarOperand())
11469 return SDValue();
11470
11471 unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
11472 assert(SplatOp < Op.getNumOperands());
11473
11474 SmallVector<SDValue, 8> Operands(Op->ops());
11475 SDValue &ScalarOp = Operands[SplatOp];
11476 MVT OpVT = ScalarOp.getSimpleValueType();
11477 MVT XLenVT = Subtarget.getXLenVT();
11478
11479 // If this isn't a scalar, or its type is XLenVT we're done.
11480 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11481 return SDValue();
11482
11483 // Simplest case is that the operand needs to be promoted to XLenVT.
11484 if (OpVT.bitsLT(XLenVT)) {
11485 // If the operand is a constant, sign extend to increase our chances
11486 // of being able to use a .vi instruction. ANY_EXTEND would become a
11487 // a zero extend and the simm5 check in isel would fail.
11488 // FIXME: Should we ignore the upper bits in isel instead?
11489 unsigned ExtOpc =
11490 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11491 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
11492 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
11493 }
11494
11495 // Use the previous operand to get the vXi64 VT. The result might be a mask
11496 // VT for compares. Using the previous operand assumes that the previous
11497 // operand will never have a smaller element size than a scalar operand and
11498 // that a widening operation never uses SEW=64.
11499 // NOTE: If this fails the below assert, we can probably just find the
11500 // element count from any operand or result and use it to construct the VT.
11501 assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
11502 MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
11503
11504 // The more complex case is when the scalar is larger than XLenVT.
11505 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
11506 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
11507
11508 // If this is a sign-extended 32-bit value, we can truncate it and rely on the
11509 // instruction to sign-extend since SEW>XLEN.
11510 if (DAG.ComputeNumSignBits(ScalarOp) > 32) {
11511 ScalarOp = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ScalarOp);
11512 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
11513 }
11514
11515 switch (IntNo) {
11516 case Intrinsic::riscv_vslide1up:
11517 case Intrinsic::riscv_vslide1down:
11518 case Intrinsic::riscv_vslide1up_mask:
11519 case Intrinsic::riscv_vslide1down_mask: {
11520 // We need to special case these when the scalar is larger than XLen.
11521 unsigned NumOps = Op.getNumOperands();
11522 bool IsMasked = NumOps == 7;
11523
11524 // Convert the vector source to the equivalent nxvXi32 vector.
11525 MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
11526 SDValue Vec = DAG.getBitcast(I32VT, Operands[2]);
11527 SDValue ScalarLo, ScalarHi;
11528 std::tie(ScalarLo, ScalarHi) =
11529 DAG.SplitScalar(ScalarOp, DL, MVT::i32, MVT::i32);
11530
11531 // Double the VL since we halved SEW.
11532 SDValue AVL = getVLOperand(Op);
11533 SDValue I32VL;
11534
11535 // Optimize for constant AVL
11536 if (isa<ConstantSDNode>(AVL)) {
11537 const auto [MinVLMAX, MaxVLMAX] =
11538 RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget);
11539
11540 uint64_t AVLInt = AVL->getAsZExtVal();
11541 if (AVLInt <= MinVLMAX) {
11542 I32VL = DAG.getConstant(2 * AVLInt, DL, XLenVT);
11543 } else if (AVLInt >= 2 * MaxVLMAX) {
11544 // Just set vl to VLMAX in this situation
11545 I32VL = DAG.getRegister(RISCV::X0, XLenVT);
11546 } else {
11547 // For AVL between (MinVLMAX, 2 * MaxVLMAX), the actual working vl
11548 // is related to the hardware implementation.
11549 // So let the following code handle
11550 }
11551 }
11552 if (!I32VL) {
11553 RISCVVType::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
11554 SDValue LMUL = DAG.getConstant(Lmul, DL, XLenVT);
11555 unsigned Sew = RISCVVType::encodeSEW(VT.getScalarSizeInBits());
11556 SDValue SEW = DAG.getConstant(Sew, DL, XLenVT);
11557 SDValue SETVL =
11558 DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, MVT::i32);
11559 // Using vsetvli instruction to get actually used length which related to
11560 // the hardware implementation
11561 SDValue VL = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, SETVL, AVL,
11562 SEW, LMUL);
11563 I32VL =
11564 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
11565 }
11566
11567 SDValue I32Mask = getAllOnesMask(I32VT, I32VL, DL, DAG);
11568
11569 // Shift the two scalar parts in using SEW=32 slide1up/slide1down
11570 // instructions.
11571 SDValue Passthru;
11572 if (IsMasked)
11573 Passthru = DAG.getUNDEF(I32VT);
11574 else
11575 Passthru = DAG.getBitcast(I32VT, Operands[1]);
11576
11577 if (IntNo == Intrinsic::riscv_vslide1up ||
11578 IntNo == Intrinsic::riscv_vslide1up_mask) {
11579 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
11580 ScalarHi, I32Mask, I32VL);
11581 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Passthru, Vec,
11582 ScalarLo, I32Mask, I32VL);
11583 } else {
11584 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
11585 ScalarLo, I32Mask, I32VL);
11586 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Passthru, Vec,
11587 ScalarHi, I32Mask, I32VL);
11588 }
11589
11590 // Convert back to nxvXi64.
11591 Vec = DAG.getBitcast(VT, Vec);
11592
11593 if (!IsMasked)
11594 return Vec;
11595 // Apply mask after the operation.
11596 SDValue Mask = Operands[NumOps - 3];
11597 SDValue MaskedOff = Operands[1];
11598 // Assume Policy operand is the last operand.
11599 uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
11600 // We don't need to select maskedoff if it's undef.
11601 if (MaskedOff.isUndef())
11602 return Vec;
11603 // TAMU
11604 if (Policy == RISCVVType::TAIL_AGNOSTIC)
11605 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
11606 DAG.getUNDEF(VT), AVL);
11607 // TUMA or TUMU: Currently we always emit tumu policy regardless of tuma.
11608 // It's fine because vmerge does not care mask policy.
11609 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, Mask, Vec, MaskedOff,
11610 MaskedOff, AVL);
11611 }
11612 }
11613
11614 // We need to convert the scalar to a splat vector.
11615 SDValue VL = getVLOperand(Op);
11616 assert(VL.getValueType() == XLenVT);
11617 ScalarOp = splatSplitI64WithVL(DL, VT, SDValue(), ScalarOp, VL, DAG);
11618 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
11619}
11620
11621// Lower the llvm.get.vector.length intrinsic to vsetvli. We only support
11622// scalable vector llvm.get.vector.length for now.
11623//
11624// We need to convert from a scalable VF to a vsetvli with VLMax equal to
11625// (vscale * VF). The vscale and VF are independent of element width. We use
11626// SEW=8 for the vsetvli because it is the only element width that supports all
11627// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
11628// (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
11629// InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
11630// SEW and LMUL are better for the surrounding vector instructions.
11631static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
11632 const RISCVSubtarget &Subtarget) {
11633 MVT XLenVT = Subtarget.getXLenVT();
11634
11635 // The smallest LMUL is only valid for the smallest element width.
11636 const unsigned ElementWidth = 8;
11637
11638 // Determine the VF that corresponds to LMUL 1 for ElementWidth.
11639 unsigned LMul1VF = RISCV::RVVBitsPerBlock / ElementWidth;
11640 // We don't support VF==1 with ELEN==32.
11641 [[maybe_unused]] unsigned MinVF =
11642 RISCV::RVVBitsPerBlock / Subtarget.getELen();
11643
11644 [[maybe_unused]] unsigned VF = N->getConstantOperandVal(2);
11645 assert(VF >= MinVF && VF <= (LMul1VF * 8) && isPowerOf2_32(VF) &&
11646 "Unexpected VF");
11647
11648 bool Fractional = VF < LMul1VF;
11649 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
11650 unsigned VLMUL = (unsigned)RISCVVType::encodeLMUL(LMulVal, Fractional);
11651 unsigned VSEW = RISCVVType::encodeSEW(ElementWidth);
11652
11653 SDLoc DL(N);
11654
11655 SDValue LMul = DAG.getTargetConstant(VLMUL, DL, XLenVT);
11656 SDValue Sew = DAG.getTargetConstant(VSEW, DL, XLenVT);
11657
11658 SDValue AVL = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, N->getOperand(1));
11659
11660 SDValue ID = DAG.getTargetConstant(Intrinsic::riscv_vsetvli, DL, XLenVT);
11661 SDValue Res =
11662 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, XLenVT, ID, AVL, Sew, LMul);
11663 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), Res);
11664}
11665
11666static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
11667 const RISCVSubtarget &Subtarget) {
11668 SDValue Op0 = Op.getOperand(0);
11669 MVT OpVT = Op0.getSimpleValueType();
11670 MVT ContainerVT = OpVT;
11671 if (OpVT.isFixedLengthVector()) {
11672 ContainerVT = getContainerForFixedLengthVector(OpVT, Subtarget);
11673 Op0 = convertToScalableVector(ContainerVT, Op0, DAG, Subtarget);
11674 }
11675 MVT XLenVT = Subtarget.getXLenVT();
11676 SDLoc DL(Op);
11677 auto [Mask, VL] = getDefaultVLOps(OpVT, ContainerVT, DL, DAG, Subtarget);
11678 SDValue Res = DAG.getNode(RISCVISD::VFIRST_VL, DL, XLenVT, Op0, Mask, VL);
11679 if (Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
11680 return Res;
11681
11682 // Convert -1 to VL.
11683 SDValue Setcc =
11684 DAG.getSetCC(DL, XLenVT, Res, DAG.getConstant(0, DL, XLenVT), ISD::SETLT);
11685 VL = DAG.getElementCount(DL, XLenVT, OpVT.getVectorElementCount());
11686 return DAG.getSelect(DL, XLenVT, Setcc, VL, Res);
11687}
11688
11689static inline void promoteVCIXScalar(SDValue Op,
11690 MutableArrayRef<SDValue> Operands,
11691 SelectionDAG &DAG) {
11692 const RISCVSubtarget &Subtarget =
11693 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11694
11695 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11696 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11697 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
11698 SDLoc DL(Op);
11699
11700 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11701 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
11702 if (!II || !II->hasScalarOperand())
11703 return;
11704
11705 unsigned SplatOp = II->ScalarOperand + 1;
11706 assert(SplatOp < Op.getNumOperands());
11707
11708 SDValue &ScalarOp = Operands[SplatOp];
11709 MVT OpVT = ScalarOp.getSimpleValueType();
11710 MVT XLenVT = Subtarget.getXLenVT();
11711
11712 // The code below is partially copied from lowerVectorIntrinsicScalars.
11713 // If this isn't a scalar, or its type is XLenVT we're done.
11714 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11715 return;
11716
11717 // Manually emit promote operation for scalar operation.
11718 if (OpVT.bitsLT(XLenVT)) {
11719 unsigned ExtOpc =
11720 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11721 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
11722 }
11723}
11724
11725static void processVCIXOperands(SDValue OrigOp,
11726 MutableArrayRef<SDValue> Operands,
11727 SelectionDAG &DAG) {
11728 promoteVCIXScalar(OrigOp, Operands, DAG);
11729 const RISCVSubtarget &Subtarget =
11730 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11731 for (SDValue &V : Operands) {
11732 EVT ValType = V.getValueType();
11733 if (ValType.isVector() && ValType.isFloatingPoint()) {
11734 MVT InterimIVT =
11735 MVT::getVectorVT(MVT::getIntegerVT(ValType.getScalarSizeInBits()),
11736 ValType.getVectorElementCount());
11737 V = DAG.getBitcast(InterimIVT, V);
11738 }
11739 if (ValType.isFixedLengthVector()) {
11740 MVT OpContainerVT =
11741 getContainerForFixedLengthVector(V.getSimpleValueType(), Subtarget);
11742 V = convertToScalableVector(OpContainerVT, V, DAG, Subtarget);
11743 }
11744 }
11745}
11746
11747// LMUL * VLEN should be greater than or equal to EGS * SEW
11748static inline bool isValidEGW(int EGS, EVT VT,
11749 const RISCVSubtarget &Subtarget) {
11750 return (Subtarget.getRealMinVLen() *
11751 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
11752 EGS * VT.getScalarSizeInBits();
11753}
11754
11755SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11756 SelectionDAG &DAG) const {
11757 unsigned IntNo = Op.getConstantOperandVal(0);
11758 SDLoc DL(Op);
11759 MVT XLenVT = Subtarget.getXLenVT();
11760
11761 switch (IntNo) {
11762 default:
11763 break; // Don't custom lower most intrinsics.
11764 case Intrinsic::riscv_tuple_insert: {
11765 SDValue Vec = Op.getOperand(1);
11766 SDValue SubVec = Op.getOperand(2);
11767 SDValue Index = Op.getOperand(3);
11768
11769 return DAG.getNode(RISCVISD::TUPLE_INSERT, DL, Op.getValueType(), Vec,
11770 SubVec, Index);
11771 }
11772 case Intrinsic::riscv_tuple_extract: {
11773 SDValue Vec = Op.getOperand(1);
11774 SDValue Index = Op.getOperand(2);
11775
11776 return DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, Op.getValueType(), Vec,
11777 Index);
11778 }
11779 case Intrinsic::thread_pointer: {
11780 EVT PtrVT = getPointerTy(DAG.getDataLayout());
11781 return DAG.getRegister(RISCV::X4, PtrVT);
11782 }
11783 case Intrinsic::riscv_pas:
11784 case Intrinsic::riscv_psa:
11785 case Intrinsic::riscv_psas:
11786 case Intrinsic::riscv_pssa:
11787 case Intrinsic::riscv_paas:
11788 case Intrinsic::riscv_pasa: {
11789 // v2i32 has no paired instruction on RV32; split into a pair of i32 ops
11790 // with cross-lane operands. The exchange shape is: even result uses
11791 // (S1[0], S2[1]); odd result uses (S1[1], S2[0]).
11792 if (Subtarget.is64Bit() || Op.getSimpleValueType() != MVT::v2i32)
11793 break;
11794
11795 unsigned EvenOpc, OddOpc;
11796 switch (IntNo) {
11797 case Intrinsic::riscv_pas:
11798 EvenOpc = ISD::SUB;
11799 OddOpc = ISD::ADD;
11800 break;
11801 case Intrinsic::riscv_psa:
11802 EvenOpc = ISD::ADD;
11803 OddOpc = ISD::SUB;
11804 break;
11805 case Intrinsic::riscv_psas:
11806 EvenOpc = ISD::SSUBSAT;
11807 OddOpc = ISD::SADDSAT;
11808 break;
11809 case Intrinsic::riscv_pssa:
11810 EvenOpc = ISD::SADDSAT;
11811 OddOpc = ISD::SSUBSAT;
11812 break;
11813 case Intrinsic::riscv_paas:
11814 EvenOpc = RISCVISD::ASUB;
11815 OddOpc = ISD::AVGFLOORS;
11816 break;
11817 case Intrinsic::riscv_pasa:
11818 EvenOpc = ISD::AVGFLOORS;
11819 OddOpc = RISCVISD::ASUB;
11820 break;
11821 default:
11822 llvm_unreachable("Unexpected exchanged add/sub intrinsic");
11823 }
11824
11825 SDValue S1 = Op.getOperand(1);
11826 SDValue S2 = Op.getOperand(2);
11827 SDValue S1Even = DAG.getExtractVectorElt(DL, MVT::i32, S1, 0);
11828 SDValue S1Odd = DAG.getExtractVectorElt(DL, MVT::i32, S1, 1);
11829 SDValue S2Even = DAG.getExtractVectorElt(DL, MVT::i32, S2, 0);
11830 SDValue S2Odd = DAG.getExtractVectorElt(DL, MVT::i32, S2, 1);
11831
11832 SDValue REven = DAG.getNode(EvenOpc, DL, MVT::i32, S1Even, S2Odd);
11833 SDValue ROdd = DAG.getNode(OddOpc, DL, MVT::i32, S1Odd, S2Even);
11834 return DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2i32, REven, ROdd);
11835 }
11836 case Intrinsic::riscv_orc_b:
11837 case Intrinsic::riscv_brev8:
11838 case Intrinsic::riscv_sha256sig0:
11839 case Intrinsic::riscv_sha256sig1:
11840 case Intrinsic::riscv_sha256sum0:
11841 case Intrinsic::riscv_sha256sum1:
11842 case Intrinsic::riscv_sm3p0:
11843 case Intrinsic::riscv_sm3p1: {
11844 unsigned Opc;
11845 switch (IntNo) {
11846 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
11847 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
11848 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
11849 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
11850 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
11851 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
11852 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
11853 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
11854 }
11855
11856 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
11857 }
11858 case Intrinsic::riscv_sm4ks:
11859 case Intrinsic::riscv_sm4ed: {
11860 unsigned Opc =
11861 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
11862
11863 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2),
11864 Op.getOperand(3));
11865 }
11866 case Intrinsic::riscv_zip:
11867 case Intrinsic::riscv_unzip: {
11868 unsigned Opc =
11869 IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
11870 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1));
11871 }
11872 case Intrinsic::riscv_mopr:
11873 return DAG.getNode(RISCVISD::MOP_R, DL, XLenVT, Op.getOperand(1),
11874 Op.getOperand(2));
11875
11876 case Intrinsic::riscv_moprr: {
11877 return DAG.getNode(RISCVISD::MOP_RR, DL, XLenVT, Op.getOperand(1),
11878 Op.getOperand(2), Op.getOperand(3));
11879 }
11880 case Intrinsic::riscv_clmulh:
11881 case Intrinsic::riscv_clmulr: {
11882 unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
11883 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
11884 }
11885 case Intrinsic::riscv_paadd:
11886 case Intrinsic::riscv_paaddu:
11887 case Intrinsic::riscv_pasub:
11888 case Intrinsic::riscv_pasubu:
11889 case Intrinsic::riscv_pabd:
11890 case Intrinsic::riscv_pabdu:
11891 case Intrinsic::riscv_psabs: {
11892 unsigned Opc;
11893 switch (IntNo) {
11894 case Intrinsic::riscv_paadd:
11895 Opc = ISD::AVGFLOORS;
11896 break;
11897 case Intrinsic::riscv_paaddu:
11898 Opc = ISD::AVGFLOORU;
11899 break;
11900 case Intrinsic::riscv_pasub:
11901 Opc = RISCVISD::ASUB;
11902 break;
11903 case Intrinsic::riscv_pasubu:
11904 Opc = RISCVISD::ASUBU;
11905 break;
11906 case Intrinsic::riscv_pabd:
11907 Opc = ISD::ABDS;
11908 break;
11909 case Intrinsic::riscv_pabdu:
11910 Opc = ISD::ABDU;
11911 break;
11912 case Intrinsic::riscv_psabs:
11913 Opc = RISCVISD::PSABS;
11914 break;
11915 }
11916
11917 if (IntNo == Intrinsic::riscv_psabs)
11918 return DAG.getNode(Opc, DL, Op.getValueType(), Op.getOperand(1));
11919
11920 return DAG.getNode(Opc, DL, Op.getValueType(), Op.getOperand(1),
11921 Op.getOperand(2));
11922 }
11923 case Intrinsic::riscv_pabdsumu:
11924 case Intrinsic::riscv_pabdsumau: {
11925 // On RV32 an i32-result absolute difference sum over a 64-bit (GPRPair)
11926 // source has no paired instruction. Split into two v4i8 halves: reduce the
11927 // low half (folding in rd when accumulating), then accumulate the high half
11928 // into that partial sum.
11929 SDValue Rs1 = Op.getOperand(Op.getNumOperands() - 2);
11930 SDValue Rs2 = Op.getOperand(Op.getNumOperands() - 1);
11931 if (Subtarget.is64Bit() || Rs1.getSimpleValueType() != MVT::v8i8)
11932 return SDValue();
11933 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
11934 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
11935 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
11936 SDValue AbdsumuId =
11937 DAG.getTargetConstant(Intrinsic::riscv_pabdsumu, DL, MVT::i32);
11938 SDValue AbdsumauId =
11939 DAG.getTargetConstant(Intrinsic::riscv_pabdsumau, DL, MVT::i32);
11940 SDValue Lo = IsAcc ? DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
11941 AbdsumauId, Op.getOperand(1), Rs1Lo, Rs2Lo)
11942 : DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
11943 AbdsumuId, Rs1Lo, Rs2Lo);
11944 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32, AbdsumauId, Lo,
11945 Rs1Hi, Rs2Hi);
11946 }
11947 case Intrinsic::riscv_pmerge: {
11948 EVT VT = Op.getValueType();
11949 auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
11950 EVT ResultVT) {
11951 MVT IntVT = MVT::getIntegerVT(ResultVT.getSizeInBits());
11952 SDValue Res =
11953 DAG.getNode(RISCVISD::MERGE, DL, IntVT, DAG.getBitcast(IntVT, Mask),
11954 DAG.getBitcast(IntVT, Rs1), DAG.getBitcast(IntVT, Rs2));
11955 return DAG.getBitcast(ResultVT, Res);
11956 };
11957
11958 // 64-bit packed types on RV32: split into two 32-bit halves. v2i32 has no
11959 // legal 32-bit vector half, so bitcast it to v4i16 (same 64 bits) first;
11960 // the merge result is identical.
11961 if (!Subtarget.is64Bit() &&
11962 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32)) {
11963 EVT WorkVT = VT == MVT::v2i32 ? EVT(MVT::v4i16) : VT;
11964 SDValue Rs1 = DAG.getBitcast(WorkVT, Op.getOperand(1));
11965 SDValue Rs2 = DAG.getBitcast(WorkVT, Op.getOperand(2));
11966 SDValue Mask = DAG.getBitcast(WorkVT, Op.getOperand(3));
11967 MVT HalfVT = WorkVT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
11968 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL, HalfVT, HalfVT);
11969 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL, HalfVT, HalfVT);
11970 auto [MaskLo, MaskHi] = DAG.SplitVector(Mask, DL, HalfVT, HalfVT);
11971 SDValue ResLo = buildMerge(Rs1Lo, Rs2Lo, MaskLo, HalfVT);
11972 SDValue ResHi = buildMerge(Rs1Hi, Rs2Hi, MaskHi, HalfVT);
11973 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, WorkVT, ResLo, ResHi);
11974 return DAG.getBitcast(VT, Res);
11975 }
11976
11977 return buildMerge(Op.getOperand(1), Op.getOperand(2), Op.getOperand(3), VT);
11978 }
11979 case Intrinsic::experimental_get_vector_length:
11980 return lowerGetVectorLength(Op.getNode(), DAG, Subtarget);
11981 case Intrinsic::riscv_vmv_x_s: {
11982 SDValue Res = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Op.getOperand(1));
11983 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Res);
11984 }
11985 case Intrinsic::riscv_vfmv_f_s:
11986 return DAG.getExtractVectorElt(DL, Op.getValueType(), Op.getOperand(1), 0);
11987 case Intrinsic::riscv_vmv_v_x:
11988 return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
11989 Op.getOperand(3), Op.getSimpleValueType(), DL, DAG,
11990 Subtarget);
11991 case Intrinsic::riscv_vfmv_v_f:
11992 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
11993 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
11994 case Intrinsic::riscv_vmv_s_x: {
11995 SDValue Scalar = Op.getOperand(2);
11996
11997 if (Scalar.getValueType().bitsLE(XLenVT)) {
11998 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
11999 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
12000 Op.getOperand(1), Scalar, Op.getOperand(3));
12001 }
12002
12003 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
12004
12005 // This is an i64 value that lives in two scalar registers. We have to
12006 // insert this in a convoluted way. First we build vXi64 splat containing
12007 // the two values that we assemble using some bit math. Next we'll use
12008 // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
12009 // to merge element 0 from our splat into the source vector.
12010 // FIXME: This is probably not the best way to do this, but it is
12011 // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
12012 // point.
12013 // sw lo, (a0)
12014 // sw hi, 4(a0)
12015 // vlse vX, (a0)
12016 //
12017 // vid.v vVid
12018 // vmseq.vx mMask, vVid, 0
12019 // vmerge.vvm vDest, vSrc, vVal, mMask
12020 MVT VT = Op.getSimpleValueType();
12021 SDValue Vec = Op.getOperand(1);
12022 SDValue VL = getVLOperand(Op);
12023
12024 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, SDValue(), Scalar, VL, DAG);
12025 if (Op.getOperand(1).isUndef())
12026 return SplattedVal;
12027 SDValue SplattedIdx =
12028 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
12029 DAG.getConstant(0, DL, MVT::i32), VL);
12030
12031 MVT MaskVT = getMaskTypeFor(VT);
12032 SDValue Mask = getAllOnesMask(VT, VL, DL, DAG);
12033 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
12034 SDValue SelectCond =
12035 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT,
12036 {VID, SplattedIdx, DAG.getCondCode(ISD::SETEQ),
12037 DAG.getUNDEF(MaskVT), Mask, VL});
12038 return DAG.getNode(RISCVISD::VMERGE_VL, DL, VT, SelectCond, SplattedVal,
12039 Vec, DAG.getUNDEF(VT), VL);
12040 }
12041 case Intrinsic::riscv_vfmv_s_f:
12042 return DAG.getNode(RISCVISD::VFMV_S_F_VL, DL, Op.getValueType(),
12043 Op.getOperand(1), Op.getOperand(2), Op.getOperand(3));
12044 // EGS * EEW >= 128 bits
12045 case Intrinsic::riscv_vaesdf_vv:
12046 case Intrinsic::riscv_vaesdf_vs:
12047 case Intrinsic::riscv_vaesdm_vv:
12048 case Intrinsic::riscv_vaesdm_vs:
12049 case Intrinsic::riscv_vaesef_vv:
12050 case Intrinsic::riscv_vaesef_vs:
12051 case Intrinsic::riscv_vaesem_vv:
12052 case Intrinsic::riscv_vaesem_vs:
12053 case Intrinsic::riscv_vaeskf1:
12054 case Intrinsic::riscv_vaeskf2:
12055 case Intrinsic::riscv_vaesz_vs:
12056 case Intrinsic::riscv_vsm4k:
12057 case Intrinsic::riscv_vsm4r_vv:
12058 case Intrinsic::riscv_vsm4r_vs: {
12059 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
12060 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
12061 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
12062 reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
12063 return Op;
12064 }
12065 // EGS * EEW >= 256 bits
12066 case Intrinsic::riscv_vsm3c:
12067 case Intrinsic::riscv_vsm3me: {
12068 if (!isValidEGW(8, Op.getSimpleValueType(), Subtarget) ||
12069 !isValidEGW(8, Op->getOperand(1).getSimpleValueType(), Subtarget))
12070 reportFatalUsageError("EGW should be greater than or equal to 8 * SEW.");
12071 return Op;
12072 }
12073 // zvknha(SEW=32)/zvknhb(SEW=[32|64])
12074 case Intrinsic::riscv_vsha2ch:
12075 case Intrinsic::riscv_vsha2cl:
12076 case Intrinsic::riscv_vsha2ms: {
12077 if (Op->getSimpleValueType(0).getScalarSizeInBits() == 64 &&
12078 !Subtarget.hasStdExtZvknhb())
12079 reportFatalUsageError("SEW=64 needs Zvknhb to be enabled.");
12080 if (!isValidEGW(4, Op.getSimpleValueType(), Subtarget) ||
12081 !isValidEGW(4, Op->getOperand(1).getSimpleValueType(), Subtarget) ||
12082 !isValidEGW(4, Op->getOperand(2).getSimpleValueType(), Subtarget))
12083 reportFatalUsageError("EGW should be greater than or equal to 4 * SEW.");
12084 return Op;
12085 }
12086 case Intrinsic::riscv_sf_vc_v_x:
12087 case Intrinsic::riscv_sf_vc_v_i:
12088 case Intrinsic::riscv_sf_vc_v_xv:
12089 case Intrinsic::riscv_sf_vc_v_iv:
12090 case Intrinsic::riscv_sf_vc_v_vv:
12091 case Intrinsic::riscv_sf_vc_v_fv:
12092 case Intrinsic::riscv_sf_vc_v_xvv:
12093 case Intrinsic::riscv_sf_vc_v_ivv:
12094 case Intrinsic::riscv_sf_vc_v_vvv:
12095 case Intrinsic::riscv_sf_vc_v_fvv:
12096 case Intrinsic::riscv_sf_vc_v_xvw:
12097 case Intrinsic::riscv_sf_vc_v_ivw:
12098 case Intrinsic::riscv_sf_vc_v_vvw:
12099 case Intrinsic::riscv_sf_vc_v_fvw: {
12100 MVT VT = Op.getSimpleValueType();
12101
12102 SmallVector<SDValue> Operands{Op->op_values()};
12103 processVCIXOperands(Op, Operands, DAG);
12104
12105 MVT RetVT = VT;
12106 if (VT.isFixedLengthVector())
12107 RetVT = getContainerForFixedLengthVector(VT);
12108 else if (VT.isFloatingPoint())
12109 RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
12110 VT.getVectorElementCount());
12111
12112 SDValue NewNode = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, RetVT, Operands);
12113
12114 if (VT.isFixedLengthVector())
12115 NewNode = convertFromScalableVector(VT, NewNode, DAG, Subtarget);
12116 else if (VT.isFloatingPoint())
12117 NewNode = DAG.getBitcast(VT, NewNode);
12118
12119 if (Op == NewNode)
12120 break;
12121
12122 return NewNode;
12123 }
12124 }
12125
12126 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12127}
12128
12129static inline SDValue getVCIXISDNodeWCHAIN(SDValue Op, SelectionDAG &DAG,
12130 unsigned Type) {
12131 SDLoc DL(Op);
12132 SmallVector<SDValue> Operands{Op->op_values()};
12133 Operands.erase(Operands.begin() + 1);
12134
12135 const RISCVSubtarget &Subtarget =
12136 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
12137 MVT VT = Op.getSimpleValueType();
12138 MVT RetVT = VT;
12139 MVT FloatVT = VT;
12140
12141 if (VT.isFloatingPoint()) {
12142 RetVT = MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits()),
12143 VT.getVectorElementCount());
12144 FloatVT = RetVT;
12145 }
12146 if (VT.isFixedLengthVector())
12147 RetVT = getContainerForFixedLengthVector(RetVT, Subtarget);
12148
12149 processVCIXOperands(Op, Operands, DAG);
12150
12151 SDVTList VTs = DAG.getVTList({RetVT, MVT::Other});
12152 SDValue NewNode = DAG.getNode(Type, DL, VTs, Operands);
12153 SDValue Chain = NewNode.getValue(1);
12154
12155 if (VT.isFixedLengthVector())
12156 NewNode = convertFromScalableVector(FloatVT, NewNode, DAG, Subtarget);
12157 if (VT.isFloatingPoint())
12158 NewNode = DAG.getBitcast(VT, NewNode);
12159
12160 NewNode = DAG.getMergeValues({NewNode, Chain}, DL);
12161
12162 return NewNode;
12163}
12164
12165static inline SDValue getVCIXISDNodeVOID(SDValue Op, SelectionDAG &DAG,
12166 unsigned Type) {
12167 SmallVector<SDValue> Operands{Op->op_values()};
12168 Operands.erase(Operands.begin() + 1);
12169 processVCIXOperands(Op, Operands, DAG);
12170
12171 return DAG.getNode(Type, SDLoc(Op), Op.getValueType(), Operands);
12172}
12173
12174static SDValue
12175lowerFixedVectorSegLoadIntrinsics(unsigned IntNo, SDValue Op,
12176 const RISCVSubtarget &Subtarget,
12177 SelectionDAG &DAG) {
12178 bool IsStrided;
12179 switch (IntNo) {
12180 case Intrinsic::riscv_seg2_load_mask:
12181 case Intrinsic::riscv_seg3_load_mask:
12182 case Intrinsic::riscv_seg4_load_mask:
12183 case Intrinsic::riscv_seg5_load_mask:
12184 case Intrinsic::riscv_seg6_load_mask:
12185 case Intrinsic::riscv_seg7_load_mask:
12186 case Intrinsic::riscv_seg8_load_mask:
12187 IsStrided = false;
12188 break;
12189 case Intrinsic::riscv_sseg2_load_mask:
12190 case Intrinsic::riscv_sseg3_load_mask:
12191 case Intrinsic::riscv_sseg4_load_mask:
12192 case Intrinsic::riscv_sseg5_load_mask:
12193 case Intrinsic::riscv_sseg6_load_mask:
12194 case Intrinsic::riscv_sseg7_load_mask:
12195 case Intrinsic::riscv_sseg8_load_mask:
12196 IsStrided = true;
12197 break;
12198 default:
12199 llvm_unreachable("unexpected intrinsic ID");
12200 };
12201
12202 static const Intrinsic::ID VlsegInts[7] = {
12203 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
12204 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
12205 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
12206 Intrinsic::riscv_vlseg8_mask};
12207 static const Intrinsic::ID VlssegInts[7] = {
12208 Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
12209 Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
12210 Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
12211 Intrinsic::riscv_vlsseg8_mask};
12212
12213 SDLoc DL(Op);
12214 unsigned NF = Op->getNumValues() - 1;
12215 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
12216 MVT XLenVT = Subtarget.getXLenVT();
12217 MVT VT = Op->getSimpleValueType(0);
12218 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
12219 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
12220 ContainerVT.getScalarSizeInBits();
12221 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
12222
12223 // Operands: (chain, int_id, pointer, mask, vl) or
12224 // (chain, int_id, pointer, offset, mask, vl)
12225 SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
12226 SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
12227 MVT MaskVT = Mask.getSimpleValueType();
12228 MVT MaskContainerVT = ::getContainerForFixedLengthVector(MaskVT, Subtarget);
12229 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
12230
12231 SDValue IntID = DAG.getTargetConstant(
12232 IsStrided ? VlssegInts[NF - 2] : VlsegInts[NF - 2], DL, XLenVT);
12233 auto *Load = cast<MemIntrinsicSDNode>(Op);
12234
12235 SDVTList VTs = DAG.getVTList({VecTupTy, MVT::Other});
12236 SmallVector<SDValue, 9> Ops = {
12237 Load->getChain(),
12238 IntID,
12239 DAG.getUNDEF(VecTupTy),
12240 Op.getOperand(2),
12241 Mask,
12242 VL,
12243 DAG.getTargetConstant(
12244 RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, XLenVT),
12245 DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
12246 // Insert the stride operand.
12247 if (IsStrided)
12248 Ops.insert(std::next(Ops.begin(), 4), Op.getOperand(3));
12249
12250 SDValue Result =
12251 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
12252 Load->getMemoryVT(), Load->getMemOperand());
12253 SmallVector<SDValue, 9> Results;
12254 for (unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
12255 SDValue SubVec = DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, ContainerVT,
12256 Result.getValue(0),
12257 DAG.getTargetConstant(RetIdx, DL, MVT::i32));
12258 Results.push_back(convertFromScalableVector(VT, SubVec, DAG, Subtarget));
12259 }
12260 Results.push_back(Result.getValue(1));
12261 return DAG.getMergeValues(Results, DL);
12262}
12263
12264SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
12265 SelectionDAG &DAG) const {
12266 unsigned IntNo = Op.getConstantOperandVal(1);
12267 switch (IntNo) {
12268 default:
12269 break;
12270 case Intrinsic::riscv_seg2_load_mask:
12271 case Intrinsic::riscv_seg3_load_mask:
12272 case Intrinsic::riscv_seg4_load_mask:
12273 case Intrinsic::riscv_seg5_load_mask:
12274 case Intrinsic::riscv_seg6_load_mask:
12275 case Intrinsic::riscv_seg7_load_mask:
12276 case Intrinsic::riscv_seg8_load_mask:
12277 case Intrinsic::riscv_sseg2_load_mask:
12278 case Intrinsic::riscv_sseg3_load_mask:
12279 case Intrinsic::riscv_sseg4_load_mask:
12280 case Intrinsic::riscv_sseg5_load_mask:
12281 case Intrinsic::riscv_sseg6_load_mask:
12282 case Intrinsic::riscv_sseg7_load_mask:
12283 case Intrinsic::riscv_sseg8_load_mask:
12284 return lowerFixedVectorSegLoadIntrinsics(IntNo, Op, Subtarget, DAG);
12285
12286 case Intrinsic::riscv_sf_vc_v_x_se:
12287 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_X_SE);
12288 case Intrinsic::riscv_sf_vc_v_i_se:
12289 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_I_SE);
12290 case Intrinsic::riscv_sf_vc_v_xv_se:
12291 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XV_SE);
12292 case Intrinsic::riscv_sf_vc_v_iv_se:
12293 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IV_SE);
12294 case Intrinsic::riscv_sf_vc_v_vv_se:
12295 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VV_SE);
12296 case Intrinsic::riscv_sf_vc_v_fv_se:
12297 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FV_SE);
12298 case Intrinsic::riscv_sf_vc_v_xvv_se:
12299 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVV_SE);
12300 case Intrinsic::riscv_sf_vc_v_ivv_se:
12301 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVV_SE);
12302 case Intrinsic::riscv_sf_vc_v_vvv_se:
12303 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVV_SE);
12304 case Intrinsic::riscv_sf_vc_v_fvv_se:
12305 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVV_SE);
12306 case Intrinsic::riscv_sf_vc_v_xvw_se:
12307 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_XVW_SE);
12308 case Intrinsic::riscv_sf_vc_v_ivw_se:
12309 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_IVW_SE);
12310 case Intrinsic::riscv_sf_vc_v_vvw_se:
12311 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_VVW_SE);
12312 case Intrinsic::riscv_sf_vc_v_fvw_se:
12313 return getVCIXISDNodeWCHAIN(Op, DAG, RISCVISD::SF_VC_V_FVW_SE);
12314 }
12315
12316 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12317}
12318
12319static SDValue
12320lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
12321 const RISCVSubtarget &Subtarget,
12322 SelectionDAG &DAG) {
12323 bool IsStrided;
12324 switch (IntNo) {
12325 case Intrinsic::riscv_seg2_store_mask:
12326 case Intrinsic::riscv_seg3_store_mask:
12327 case Intrinsic::riscv_seg4_store_mask:
12328 case Intrinsic::riscv_seg5_store_mask:
12329 case Intrinsic::riscv_seg6_store_mask:
12330 case Intrinsic::riscv_seg7_store_mask:
12331 case Intrinsic::riscv_seg8_store_mask:
12332 IsStrided = false;
12333 break;
12334 case Intrinsic::riscv_sseg2_store_mask:
12335 case Intrinsic::riscv_sseg3_store_mask:
12336 case Intrinsic::riscv_sseg4_store_mask:
12337 case Intrinsic::riscv_sseg5_store_mask:
12338 case Intrinsic::riscv_sseg6_store_mask:
12339 case Intrinsic::riscv_sseg7_store_mask:
12340 case Intrinsic::riscv_sseg8_store_mask:
12341 IsStrided = true;
12342 break;
12343 default:
12344 llvm_unreachable("unexpected intrinsic ID");
12345 }
12346
12347 SDLoc DL(Op);
12348 static const Intrinsic::ID VssegInts[] = {
12349 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
12350 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
12351 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
12352 Intrinsic::riscv_vsseg8_mask};
12353 static const Intrinsic::ID VsssegInts[] = {
12354 Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
12355 Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
12356 Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
12357 Intrinsic::riscv_vssseg8_mask};
12358
12359 // Operands: (chain, int_id, vec*, ptr, mask, vl) or
12360 // (chain, int_id, vec*, ptr, stride, mask, vl)
12361 unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
12362 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
12363 MVT XLenVT = Subtarget.getXLenVT();
12364 MVT VT = Op->getOperand(2).getSimpleValueType();
12365 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
12366 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
12367 ContainerVT.getScalarSizeInBits();
12368 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NF);
12369
12370 SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
12371 SDValue Mask = Op.getOperand(Op.getNumOperands() - 2);
12372 MVT MaskVT = Mask.getSimpleValueType();
12373 MVT MaskContainerVT = ::getContainerForFixedLengthVector(MaskVT, Subtarget);
12374 Mask = convertToScalableVector(MaskContainerVT, Mask, DAG, Subtarget);
12375
12376 SDValue IntID = DAG.getTargetConstant(
12377 IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, XLenVT);
12378 SDValue Ptr = Op->getOperand(NF + 2);
12379
12380 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Op);
12381
12382 SDValue StoredVal = DAG.getUNDEF(VecTupTy);
12383 for (unsigned i = 0; i < NF; i++)
12384 StoredVal = DAG.getNode(
12385 RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
12386 convertToScalableVector(ContainerVT, FixedIntrinsic->getOperand(2 + i),
12387 DAG, Subtarget),
12388 DAG.getTargetConstant(i, DL, MVT::i32));
12389
12390 SmallVector<SDValue, 10> Ops = {
12391 FixedIntrinsic->getChain(),
12392 IntID,
12393 StoredVal,
12394 Ptr,
12395 Mask,
12396 VL,
12397 DAG.getTargetConstant(Log2_64(VT.getScalarSizeInBits()), DL, XLenVT)};
12398 // Insert the stride operand.
12399 if (IsStrided)
12400 Ops.insert(std::next(Ops.begin(), 4),
12401 Op.getOperand(Op.getNumOperands() - 3));
12402
12403 return DAG.getMemIntrinsicNode(
12404 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
12405 FixedIntrinsic->getMemoryVT(), FixedIntrinsic->getMemOperand());
12406}
12407
12408SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12409 SelectionDAG &DAG) const {
12410 unsigned IntNo = Op.getConstantOperandVal(1);
12411 switch (IntNo) {
12412 default:
12413 break;
12414 case Intrinsic::riscv_seg2_store_mask:
12415 case Intrinsic::riscv_seg3_store_mask:
12416 case Intrinsic::riscv_seg4_store_mask:
12417 case Intrinsic::riscv_seg5_store_mask:
12418 case Intrinsic::riscv_seg6_store_mask:
12419 case Intrinsic::riscv_seg7_store_mask:
12420 case Intrinsic::riscv_seg8_store_mask:
12421 case Intrinsic::riscv_sseg2_store_mask:
12422 case Intrinsic::riscv_sseg3_store_mask:
12423 case Intrinsic::riscv_sseg4_store_mask:
12424 case Intrinsic::riscv_sseg5_store_mask:
12425 case Intrinsic::riscv_sseg6_store_mask:
12426 case Intrinsic::riscv_sseg7_store_mask:
12427 case Intrinsic::riscv_sseg8_store_mask:
12428 return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
12429
12430 case Intrinsic::riscv_sf_vc_xv_se:
12431 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XV_SE);
12432 case Intrinsic::riscv_sf_vc_iv_se:
12433 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IV_SE);
12434 case Intrinsic::riscv_sf_vc_vv_se:
12435 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VV_SE);
12436 case Intrinsic::riscv_sf_vc_fv_se:
12437 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FV_SE);
12438 case Intrinsic::riscv_sf_vc_xvv_se:
12439 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVV_SE);
12440 case Intrinsic::riscv_sf_vc_ivv_se:
12441 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVV_SE);
12442 case Intrinsic::riscv_sf_vc_vvv_se:
12443 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVV_SE);
12444 case Intrinsic::riscv_sf_vc_fvv_se:
12445 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVV_SE);
12446 case Intrinsic::riscv_sf_vc_xvw_se:
12447 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_XVW_SE);
12448 case Intrinsic::riscv_sf_vc_ivw_se:
12449 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_IVW_SE);
12450 case Intrinsic::riscv_sf_vc_vvw_se:
12451 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_VVW_SE);
12452 case Intrinsic::riscv_sf_vc_fvw_se:
12453 return getVCIXISDNodeVOID(Op, DAG, RISCVISD::SF_VC_FVW_SE);
12454 }
12455
12456 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12457}
12458
12459static unsigned getRVVReductionOp(unsigned ISDOpcode) {
12460 switch (ISDOpcode) {
12461 default:
12462 llvm_unreachable("Unhandled reduction");
12463 case ISD::VP_REDUCE_ADD:
12464 case ISD::VECREDUCE_ADD:
12465 return RISCVISD::VECREDUCE_ADD_VL;
12466 case ISD::VP_REDUCE_UMAX:
12467 case ISD::VECREDUCE_UMAX:
12468 return RISCVISD::VECREDUCE_UMAX_VL;
12469 case ISD::VP_REDUCE_SMAX:
12470 case ISD::VECREDUCE_SMAX:
12471 return RISCVISD::VECREDUCE_SMAX_VL;
12472 case ISD::VP_REDUCE_UMIN:
12473 case ISD::VECREDUCE_UMIN:
12474 return RISCVISD::VECREDUCE_UMIN_VL;
12475 case ISD::VP_REDUCE_SMIN:
12476 case ISD::VECREDUCE_SMIN:
12477 return RISCVISD::VECREDUCE_SMIN_VL;
12478 case ISD::VP_REDUCE_AND:
12479 case ISD::VECREDUCE_AND:
12480 return RISCVISD::VECREDUCE_AND_VL;
12481 case ISD::VP_REDUCE_OR:
12482 case ISD::VECREDUCE_OR:
12483 return RISCVISD::VECREDUCE_OR_VL;
12484 case ISD::VP_REDUCE_XOR:
12485 case ISD::VECREDUCE_XOR:
12486 return RISCVISD::VECREDUCE_XOR_VL;
12487 case ISD::VP_REDUCE_FADD:
12488 return RISCVISD::VECREDUCE_FADD_VL;
12489 case ISD::VP_REDUCE_SEQ_FADD:
12490 return RISCVISD::VECREDUCE_SEQ_FADD_VL;
12491 case ISD::VP_REDUCE_FMAX:
12492 case ISD::VP_REDUCE_FMAXIMUM:
12493 return RISCVISD::VECREDUCE_FMAX_VL;
12494 case ISD::VP_REDUCE_FMIN:
12495 case ISD::VP_REDUCE_FMINIMUM:
12496 return RISCVISD::VECREDUCE_FMIN_VL;
12497 }
12498
12499}
12500
12501SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
12502 SelectionDAG &DAG,
12503 bool IsVP) const {
12504 SDLoc DL(Op);
12505 SDValue Vec = Op.getOperand(IsVP ? 1 : 0);
12506 MVT VecVT = Vec.getSimpleValueType();
12507 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
12508 Op.getOpcode() == ISD::VECREDUCE_OR ||
12509 Op.getOpcode() == ISD::VECREDUCE_XOR ||
12510 Op.getOpcode() == ISD::VP_REDUCE_AND ||
12511 Op.getOpcode() == ISD::VP_REDUCE_OR ||
12512 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
12513 "Unexpected reduction lowering");
12514
12515 MVT XLenVT = Subtarget.getXLenVT();
12516
12517 MVT ContainerVT = VecVT;
12518 if (VecVT.isFixedLengthVector()) {
12519 ContainerVT = getContainerForFixedLengthVector(VecVT);
12520 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12521 }
12522
12523 SDValue Mask, VL;
12524 if (IsVP) {
12525 Mask = Op.getOperand(2);
12526 VL = Op.getOperand(3);
12527 } else {
12528 std::tie(Mask, VL) =
12529 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12530 }
12531
12532 ISD::CondCode CC;
12533 switch (Op.getOpcode()) {
12534 default:
12535 llvm_unreachable("Unhandled reduction");
12536 case ISD::VECREDUCE_AND:
12537 case ISD::VP_REDUCE_AND: {
12538 // vcpop ~x == 0
12539 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
12540 if (IsVP || VecVT.isFixedLengthVector())
12541 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, TrueMask, VL);
12542 else
12543 Vec = DAG.getNode(ISD::XOR, DL, ContainerVT, Vec, TrueMask);
12544 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
12545 CC = ISD::SETEQ;
12546 break;
12547 }
12548 case ISD::VECREDUCE_OR:
12549 case ISD::VP_REDUCE_OR:
12550 // vcpop x != 0
12551 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
12552 CC = ISD::SETNE;
12553 break;
12554 case ISD::VECREDUCE_XOR:
12555 case ISD::VP_REDUCE_XOR: {
12556 // ((vcpop x) & 1) != 0
12557 SDValue One = DAG.getConstant(1, DL, XLenVT);
12558 Vec = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Vec, Mask, VL);
12559 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
12560 CC = ISD::SETNE;
12561 break;
12562 }
12563 }
12564
12565 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
12566 SDValue SetCC = DAG.getSetCC(DL, XLenVT, Vec, Zero, CC);
12567 SetCC = DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), SetCC);
12568
12569 if (!IsVP)
12570 return SetCC;
12571
12572 // Now include the start value in the operation.
12573 // Note that we must return the start value when no elements are operated
12574 // upon. The vcpop instructions we've emitted in each case above will return
12575 // 0 for an inactive vector, and so we've already received the neutral value:
12576 // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
12577 // can simply include the start value.
12578 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
12579 return DAG.getNode(BaseOpc, DL, Op.getValueType(), SetCC, Op.getOperand(0));
12580}
12581
12582static bool isNonZeroAVL(SDValue AVL) {
12583 auto *RegisterAVL = dyn_cast<RegisterSDNode>(AVL);
12584 auto *ImmAVL = dyn_cast<ConstantSDNode>(AVL);
12585 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
12586 (ImmAVL && ImmAVL->getZExtValue() >= 1);
12587}
12588
12589/// Helper to lower a reduction sequence of the form:
12590/// scalar = reduce_op vec, scalar_start
12591static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
12592 SDValue StartValue, SDValue Vec, SDValue Mask,
12593 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
12594 const RISCVSubtarget &Subtarget) {
12595 const MVT VecVT = Vec.getSimpleValueType();
12596 const MVT M1VT = RISCVTargetLowering::getM1VT(VecVT);
12597 const MVT XLenVT = Subtarget.getXLenVT();
12598 const bool NonZeroAVL = isNonZeroAVL(VL);
12599
12600 // The reduction needs an LMUL1 input; do the splat at either LMUL1
12601 // or the original VT if fractional.
12602 auto InnerVT = VecVT.bitsLE(M1VT) ? VecVT : M1VT;
12603 // We reuse the VL of the reduction to reduce vsetvli toggles if we can
12604 // prove it is non-zero. For the AVL=0 case, we need the scalar to
12605 // be the result of the reduction operation.
12606 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(1, DL, XLenVT);
12607 SDValue InitialValue =
12608 lowerScalarInsert(StartValue, InnerVL, InnerVT, DL, DAG, Subtarget);
12609 if (M1VT != InnerVT)
12610 InitialValue =
12611 DAG.getInsertSubvector(DL, DAG.getUNDEF(M1VT), InitialValue, 0);
12612 SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(M1VT) : InitialValue;
12613 SDValue Policy = DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT);
12614 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
12615 SDValue Reduction = DAG.getNode(RVVOpcode, DL, M1VT, Ops);
12616 return DAG.getExtractVectorElt(DL, ResVT, Reduction, 0);
12617}
12618
12619SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
12620 SelectionDAG &DAG) const {
12621 SDLoc DL(Op);
12622 SDValue Vec = Op.getOperand(0);
12623 EVT VecEVT = Vec.getValueType();
12624
12625 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
12626
12627 // Due to ordering in legalize types we may have a vector type that needs to
12628 // be split. Do that manually so we can get down to a legal type.
12629 while (getTypeAction(*DAG.getContext(), VecEVT) ==
12630 TargetLowering::TypeSplitVector) {
12631 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
12632 VecEVT = Lo.getValueType();
12633 Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
12634 }
12635
12636 // TODO: The type may need to be widened rather than split. Or widened before
12637 // it can be split.
12638 if (!isTypeLegal(VecEVT))
12639 return SDValue();
12640
12641 MVT VecVT = VecEVT.getSimpleVT();
12642 MVT VecEltVT = VecVT.getVectorElementType();
12643 unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
12644
12645 MVT ContainerVT = VecVT;
12646 if (VecVT.isFixedLengthVector()) {
12647 ContainerVT = getContainerForFixedLengthVector(VecVT);
12648 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12649 }
12650
12651 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12652
12653 SDValue StartV;
12654 switch (BaseOpc) {
12655 default:
12656 StartV = DAG.getIdentityElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
12657 break;
12658 case ISD::AND:
12659 case ISD::OR:
12660 case ISD::UMAX:
12661 case ISD::UMIN:
12662 case ISD::SMAX:
12663 case ISD::SMIN:
12664 StartV = DAG.getExtractVectorElt(DL, VecEltVT, Vec, 0);
12665 break;
12666 }
12667 return lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), StartV, Vec,
12668 Mask, VL, DL, DAG, Subtarget);
12669}
12670
12671// Given a reduction op, this function returns the matching reduction opcode,
12672// the vector SDValue and the scalar SDValue required to lower this to a
12673// RISCVISD node.
12674static std::tuple<unsigned, SDValue, SDValue>
12675getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
12676 const RISCVSubtarget &Subtarget) {
12677 SDLoc DL(Op);
12678 auto Flags = Op->getFlags();
12679 unsigned Opcode = Op.getOpcode();
12680 switch (Opcode) {
12681 default:
12682 llvm_unreachable("Unhandled reduction");
12683 case ISD::VECREDUCE_FADD: {
12684 // Use positive zero if we can. It is cheaper to materialize.
12685 SDValue Zero =
12686 DAG.getConstantFP(Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, EltVT);
12687 return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0), Zero);
12688 }
12689 case ISD::VECREDUCE_SEQ_FADD:
12690 return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
12691 Op.getOperand(0));
12692 case ISD::VECREDUCE_FMINIMUM:
12693 case ISD::VECREDUCE_FMAXIMUM:
12694 case ISD::VECREDUCE_FMIN:
12695 case ISD::VECREDUCE_FMAX: {
12696 SDValue Front = DAG.getExtractVectorElt(DL, EltVT, Op.getOperand(0), 0);
12697 unsigned RVVOpc =
12698 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
12699 ? RISCVISD::VECREDUCE_FMIN_VL
12700 : RISCVISD::VECREDUCE_FMAX_VL;
12701 return std::make_tuple(RVVOpc, Op.getOperand(0), Front);
12702 }
12703 }
12704}
12705
12706SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
12707 SelectionDAG &DAG) const {
12708 SDLoc DL(Op);
12709 MVT VecEltVT = Op.getSimpleValueType();
12710
12711 unsigned RVVOpcode;
12712 SDValue VectorVal, ScalarVal;
12713 std::tie(RVVOpcode, VectorVal, ScalarVal) =
12714 getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT, Subtarget);
12715 MVT VecVT = VectorVal.getSimpleValueType();
12716
12717 MVT ContainerVT = VecVT;
12718 if (VecVT.isFixedLengthVector()) {
12719 ContainerVT = getContainerForFixedLengthVector(VecVT);
12720 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
12721 }
12722
12723 MVT ResVT = Op.getSimpleValueType();
12724 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12725 SDValue Res = lowerReductionSeq(RVVOpcode, ResVT, ScalarVal, VectorVal, Mask,
12726 VL, DL, DAG, Subtarget);
12727 if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
12728 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
12729 return Res;
12730
12731 if (Op->getFlags().hasNoNaNs())
12732 return Res;
12733
12734 // Force output to NaN if any element is Nan.
12735 SDValue IsNan =
12736 DAG.getNode(RISCVISD::SETCC_VL, DL, Mask.getValueType(),
12737 {VectorVal, VectorVal, DAG.getCondCode(ISD::SETNE),
12738 DAG.getUNDEF(Mask.getValueType()), Mask, VL});
12739 MVT XLenVT = Subtarget.getXLenVT();
12740 SDValue CPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNan, Mask, VL);
12741 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, CPop,
12742 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
12743 return DAG.getSelect(
12744 DL, ResVT, NoNaNs, Res,
12745 DAG.getConstantFP(APFloat::getNaN(ResVT.getFltSemantics()), DL, ResVT));
12746}
12747
12748SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
12749 SelectionDAG &DAG) const {
12750 SDLoc DL(Op);
12751 unsigned Opc = Op.getOpcode();
12752 SDValue Start = Op.getOperand(0);
12753 SDValue Vec = Op.getOperand(1);
12754 EVT VecEVT = Vec.getValueType();
12755 MVT XLenVT = Subtarget.getXLenVT();
12756
12757 // TODO: The type may need to be widened rather than split. Or widened before
12758 // it can be split.
12759 if (!isTypeLegal(VecEVT))
12760 return SDValue();
12761
12762 MVT VecVT = VecEVT.getSimpleVT();
12763 unsigned RVVOpcode = getRVVReductionOp(Opc);
12764
12765 if (VecVT.isFixedLengthVector()) {
12766 auto ContainerVT = getContainerForFixedLengthVector(VecVT);
12767 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12768 }
12769
12770 SDValue VL = Op.getOperand(3);
12771 SDValue Mask = Op.getOperand(2);
12772 SDValue Res =
12773 lowerReductionSeq(RVVOpcode, Op.getSimpleValueType(), Op.getOperand(0),
12774 Vec, Mask, VL, DL, DAG, Subtarget);
12775 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
12776 Op->getFlags().hasNoNaNs())
12777 return Res;
12778
12779 // Propagate NaNs.
12780 MVT PredVT = getMaskTypeFor(Vec.getSimpleValueType());
12781 // Check if any of the elements in Vec is NaN.
12782 SDValue IsNaN = DAG.getNode(
12783 RISCVISD::SETCC_VL, DL, PredVT,
12784 {Vec, Vec, DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(PredVT), Mask, VL});
12785 SDValue VCPop = DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, IsNaN, Mask, VL);
12786 // Check if the start value is NaN.
12787 SDValue StartIsNaN = DAG.getSetCC(DL, XLenVT, Start, Start, ISD::SETUO);
12788 VCPop = DAG.getNode(ISD::OR, DL, XLenVT, VCPop, StartIsNaN);
12789 SDValue NoNaNs = DAG.getSetCC(DL, XLenVT, VCPop,
12790 DAG.getConstant(0, DL, XLenVT), ISD::SETEQ);
12791 MVT ResVT = Res.getSimpleValueType();
12792 return DAG.getSelect(
12793 DL, ResVT, NoNaNs, Res,
12794 DAG.getConstantFP(APFloat::getNaN(ResVT.getFltSemantics()), DL, ResVT));
12795}
12796
12797SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
12798 SelectionDAG &DAG) const {
12799 SDValue Vec = Op.getOperand(0);
12800 SDValue SubVec = Op.getOperand(1);
12801 MVT VecVT = Vec.getSimpleValueType();
12802 MVT SubVecVT = SubVec.getSimpleValueType();
12803
12804 SDLoc DL(Op);
12805 MVT XLenVT = Subtarget.getXLenVT();
12806 unsigned OrigIdx = Op.getConstantOperandVal(2);
12807 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
12808
12809 if (OrigIdx == 0 && Vec.isUndef())
12810 return Op;
12811
12812 // We don't have the ability to slide mask vectors up indexed by their i1
12813 // elements; the smallest we can do is i8. Often we are able to bitcast to
12814 // equivalent i8 vectors. Note that when inserting a fixed-length vector
12815 // into a scalable one, we might not necessarily have enough scalable
12816 // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
12817 if (SubVecVT.getVectorElementType() == MVT::i1) {
12818 if (VecVT.getVectorMinNumElements() >= 8 &&
12819 SubVecVT.getVectorMinNumElements() >= 8) {
12820 assert(OrigIdx % 8 == 0 && "Invalid index");
12821 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
12822 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
12823 "Unexpected mask vector lowering");
12824 OrigIdx /= 8;
12825 SubVecVT =
12826 MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
12827 SubVecVT.isScalableVector());
12828 VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
12829 VecVT.isScalableVector());
12830 Vec = DAG.getBitcast(VecVT, Vec);
12831 SubVec = DAG.getBitcast(SubVecVT, SubVec);
12832 } else {
12833 // We can't slide this mask vector up indexed by its i1 elements.
12834 // This poses a problem when we wish to insert a scalable vector which
12835 // can't be re-expressed as a larger type. Just choose the slow path and
12836 // extend to a larger type, then truncate back down.
12837 MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
12838 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
12839 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
12840 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
12841 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
12842 Op.getOperand(2));
12843 SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
12844 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
12845 }
12846 }
12847
12848 // If the subvector vector is a fixed-length type and we don't know VLEN
12849 // exactly, we cannot use subregister manipulation to simplify the codegen; we
12850 // don't know which register of a LMUL group contains the specific subvector
12851 // as we only know the minimum register size. Therefore we must slide the
12852 // vector group up the full amount.
12853 const auto VLen = Subtarget.getRealVLen();
12854 if (SubVecVT.isFixedLengthVector() && !VLen) {
12855 MVT ContainerVT = VecVT;
12856 if (VecVT.isFixedLengthVector()) {
12857 ContainerVT = getContainerForFixedLengthVector(VecVT);
12858 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
12859 }
12860
12861 SubVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(ContainerVT), SubVec, 0);
12862
12863 SDValue Mask =
12864 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
12865 // Set the vector length to only the number of elements we care about. Note
12866 // that for slideup this includes the offset.
12867 unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
12868 SDValue VL = DAG.getConstant(EndIndex, DL, XLenVT);
12869
12870 // Use tail agnostic policy if we're inserting over Vec's tail.
12871 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
12872 if (VecVT.isFixedLengthVector() && EndIndex == VecVT.getVectorNumElements())
12873 Policy = RISCVVType::TAIL_AGNOSTIC;
12874
12875 // If we're inserting into the lowest elements, use a tail undisturbed
12876 // vmv.v.v.
12877 if (OrigIdx == 0) {
12878 SubVec =
12879 DAG.getNode(RISCVISD::VMV_V_V_VL, DL, ContainerVT, Vec, SubVec, VL);
12880 } else {
12881 SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
12882 SubVec = getVSlideup(DAG, Subtarget, DL, ContainerVT, Vec, SubVec,
12883 SlideupAmt, Mask, VL, Policy);
12884 }
12885
12886 if (VecVT.isFixedLengthVector())
12887 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
12888 return DAG.getBitcast(Op.getValueType(), SubVec);
12889 }
12890
12891 MVT ContainerVecVT = VecVT;
12892 if (VecVT.isFixedLengthVector()) {
12893 ContainerVecVT = getContainerForFixedLengthVector(VecVT);
12894 Vec = convertToScalableVector(ContainerVecVT, Vec, DAG, Subtarget);
12895 }
12896
12897 MVT ContainerSubVecVT = SubVecVT;
12898 if (SubVecVT.isFixedLengthVector()) {
12899 ContainerSubVecVT = getContainerForFixedLengthVector(SubVecVT);
12900 SubVec = convertToScalableVector(ContainerSubVecVT, SubVec, DAG, Subtarget);
12901 }
12902
12903 unsigned SubRegIdx;
12904 ElementCount RemIdx;
12905 // insert_subvector scales the index by vscale if the subvector is scalable,
12906 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
12907 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
12908 if (SubVecVT.isFixedLengthVector()) {
12909 assert(VLen);
12910 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12911 auto Decompose =
12912 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12913 ContainerVecVT, ContainerSubVecVT, OrigIdx / Vscale, TRI);
12914 SubRegIdx = Decompose.first;
12915 RemIdx = ElementCount::getFixed((Decompose.second * Vscale) +
12916 (OrigIdx % Vscale));
12917 } else {
12918 auto Decompose =
12919 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12920 ContainerVecVT, ContainerSubVecVT, OrigIdx, TRI);
12921 SubRegIdx = Decompose.first;
12922 RemIdx = ElementCount::getScalable(Decompose.second);
12923 }
12924
12925 TypeSize VecRegSize = TypeSize::getScalable(RISCV::RVVBitsPerBlock);
12926 assert(isPowerOf2_64(
12927 Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue()));
12928 bool ExactlyVecRegSized =
12929 Subtarget.expandVScale(SubVecVT.getSizeInBits())
12930 .isKnownMultipleOf(Subtarget.expandVScale(VecRegSize));
12931
12932 // 1. If the Idx has been completely eliminated and this subvector's size is
12933 // a vector register or a multiple thereof, or the surrounding elements are
12934 // undef, then this is a subvector insert which naturally aligns to a vector
12935 // register. These can easily be handled using subregister manipulation.
12936 // 2. If the subvector isn't an exact multiple of a valid register group size,
12937 // then the insertion must preserve the undisturbed elements of the register.
12938 // We do this by lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1
12939 // vector type (which resolves to a subregister copy), performing a VSLIDEUP
12940 // to place the subvector within the vector register, and an INSERT_SUBVECTOR
12941 // of that LMUL=1 type back into the larger vector (resolving to another
12942 // subregister operation). See below for how our VSLIDEUP works. We go via a
12943 // LMUL=1 type to avoid allocating a large register group to hold our
12944 // subvector.
12945 if (RemIdx.isZero() && (ExactlyVecRegSized || Vec.isUndef())) {
12946 if (SubVecVT.isFixedLengthVector()) {
12947 // We may get NoSubRegister if inserting at index 0 and the subvec
12948 // container is the same as the vector, e.g. vec=v4i32,subvec=v4i32,idx=0
12949 if (SubRegIdx == RISCV::NoSubRegister) {
12950 assert(OrigIdx == 0);
12951 return Op;
12952 }
12953
12954 // Use a insert_subvector that will resolve to an insert subreg.
12955 assert(VLen);
12956 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12957 SDValue Insert =
12958 DAG.getInsertSubvector(DL, Vec, SubVec, OrigIdx / Vscale);
12959 if (VecVT.isFixedLengthVector())
12960 Insert = convertFromScalableVector(VecVT, Insert, DAG, Subtarget);
12961 return Insert;
12962 }
12963 return Op;
12964 }
12965
12966 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
12967 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
12968 // (in our case undisturbed). This means we can set up a subvector insertion
12969 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
12970 // size of the subvector.
12971 MVT InterSubVT = ContainerVecVT;
12972 SDValue AlignedExtract = Vec;
12973 unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
12974 if (SubVecVT.isFixedLengthVector()) {
12975 assert(VLen);
12976 AlignedIdx /= *VLen / RISCV::RVVBitsPerBlock;
12977 }
12978 if (ContainerVecVT.bitsGT(RISCVTargetLowering::getM1VT(ContainerVecVT))) {
12979 InterSubVT = RISCVTargetLowering::getM1VT(ContainerVecVT);
12980 // Extract a subvector equal to the nearest full vector register type. This
12981 // should resolve to a EXTRACT_SUBREG instruction.
12982 AlignedExtract = DAG.getExtractSubvector(DL, InterSubVT, Vec, AlignedIdx);
12983 }
12984
12985 SubVec = DAG.getInsertSubvector(DL, DAG.getUNDEF(InterSubVT), SubVec, 0);
12986
12987 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
12988
12989 ElementCount EndIndex = RemIdx + SubVecVT.getVectorElementCount();
12990 VL = DAG.getElementCount(DL, XLenVT, SubVecVT.getVectorElementCount());
12991
12992 // Use tail agnostic policy if we're inserting over InterSubVT's tail.
12993 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
12994 if (Subtarget.expandVScale(EndIndex) ==
12995 Subtarget.expandVScale(InterSubVT.getVectorElementCount()))
12996 Policy = RISCVVType::TAIL_AGNOSTIC;
12997
12998 // If we're inserting into the lowest elements, use a tail undisturbed
12999 // vmv.v.v.
13000 if (RemIdx.isZero()) {
13001 SubVec = DAG.getNode(RISCVISD::VMV_V_V_VL, DL, InterSubVT, AlignedExtract,
13002 SubVec, VL);
13003 } else {
13004 SDValue SlideupAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
13005
13006 // Construct the vector length corresponding to RemIdx + length(SubVecVT).
13007 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
13008
13009 SubVec = getVSlideup(DAG, Subtarget, DL, InterSubVT, AlignedExtract, SubVec,
13010 SlideupAmt, Mask, VL, Policy);
13011 }
13012
13013 // If required, insert this subvector back into the correct vector register.
13014 // This should resolve to an INSERT_SUBREG instruction.
13015 if (ContainerVecVT.bitsGT(InterSubVT))
13016 SubVec = DAG.getInsertSubvector(DL, Vec, SubVec, AlignedIdx);
13017
13018 if (VecVT.isFixedLengthVector())
13019 SubVec = convertFromScalableVector(VecVT, SubVec, DAG, Subtarget);
13020
13021 // We might have bitcast from a mask type: cast back to the original type if
13022 // required.
13023 return DAG.getBitcast(Op.getSimpleValueType(), SubVec);
13024}
13025
13026SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
13027 SelectionDAG &DAG) const {
13028 SDValue Vec = Op.getOperand(0);
13029 MVT SubVecVT = Op.getSimpleValueType();
13030 MVT VecVT = Vec.getSimpleValueType();
13031
13032 SDLoc DL(Op);
13033 MVT XLenVT = Subtarget.getXLenVT();
13034 unsigned OrigIdx = Op.getConstantOperandVal(1);
13035 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
13036
13037 // With an index of 0 this is a cast-like subvector, which can be performed
13038 // with subregister operations.
13039 if (OrigIdx == 0)
13040 return Op;
13041
13042 // We don't have the ability to slide mask vectors down indexed by their i1
13043 // elements; the smallest we can do is i8. Often we are able to bitcast to
13044 // equivalent i8 vectors. Note that when extracting a fixed-length vector
13045 // from a scalable one, we might not necessarily have enough scalable
13046 // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
13047 if (SubVecVT.getVectorElementType() == MVT::i1) {
13048 if (VecVT.getVectorMinNumElements() >= 8 &&
13049 SubVecVT.getVectorMinNumElements() >= 8) {
13050 assert(OrigIdx % 8 == 0 && "Invalid index");
13051 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
13052 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
13053 "Unexpected mask vector lowering");
13054 OrigIdx /= 8;
13055 SubVecVT =
13056 MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
13057 SubVecVT.isScalableVector());
13058 VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
13059 VecVT.isScalableVector());
13060 Vec = DAG.getBitcast(VecVT, Vec);
13061 } else {
13062 // We can't slide this mask vector down, indexed by its i1 elements.
13063 // This poses a problem when we wish to extract a scalable vector which
13064 // can't be re-expressed as a larger type. Just choose the slow path and
13065 // extend to a larger type, then truncate back down.
13066 // TODO: We could probably improve this when extracting certain fixed
13067 // from fixed, where we can extract as i8 and shift the correct element
13068 // right to reach the desired subvector?
13069 MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
13070 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
13071 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
13072 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
13073 Op.getOperand(1));
13074 SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
13075 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
13076 }
13077 }
13078
13079 const auto VLen = Subtarget.getRealVLen();
13080
13081 // If the subvector vector is a fixed-length type and we don't know VLEN
13082 // exactly, we cannot use subregister manipulation to simplify the codegen; we
13083 // don't know which register of a LMUL group contains the specific subvector
13084 // as we only know the minimum register size. Therefore we must slide the
13085 // vector group down the full amount.
13086 if (SubVecVT.isFixedLengthVector() && !VLen) {
13087 MVT ContainerVT = VecVT;
13088 if (VecVT.isFixedLengthVector()) {
13089 ContainerVT = getContainerForFixedLengthVector(VecVT);
13090 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
13091 }
13092
13093 // Shrink down Vec so we're performing the slidedown on a smaller LMUL.
13094 unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
13095 if (auto ShrunkVT =
13096 getSmallestVTForIndex(ContainerVT, LastIdx, DL, DAG, Subtarget)) {
13097 ContainerVT = *ShrunkVT;
13098 Vec = DAG.getExtractSubvector(DL, ContainerVT, Vec, 0);
13099 }
13100
13101 SDValue Mask =
13102 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
13103 // Set the vector length to only the number of elements we care about. This
13104 // avoids sliding down elements we're going to discard straight away.
13105 SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
13106 SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
13107 SDValue Slidedown =
13108 getVSlidedown(DAG, Subtarget, DL, ContainerVT,
13109 DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
13110 // Now we can use a cast-like subvector extract to get the result.
13111 Slidedown = DAG.getExtractSubvector(DL, SubVecVT, Slidedown, 0);
13112 return DAG.getBitcast(Op.getValueType(), Slidedown);
13113 }
13114
13115 if (VecVT.isFixedLengthVector()) {
13116 VecVT = getContainerForFixedLengthVector(VecVT);
13117 Vec = convertToScalableVector(VecVT, Vec, DAG, Subtarget);
13118 }
13119
13120 MVT ContainerSubVecVT = SubVecVT;
13121 if (SubVecVT.isFixedLengthVector())
13122 ContainerSubVecVT = getContainerForFixedLengthVector(SubVecVT);
13123
13124 unsigned SubRegIdx;
13125 ElementCount RemIdx;
13126 // extract_subvector scales the index by vscale if the subvector is scalable,
13127 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
13128 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
13129 if (SubVecVT.isFixedLengthVector()) {
13130 assert(VLen);
13131 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13132 auto Decompose =
13133 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13134 VecVT, ContainerSubVecVT, OrigIdx / Vscale, TRI);
13135 SubRegIdx = Decompose.first;
13136 RemIdx = ElementCount::getFixed((Decompose.second * Vscale) +
13137 (OrigIdx % Vscale));
13138 } else {
13139 auto Decompose =
13140 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13141 VecVT, ContainerSubVecVT, OrigIdx, TRI);
13142 SubRegIdx = Decompose.first;
13143 RemIdx = ElementCount::getScalable(Decompose.second);
13144 }
13145
13146 // If the Idx has been completely eliminated then this is a subvector extract
13147 // which naturally aligns to a vector register. These can easily be handled
13148 // using subregister manipulation. We use an extract_subvector that will
13149 // resolve to an extract subreg.
13150 if (RemIdx.isZero()) {
13151 if (SubVecVT.isFixedLengthVector()) {
13152 assert(VLen);
13153 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13154 Vec =
13155 DAG.getExtractSubvector(DL, ContainerSubVecVT, Vec, OrigIdx / Vscale);
13156 return convertFromScalableVector(SubVecVT, Vec, DAG, Subtarget);
13157 }
13158 return Op;
13159 }
13160
13161 // Else SubVecVT is M1 or smaller and may need to be slid down: if SubVecVT
13162 // was > M1 then the index would need to be a multiple of VLMAX, and so would
13163 // divide exactly.
13164 assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second ||
13165 getLMUL(ContainerSubVecVT) == RISCVVType::LMUL_1);
13166
13167 // If the vector type is an LMUL-group type, extract a subvector equal to the
13168 // nearest full vector register type.
13169 MVT InterSubVT = VecVT;
13170 if (VecVT.bitsGT(RISCVTargetLowering::getM1VT(VecVT))) {
13171 // If VecVT has an LMUL > 1, then SubVecVT should have a smaller LMUL, and
13172 // we should have successfully decomposed the extract into a subregister.
13173 // We use an extract_subvector that will resolve to a subreg extract.
13174 assert(SubRegIdx != RISCV::NoSubRegister);
13175 (void)SubRegIdx;
13176 unsigned Idx = OrigIdx - RemIdx.getKnownMinValue();
13177 if (SubVecVT.isFixedLengthVector()) {
13178 assert(VLen);
13179 Idx /= *VLen / RISCV::RVVBitsPerBlock;
13180 }
13181 InterSubVT = RISCVTargetLowering::getM1VT(VecVT);
13182 Vec = DAG.getExtractSubvector(DL, InterSubVT, Vec, Idx);
13183 }
13184
13185 // Slide this vector register down by the desired number of elements in order
13186 // to place the desired subvector starting at element 0.
13187 SDValue SlidedownAmt = DAG.getElementCount(DL, XLenVT, RemIdx);
13188 auto [Mask, VL] = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
13189 if (SubVecVT.isFixedLengthVector())
13190 VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
13191 SDValue Slidedown =
13192 getVSlidedown(DAG, Subtarget, DL, InterSubVT, DAG.getUNDEF(InterSubVT),
13193 Vec, SlidedownAmt, Mask, VL);
13194
13195 // Now the vector is in the right position, extract our final subvector. This
13196 // should resolve to a COPY.
13197 Slidedown = DAG.getExtractSubvector(DL, SubVecVT, Slidedown, 0);
13198
13199 // We might have bitcast from a mask type: cast back to the original type if
13200 // required.
13201 return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
13202}
13203
13204// Widen a vector's operands to i8, then truncate its results back to the
13205// original type, typically i1. All operand and result types must be the same.
13206static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
13207 SelectionDAG &DAG) {
13208 MVT VT = N.getSimpleValueType();
13209 MVT WideVT = VT.changeVectorElementType(MVT::i8);
13210 SmallVector<SDValue, 4> WideOps;
13211 for (SDValue Op : N->ops()) {
13212 assert(Op.getSimpleValueType() == VT &&
13213 "Operands and result must be same type");
13214 WideOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Op));
13215 }
13216
13217 unsigned NumVals = N->getNumValues();
13218
13219 SDVTList VTs = DAG.getVTList(SmallVector<EVT, 4>(
13220 NumVals,
13221 N.getValueType().changeVectorElementType(*DAG.getContext(), MVT::i8)));
13222 SDValue WideN = DAG.getNode(N.getOpcode(), DL, VTs, WideOps);
13223 SmallVector<SDValue, 4> TruncVals;
13224 for (unsigned I = 0; I < NumVals; I++) {
13225 TruncVals.push_back(
13226 DAG.getSetCC(DL, N->getSimpleValueType(I), WideN.getValue(I),
13227 DAG.getConstant(0, DL, WideVT), ISD::SETNE));
13228 }
13229
13230 if (TruncVals.size() > 1)
13231 return DAG.getMergeValues(TruncVals, DL);
13232 return TruncVals.front();
13233}
13234
13235SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
13236 SelectionDAG &DAG) const {
13237 SDLoc DL(Op);
13238 MVT VecVT = Op.getSimpleValueType();
13239
13240 const unsigned Factor = Op->getNumValues();
13241 assert(Factor <= 8);
13242
13243 // 1 bit element vectors need to be widened to e8
13244 if (VecVT.getVectorElementType() == MVT::i1)
13245 return widenVectorOpsToi8(Op, DL, DAG);
13246
13247 bool IsFixedVector = VecVT.isFixedLengthVector();
13248
13249 MVT ContainerVecVT = VecVT;
13250 if (IsFixedVector)
13251 ContainerVecVT = getContainerForFixedLengthVector(VecVT);
13252
13253 // If concatenating would exceed LMUL=8, we need to split.
13254 if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
13255 (8 * RISCV::RVVBitsPerBlock)) {
13256 SmallVector<SDValue, 8> Ops(Factor * 2);
13257 for (unsigned i = 0; i != Factor; ++i) {
13258 auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
13259 Ops[i * 2] = OpLo;
13260 Ops[i * 2 + 1] = OpHi;
13261 }
13262
13263 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13264
13265 SDValue Lo = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
13266 ArrayRef(Ops).slice(0, Factor));
13267 SDValue Hi = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, VTs,
13268 ArrayRef(Ops).slice(Factor, Factor));
13269
13270 SmallVector<SDValue, 8> Res(Factor);
13271 for (unsigned i = 0; i != Factor; ++i)
13272 Res[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Lo.getValue(i),
13273 Hi.getValue(i));
13274
13275 return DAG.getMergeValues(Res, DL);
13276 }
13277
13278 if (Subtarget.hasStdExtZvzip() && Factor == 2 && !IsFixedVector) {
13279 MVT VT = Op->getSimpleValueType(0);
13280 MVT NewVT = VT.getDoubleNumVectorElementsVT();
13281 if (isTypeLegal(NewVT) && isLegalVTForZvzipOperand(VT, Subtarget)) {
13282 SDValue V1 = Op->getOperand(0);
13283 SDValue V2 = Op->getOperand(1);
13284 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, V1, V2);
13285 SDValue Even =
13286 lowerZvzipVUNZIP(RISCVISD::VUNZIPE_VL, V, DL, DAG, Subtarget);
13287 SDValue Odd =
13288 lowerZvzipVUNZIP(RISCVISD::VUNZIPO_VL, V, DL, DAG, Subtarget);
13289 return DAG.getMergeValues({Even, Odd}, DL);
13290 }
13291 }
13292
13293 SmallVector<SDValue, 8> Ops(Op->op_values());
13294
13295 // Concatenate the vectors as one vector to deinterleave
13296 MVT ConcatVT =
13297 MVT::getVectorVT(VecVT.getVectorElementType(),
13298 VecVT.getVectorElementCount().multiplyCoefficientBy(
13299 PowerOf2Ceil(Factor)));
13300 if (Ops.size() < PowerOf2Ceil(Factor))
13301 Ops.append(PowerOf2Ceil(Factor) - Factor, DAG.getUNDEF(VecVT));
13302 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, Ops);
13303
13304 if (Factor == 2 && !IsFixedVector) {
13305 // We can deinterleave through vnsrl.wi if the element type is smaller than
13306 // ELEN
13307 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13308 SDValue Even = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 0, DAG);
13309 SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VecVT, Concat, 2, 1, DAG);
13310 return DAG.getMergeValues({Even, Odd}, DL);
13311 }
13312
13313 // For the indices, use the vmv.v.x of an i8 constant to fill the largest
13314 // possibly mask vector, then extract the required subvector. Doing this
13315 // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
13316 // creation to be rematerialized during register allocation to reduce
13317 // register pressure if needed.
13318
13319 MVT MaskVT = ConcatVT.changeVectorElementType(MVT::i1);
13320
13321 SDValue EvenSplat = DAG.getConstant(0b01010101, DL, MVT::nxv8i8);
13322 EvenSplat = DAG.getBitcast(MVT::nxv64i1, EvenSplat);
13323 SDValue EvenMask = DAG.getExtractSubvector(DL, MaskVT, EvenSplat, 0);
13324
13325 SDValue OddSplat = DAG.getConstant(0b10101010, DL, MVT::nxv8i8);
13326 OddSplat = DAG.getBitcast(MVT::nxv64i1, OddSplat);
13327 SDValue OddMask = DAG.getExtractSubvector(DL, MaskVT, OddSplat, 0);
13328
13329 // vcompress the even and odd elements into two separate vectors
13330 SDValue EvenWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
13331 EvenMask, DAG.getUNDEF(ConcatVT));
13332 SDValue OddWide = DAG.getNode(ISD::VECTOR_COMPRESS, DL, ConcatVT, Concat,
13333 OddMask, DAG.getUNDEF(ConcatVT));
13334
13335 // Extract the result half of the gather for even and odd
13336 SDValue Even = DAG.getExtractSubvector(DL, VecVT, EvenWide, 0);
13337 SDValue Odd = DAG.getExtractSubvector(DL, VecVT, OddWide, 0);
13338
13339 return DAG.getMergeValues({Even, Odd}, DL);
13340 }
13341
13342 // Store with unit-stride store and load it back with segmented load.
13343 SDValue Mask, VL;
13344 MVT XLenVT = Subtarget.getXLenVT();
13345 auto &MF = DAG.getMachineFunction();
13346 SDValue Chain = DAG.getEntryNode();
13347 Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
13348 SDValue StackPtr;
13349 MachinePointerInfo PtrInfo;
13350 if (IsFixedVector) {
13351 // Calculating the stack size.
13352 ElementCount ActualConcatEC =
13353 VecVT.getVectorElementCount().multiplyCoefficientBy(Factor);
13354 EVT ConcatEVT = EVT::getVectorVT(
13355 *DAG.getContext(), VecVT.getVectorElementType(), ActualConcatEC);
13356 StackPtr = DAG.CreateStackTemporary(ConcatEVT.getStoreSize(), Alignment);
13357 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
13358 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
13359
13360 // If this is a fixed vector, instead of using the concat vector, we simply
13361 // store each fixed vector operand directly onto the stack, individually.
13362 // The reason being that if the fixed vector is (much) smaller than the
13363 // container vector, we will be wasting space on stack.
13364 TypeSize VecSize = VecVT.getStoreSize();
13365 SDValue BasePtr = StackPtr;
13366 MachinePointerInfo PI = PtrInfo;
13367 SmallVector<SDValue, 8> Tokens(Factor);
13368 for (auto [Idx, FieldOp] : enumerate(Op->op_values())) {
13369 if (Idx) {
13370 // Advance the pointer.
13371 BasePtr = DAG.getObjectPtrOffset(DL, BasePtr, VecSize);
13372 PI = PI.getWithOffset(VecSize);
13373 }
13374 Tokens[Idx] = DAG.getStore(Chain, DL, FieldOp, BasePtr, PI, Alignment);
13375 }
13376 Chain = DAG.getTokenFactor(DL, Tokens);
13377
13378 // Calculating Mask and VL for later usages.
13379 std::tie(Mask, VL) =
13380 getDefaultVLOps(VecVT, ContainerVecVT, DL, DAG, Subtarget);
13381 ConcatVT = getContainerForFixedLengthVector(ConcatVT);
13382 } else {
13383 std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13384 StackPtr = DAG.CreateStackTemporary(ConcatVT.getStoreSize(), Alignment);
13385 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
13386 PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
13387
13388 SDValue StoreOps[] = {
13389 Chain, DAG.getTargetConstant(Intrinsic::riscv_vse, DL, XLenVT), Concat,
13390 StackPtr, VL};
13391
13392 Chain = DAG.getMemIntrinsicNode(
13393 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), StoreOps,
13394 ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13395 MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
13396 }
13397
13398 // Load it back with segmented load.
13399 SDValue Passthru = DAG.getUNDEF(ConcatVT);
13400 static const Intrinsic::ID VlsegIntrinsicsIds[] = {
13401 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
13402 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
13403 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
13404 Intrinsic::riscv_vlseg8_mask};
13405
13406 SDValue LoadOps[] = {
13407 Chain,
13408 DAG.getTargetConstant(VlsegIntrinsicsIds[Factor - 2], DL, XLenVT),
13409 Passthru,
13410 StackPtr,
13411 Mask,
13412 VL,
13413 DAG.getTargetConstant(
13414 RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, XLenVT),
13415 DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()), DL, XLenVT)};
13416
13417 unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
13418 ContainerVecVT.getScalarSizeInBits();
13419 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
13420
13421 SDValue Load = DAG.getMemIntrinsicNode(
13422 ISD::INTRINSIC_W_CHAIN, DL, DAG.getVTList({VecTupTy, MVT::Other}),
13423 LoadOps, ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13424 MachineMemOperand::MOLoad, LocationSize::beforeOrAfterPointer());
13425
13426 SmallVector<SDValue, 8> Res(Factor);
13427
13428 for (unsigned i = 0U; i < Factor; ++i) {
13429 SDValue FieldRes =
13430 DAG.getNode(RISCVISD::TUPLE_EXTRACT, DL, ContainerVecVT, Load,
13431 DAG.getTargetConstant(i, DL, MVT::i32));
13432 if (IsFixedVector)
13433 FieldRes = convertFromScalableVector(VecVT, FieldRes, DAG, Subtarget);
13434 Res[i] = FieldRes;
13435 }
13436
13437 return DAG.getMergeValues(Res, DL);
13438}
13439
13440SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
13441 SelectionDAG &DAG) const {
13442 SDLoc DL(Op);
13443 MVT VecVT = Op.getSimpleValueType();
13444
13445 const unsigned Factor = Op.getNumOperands();
13446 assert(Factor <= 8);
13447
13448 // i1 vectors need to be widened to i8
13449 if (VecVT.getVectorElementType() == MVT::i1)
13450 return widenVectorOpsToi8(Op, DL, DAG);
13451
13452 // Convert to scalable vectors first.
13453 if (VecVT.isFixedLengthVector()) {
13454 MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
13455 SmallVector<SDValue, 8> Ops(Factor);
13456 for (unsigned i = 0U; i < Factor; ++i)
13457 Ops[i] = convertToScalableVector(ContainerVT, Op.getOperand(i), DAG,
13458 Subtarget);
13459
13460 SmallVector<EVT, 8> VTs(Factor, ContainerVT);
13461 SDValue NewInterleave = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs, Ops);
13462
13463 SmallVector<SDValue, 8> Res(Factor);
13464 for (unsigned i = 0U; i < Factor; ++i)
13465 Res[i] = convertFromScalableVector(VecVT, NewInterleave.getValue(i), DAG,
13466 Subtarget);
13467 return DAG.getMergeValues(Res, DL);
13468 }
13469
13470 MVT XLenVT = Subtarget.getXLenVT();
13471 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13472
13473 // If the VT is larger than LMUL=8, we need to split and reassemble.
13474 if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
13475 (8 * RISCV::RVVBitsPerBlock)) {
13476 SmallVector<SDValue, 8> Ops(Factor * 2);
13477 for (unsigned i = 0; i != Factor; ++i) {
13478 auto [OpLo, OpHi] = DAG.SplitVectorOperand(Op.getNode(), i);
13479 Ops[i] = OpLo;
13480 Ops[i + Factor] = OpHi;
13481 }
13482
13483 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13484
13485 SDValue Res[] = {DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
13486 ArrayRef(Ops).take_front(Factor)),
13487 DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, VTs,
13488 ArrayRef(Ops).drop_front(Factor))};
13489
13490 SmallVector<SDValue, 8> Concats(Factor);
13491 for (unsigned i = 0; i != Factor; ++i) {
13492 unsigned IdxLo = 2 * i;
13493 unsigned IdxHi = 2 * i + 1;
13494 Concats[i] = DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT,
13495 Res[IdxLo / Factor].getValue(IdxLo % Factor),
13496 Res[IdxHi / Factor].getValue(IdxHi % Factor));
13497 }
13498
13499 return DAG.getMergeValues(Concats, DL);
13500 }
13501
13502 SDValue Interleaved;
13503
13504 // Spill to the stack using a segment store for simplicity.
13505 if (Factor != 2) {
13506 EVT MemVT =
13507 EVT::getVectorVT(*DAG.getContext(), VecVT.getVectorElementType(),
13508 VecVT.getVectorElementCount() * Factor);
13509
13510 // Allocate a stack slot.
13511 Align Alignment = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
13512 SDValue StackPtr =
13513 DAG.CreateStackTemporary(MemVT.getStoreSize(), Alignment);
13514 EVT PtrVT = StackPtr.getValueType();
13515 auto &MF = DAG.getMachineFunction();
13516 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
13517 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
13518
13519 static const Intrinsic::ID IntrIds[] = {
13520 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
13521 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
13522 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
13523 Intrinsic::riscv_vsseg8_mask,
13524 };
13525
13526 unsigned Sz =
13527 Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
13528 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, Factor);
13529
13530 SDValue StoredVal = DAG.getUNDEF(VecTupTy);
13531 for (unsigned i = 0; i < Factor; i++)
13532 StoredVal =
13533 DAG.getNode(RISCVISD::TUPLE_INSERT, DL, VecTupTy, StoredVal,
13534 Op.getOperand(i), DAG.getTargetConstant(i, DL, MVT::i32));
13535
13536 SDValue Ops[] = {DAG.getEntryNode(),
13537 DAG.getTargetConstant(IntrIds[Factor - 2], DL, XLenVT),
13538 StoredVal,
13539 StackPtr,
13540 Mask,
13541 VL,
13542 DAG.getTargetConstant(Log2_64(VecVT.getScalarSizeInBits()),
13543 DL, XLenVT)};
13544
13545 SDValue Chain = DAG.getMemIntrinsicNode(
13546 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops,
13547 VecVT.getVectorElementType(), PtrInfo, Alignment,
13548 MachineMemOperand::MOStore, LocationSize::beforeOrAfterPointer());
13549
13550 SmallVector<SDValue, 8> Loads(Factor);
13551
13552 SDValue Increment = DAG.getTypeSize(DL, PtrVT, VecVT.getStoreSize());
13553 for (unsigned i = 0; i != Factor; ++i) {
13554 if (i != 0)
13555 StackPtr = DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr, Increment);
13556
13557 Loads[i] = DAG.getLoad(VecVT, DL, Chain, StackPtr, PtrInfo);
13558 }
13559
13560 return DAG.getMergeValues(Loads, DL);
13561 }
13562
13563 if (Subtarget.hasStdExtZvzip() && !Op.getOperand(0).isUndef() &&
13564 !Op.getOperand(1).isUndef()) {
13565 MVT VT = Op->getSimpleValueType(0);
13566 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
13567 // Freeze the sources so we can increase their use count.
13568 SDValue V1 = DAG.getFreeze(Op->getOperand(0));
13569 SDValue V2 = DAG.getFreeze(Op->getOperand(1));
13570 SDValue Interleaved = lowerZvzipVZIP(V1, V2, DL, DAG, Subtarget);
13571 SDValue Lo = DAG.getExtractSubvector(DL, VT, Interleaved, 0);
13572 SDValue Hi = DAG.getExtractSubvector(DL, VT, Interleaved,
13573 VT.getVectorMinNumElements());
13574 return DAG.getMergeValues({Lo, Hi}, DL);
13575 }
13576 }
13577
13578 // If the element type is smaller than ELEN, then we can interleave with
13579 // vwaddu.vv and vwmaccu.vx
13580 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13581 Interleaved = getWideningInterleave(Op.getOperand(0), Op.getOperand(1), DL,
13582 DAG, Subtarget);
13583 } else {
13584 // Otherwise, fallback to using vrgathere16.vv
13585 MVT ConcatVT =
13586 MVT::getVectorVT(VecVT.getVectorElementType(),
13587 VecVT.getVectorElementCount().multiplyCoefficientBy(2));
13588 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT,
13589 Op.getOperand(0), Op.getOperand(1));
13590
13591 MVT IdxVT = ConcatVT.changeVectorElementType(MVT::i16);
13592
13593 // 0 1 2 3 4 5 6 7 ...
13594 SDValue StepVec = DAG.getStepVector(DL, IdxVT);
13595
13596 // 1 1 1 1 1 1 1 1 ...
13597 SDValue Ones = DAG.getSplatVector(IdxVT, DL, DAG.getConstant(1, DL, XLenVT));
13598
13599 // 1 0 1 0 1 0 1 0 ...
13600 SDValue OddMask = DAG.getNode(ISD::AND, DL, IdxVT, StepVec, Ones);
13601 OddMask = DAG.getSetCC(
13602 DL, IdxVT.changeVectorElementType(MVT::i1), OddMask,
13603 DAG.getSplatVector(IdxVT, DL, DAG.getConstant(0, DL, XLenVT)),
13604 ISD::CondCode::SETNE);
13605
13606 SDValue VLMax = DAG.getSplatVector(IdxVT, DL, computeVLMax(VecVT, DL, DAG));
13607
13608 // Build up the index vector for interleaving the concatenated vector
13609 // 0 0 1 1 2 2 3 3 ...
13610 SDValue Idx = DAG.getNode(ISD::SRL, DL, IdxVT, StepVec, Ones);
13611 // 0 n 1 n+1 2 n+2 3 n+3 ...
13612 Idx =
13613 DAG.getNode(RISCVISD::ADD_VL, DL, IdxVT, Idx, VLMax, Idx, OddMask, VL);
13614
13615 // Then perform the interleave
13616 // v[0] v[n] v[1] v[n+1] v[2] v[n+2] v[3] v[n+3] ...
13617 SDValue TrueMask = getAllOnesMask(IdxVT, VL, DL, DAG);
13618 Interleaved = DAG.getNode(RISCVISD::VRGATHEREI16_VV_VL, DL, ConcatVT,
13619 Concat, Idx, DAG.getUNDEF(ConcatVT), TrueMask, VL);
13620 }
13621
13622 // Extract the two halves from the interleaved result
13623 SDValue Lo = DAG.getExtractSubvector(DL, VecVT, Interleaved, 0);
13624 SDValue Hi = DAG.getExtractSubvector(DL, VecVT, Interleaved,
13625 VecVT.getVectorMinNumElements());
13626
13627 return DAG.getMergeValues({Lo, Hi}, DL);
13628}
13629
13630// Lower step_vector to the vid instruction. Any non-identity step value must
13631// be accounted for my manual expansion.
13632SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
13633 SelectionDAG &DAG) const {
13634 SDLoc DL(Op);
13635 MVT VT = Op.getSimpleValueType();
13636 assert(VT.isScalableVector() && "Expected scalable vector");
13637 MVT XLenVT = Subtarget.getXLenVT();
13638 auto [Mask, VL] = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
13639 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
13640 uint64_t StepValImm = Op.getConstantOperandVal(0);
13641 if (StepValImm != 1) {
13642 if (isPowerOf2_64(StepValImm)) {
13643 SDValue StepVal =
13644 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, DAG.getUNDEF(VT),
13645 DAG.getConstant(Log2_64(StepValImm), DL, XLenVT), VL);
13646 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
13647 } else {
13648 SDValue StepVal = lowerScalarSplat(
13649 SDValue(), DAG.getConstant(StepValImm, DL, VT.getVectorElementType()),
13650 VL, VT, DL, DAG, Subtarget);
13651 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
13652 }
13653 }
13654 return StepVec;
13655}
13656
13657// Implement vector_reverse using vrgather.vv with indices determined by
13658// subtracting the id of each element from (VLMAX-1). This will convert
13659// the indices like so:
13660// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
13661// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
13662SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
13663 SelectionDAG &DAG) const {
13664 SDLoc DL(Op);
13665 MVT VecVT = Op.getSimpleValueType();
13666
13667 // Reverse a 64-bit packed vector on RV32 by reversing each 32-bit half and
13668 // swapping them.
13669 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
13670 assert(!Subtarget.is64Bit() && VecVT.getSizeInBits() == 64 &&
13671 "Unexpected packed VECTOR_REVERSE type");
13672 SDValue V = Op.getOperand(0);
13673 if (VecVT == MVT::v2i32) {
13674 // A 2-element reverse is just an element swap.
13675 SDValue Lo = DAG.getExtractVectorElt(DL, MVT::i32, V, 0);
13676 SDValue Hi = DAG.getExtractVectorElt(DL, MVT::i32, V, 1);
13677 return DAG.getBuildVector(VecVT, DL, {Hi, Lo});
13678 }
13679 auto [Lo, Hi] = DAG.SplitVector(V, DL);
13680 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, Lo.getSimpleValueType(), Lo);
13681 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, Hi.getSimpleValueType(), Hi);
13682 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Hi, Lo);
13683 }
13684
13685 if (VecVT.getVectorElementType() == MVT::i1) {
13686 MVT WidenVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
13687 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenVT, Op.getOperand(0));
13688 SDValue Op2 = DAG.getNode(ISD::VECTOR_REVERSE, DL, WidenVT, Op1);
13689 return DAG.getSetCC(DL, VecVT, Op2,
13690 DAG.getConstant(0, DL, Op2.getValueType()), ISD::SETNE);
13691 }
13692
13693 MVT ContainerVT = VecVT;
13694 SDValue Vec = Op.getOperand(0);
13695 if (VecVT.isFixedLengthVector()) {
13696 ContainerVT = getContainerForFixedLengthVector(VecVT);
13697 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
13698 }
13699
13700 MVT XLenVT = Subtarget.getXLenVT();
13701 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
13702
13703 // On some uarchs vrgather.vv will read from every input register for each
13704 // output register, regardless of the indices. However to reverse a vector
13705 // each output register only needs to read from one register. So decompose it
13706 // into LMUL * M1 vrgather.vvs, so we get O(LMUL) performance instead of
13707 // O(LMUL^2).
13708 //
13709 // vsetvli a1, zero, e64, m4, ta, ma
13710 // vrgatherei16.vv v12, v8, v16
13711 // ->
13712 // vsetvli a1, zero, e64, m1, ta, ma
13713 // vrgather.vv v15, v8, v16
13714 // vrgather.vv v14, v9, v16
13715 // vrgather.vv v13, v10, v16
13716 // vrgather.vv v12, v11, v16
13717 if (ContainerVT.bitsGT(RISCVTargetLowering::getM1VT(ContainerVT)) &&
13718 ContainerVT.getVectorElementCount().isKnownMultipleOf(2)) {
13719 auto [Lo, Hi] = DAG.SplitVector(Vec, DL);
13720 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, Lo.getValueType(), Lo);
13721 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, Hi.getValueType(), Hi);
13722 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, ContainerVT, Hi, Lo);
13723
13724 // Fixed length vectors might not fit exactly into their container, and so
13725 // leave a gap in the front of the vector after being reversed. Slide this
13726 // away.
13727 //
13728 // x x x x 3 2 1 0 <- v4i16 @ vlen=128
13729 // 0 1 2 3 x x x x <- reverse
13730 // x x x x 0 1 2 3 <- vslidedown.vx
13731 if (VecVT.isFixedLengthVector()) {
13732 SDValue Offset = DAG.getNode(
13733 ISD::SUB, DL, XLenVT,
13734 DAG.getElementCount(DL, XLenVT, ContainerVT.getVectorElementCount()),
13735 DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()));
13736 Concat =
13737 getVSlidedown(DAG, Subtarget, DL, ContainerVT,
13738 DAG.getUNDEF(ContainerVT), Concat, Offset, Mask, VL);
13739 Concat = convertFromScalableVector(VecVT, Concat, DAG, Subtarget);
13740 }
13741 return Concat;
13742 }
13743
13744 unsigned EltSize = ContainerVT.getScalarSizeInBits();
13745 unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
13746 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
13747 unsigned MaxVLMAX =
13748 VecVT.isFixedLengthVector()
13749 ? VecVT.getVectorNumElements()
13750 : RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
13751
13752 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
13753 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
13754
13755 // If this is SEW=8 and VLMAX is potentially more than 256, we need
13756 // to use vrgatherei16.vv.
13757 if (MaxVLMAX > 256 && EltSize == 8) {
13758 // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
13759 // Reverse each half, then reassemble them in reverse order.
13760 // NOTE: It's also possible that after splitting that VLMAX no longer
13761 // requires vrgatherei16.vv.
13762 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
13763 auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0);
13764 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VecVT);
13765 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
13766 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
13767 // Reassemble the low and high pieces reversed.
13768 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VecVT, Hi, Lo);
13769 }
13770
13771 // Just promote the int type to i16 which will double the LMUL.
13772 IntVT = MVT::getVectorVT(MVT::i16, ContainerVT.getVectorElementCount());
13773 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
13774 }
13775
13776 // At LMUL > 1, do the index computation in 16 bits to reduce register
13777 // pressure.
13778 if (IntVT.getScalarType().bitsGT(MVT::i16) &&
13779 IntVT.bitsGT(RISCVTargetLowering::getM1VT(IntVT))) {
13780 assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
13781 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
13782 IntVT = IntVT.changeVectorElementType(MVT::i16);
13783 }
13784
13785 // Calculate VLMAX-1 for the desired SEW.
13786 SDValue VLMinus1 = DAG.getNode(
13787 ISD::SUB, DL, XLenVT,
13788 DAG.getElementCount(DL, XLenVT, VecVT.getVectorElementCount()),
13789 DAG.getConstant(1, DL, XLenVT));
13790
13791 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
13792 bool IsRV32E64 =
13793 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
13794 SDValue SplatVL;
13795 if (!IsRV32E64)
13796 SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
13797 else
13798 SplatVL = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IntVT, DAG.getUNDEF(IntVT),
13799 VLMinus1, DAG.getRegister(RISCV::X0, XLenVT));
13800
13801 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
13802 SDValue Indices = DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID,
13803 DAG.getUNDEF(IntVT), Mask, VL);
13804
13805 SDValue Gather = DAG.getNode(GatherOpc, DL, ContainerVT, Vec, Indices,
13806 DAG.getUNDEF(ContainerVT), Mask, VL);
13807 if (VecVT.isFixedLengthVector())
13808 Gather = convertFromScalableVector(VecVT, Gather, DAG, Subtarget);
13809 return Gather;
13810}
13811
13812SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
13813 SelectionDAG &DAG) const {
13814 SDLoc DL(Op);
13815 SDValue V1 = Op.getOperand(0);
13816 SDValue V2 = Op.getOperand(1);
13817 SDValue Offset = Op.getOperand(2);
13818 MVT XLenVT = Subtarget.getXLenVT();
13819 MVT VecVT = Op.getSimpleValueType();
13820
13821 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
13822
13823 SDValue DownOffset, UpOffset;
13824 if (Op.getOpcode() == ISD::VECTOR_SPLICE_LEFT) {
13825 // The operand is a TargetConstant, we need to rebuild it as a regular
13826 // constant.
13827 DownOffset = Offset;
13828 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, Offset);
13829 } else {
13830 // The operand is a TargetConstant, we need to rebuild it as a regular
13831 // constant rather than negating the original operand.
13832 UpOffset = Offset;
13833 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, Offset);
13834 }
13835
13836 SDValue TrueMask = getAllOnesMask(VecVT, VLMax, DL, DAG);
13837
13838 SDValue SlideDown = getVSlidedown(
13839 DAG, Subtarget, DL, VecVT, DAG.getUNDEF(VecVT), V1, DownOffset, TrueMask,
13840 Subtarget.hasVLDependentLatency() ? UpOffset
13841 : DAG.getRegister(RISCV::X0, XLenVT));
13842 return getVSlideup(DAG, Subtarget, DL, VecVT, SlideDown, V2, UpOffset,
13843 TrueMask, DAG.getRegister(RISCV::X0, XLenVT),
13844 RISCVVType::TAIL_AGNOSTIC);
13845}
13846
13847SDValue
13848RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
13849 SelectionDAG &DAG) const {
13850 SDLoc DL(Op);
13851 auto *Load = cast<LoadSDNode>(Op);
13852
13853 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
13854 Load->getMemoryVT(),
13855 *Load->getMemOperand()) &&
13856 "Expecting a correctly-aligned load");
13857
13858 MVT VT = Op.getSimpleValueType();
13859 MVT XLenVT = Subtarget.getXLenVT();
13860 MVT ContainerVT = getContainerForFixedLengthVector(VT);
13861
13862 // If we know the exact VLEN and our fixed length vector completely fills
13863 // the container, use a whole register load instead.
13864 const auto [MinVLMAX, MaxVLMAX] =
13865 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
13866 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
13867 RISCVTargetLowering::getM1VT(ContainerVT).bitsLE(ContainerVT)) {
13868 MachineMemOperand *MMO = Load->getMemOperand();
13869 SDValue NewLoad =
13870 DAG.getLoad(ContainerVT, DL, Load->getChain(), Load->getBasePtr(),
13871 MMO->getPointerInfo(), MMO->getBaseAlign(), MMO->getFlags(),
13872 MMO->getAAInfo(), MMO->getRanges());
13873 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
13874 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
13875 }
13876
13877 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
13878
13879 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
13880 SDValue IntID = DAG.getTargetConstant(
13881 IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, XLenVT);
13882 SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
13883 if (!IsMaskOp)
13884 Ops.push_back(DAG.getUNDEF(ContainerVT));
13885 Ops.push_back(Load->getBasePtr());
13886 Ops.push_back(VL);
13887 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
13888 SDValue NewLoad =
13889 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
13890 Load->getMemoryVT(), Load->getMemOperand());
13891
13892 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
13893 return DAG.getMergeValues({Result, NewLoad.getValue(1)}, DL);
13894}
13895
13896SDValue
13897RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
13898 SelectionDAG &DAG) const {
13899 SDLoc DL(Op);
13900 auto *Store = cast<StoreSDNode>(Op);
13901
13902 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
13903 Store->getMemoryVT(),
13904 *Store->getMemOperand()) &&
13905 "Expecting a correctly-aligned store");
13906
13907 SDValue StoreVal = Store->getValue();
13908 MVT VT = StoreVal.getSimpleValueType();
13909 MVT XLenVT = Subtarget.getXLenVT();
13910
13911 // If the size less than a byte, we need to pad with zeros to make a byte.
13912 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
13913 VT = MVT::v8i1;
13914 StoreVal =
13915 DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, VT), StoreVal, 0);
13916 }
13917
13918 MVT ContainerVT = getContainerForFixedLengthVector(VT);
13919
13920 SDValue NewValue =
13921 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
13922
13923 // If we know the exact VLEN and our fixed length vector completely fills
13924 // the container, use a whole register store instead.
13925 const auto [MinVLMAX, MaxVLMAX] =
13926 RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
13927 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
13928 RISCVTargetLowering::getM1VT(ContainerVT).bitsLE(ContainerVT)) {
13929 MachineMemOperand *MMO = Store->getMemOperand();
13930 return DAG.getStore(Store->getChain(), DL, NewValue, Store->getBasePtr(),
13931 MMO->getPointerInfo(), MMO->getBaseAlign(),
13932 MMO->getFlags(), MMO->getAAInfo());
13933 }
13934
13935 SDValue VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
13936
13937 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
13938 SDValue IntID = DAG.getTargetConstant(
13939 IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, XLenVT);
13940 return DAG.getMemIntrinsicNode(
13941 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
13942 {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
13943 Store->getMemoryVT(), Store->getMemOperand());
13944}
13945
13946SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
13947 SelectionDAG &DAG) const {
13948 SDLoc DL(Op);
13949 MVT VT = Op.getSimpleValueType();
13950
13951 const auto *MemSD = cast<MemSDNode>(Op);
13952 EVT MemVT = MemSD->getMemoryVT();
13953 MachineMemOperand *MMO = MemSD->getMemOperand();
13954 SDValue Chain = MemSD->getChain();
13955 SDValue BasePtr = MemSD->getBasePtr();
13956
13957 SDValue Mask, PassThru, VL;
13958 bool IsExpandingLoad = false;
13959 if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Op)) {
13960 Mask = VPLoad->getMask();
13961 PassThru = DAG.getUNDEF(VT);
13962 VL = VPLoad->getVectorLength();
13963 } else {
13964 const auto *MLoad = cast<MaskedLoadSDNode>(Op);
13965 Mask = MLoad->getMask();
13966 PassThru = MLoad->getPassThru();
13967 IsExpandingLoad = MLoad->isExpandingLoad();
13968 }
13969
13970 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
13971
13972 MVT XLenVT = Subtarget.getXLenVT();
13973
13974 MVT ContainerVT = VT;
13975 if (VT.isFixedLengthVector()) {
13976 ContainerVT = getContainerForFixedLengthVector(VT);
13977 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
13978 if (!IsUnmasked) {
13979 MVT MaskVT = getMaskTypeFor(ContainerVT);
13980 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
13981 }
13982 }
13983
13984 if (!VL)
13985 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
13986
13987 SDValue ExpandingVL;
13988 if (!IsUnmasked && IsExpandingLoad) {
13989 ExpandingVL = VL;
13990 VL =
13991 DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
13992 getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
13993 }
13994
13995 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
13996 : Intrinsic::riscv_vle_mask;
13997 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
13998 if (IntID == Intrinsic::riscv_vle)
13999 Ops.push_back(DAG.getUNDEF(ContainerVT));
14000 else
14001 Ops.push_back(PassThru);
14002 Ops.push_back(BasePtr);
14003 if (IntID == Intrinsic::riscv_vle_mask)
14004 Ops.push_back(Mask);
14005 Ops.push_back(VL);
14006 if (IntID == Intrinsic::riscv_vle_mask)
14007 Ops.push_back(DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT));
14008
14009 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
14010
14011 SDValue Result =
14012 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
14013 Chain = Result.getValue(1);
14014 if (ExpandingVL) {
14015 MVT IndexVT = ContainerVT;
14016 if (ContainerVT.isFloatingPoint())
14017 IndexVT = ContainerVT.changeVectorElementTypeToInteger();
14018
14019 MVT IndexEltVT = IndexVT.getVectorElementType();
14020 bool UseVRGATHEREI16 = false;
14021 // If index vector is an i8 vector and the element count exceeds 256, we
14022 // should change the element type of index vector to i16 to avoid
14023 // overflow.
14024 if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
14025 // FIXME: We need to do vector splitting manually for LMUL=8 cases.
14026 assert(getLMUL(IndexVT) != RISCVVType::LMUL_8);
14027 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
14028 UseVRGATHEREI16 = true;
14029 }
14030
14031 SDValue Iota =
14032 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, IndexVT,
14033 DAG.getTargetConstant(Intrinsic::riscv_viota, DL, XLenVT),
14034 DAG.getUNDEF(IndexVT), Mask, ExpandingVL);
14035 Result =
14036 DAG.getNode(UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
14037 : RISCVISD::VRGATHER_VV_VL,
14038 DL, ContainerVT, Result, Iota, PassThru, Mask, ExpandingVL);
14039 }
14040
14041 if (VT.isFixedLengthVector())
14042 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
14043
14044 return DAG.getMergeValues({Result, Chain}, DL);
14045}
14046
14047SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const {
14048 SDLoc DL(Op);
14049 MVT VT = Op->getSimpleValueType(0);
14050
14051 const auto *VPLoadFF = cast<VPLoadFFSDNode>(Op);
14052 EVT MemVT = VPLoadFF->getMemoryVT();
14053 MachineMemOperand *MMO = VPLoadFF->getMemOperand();
14054 SDValue Chain = VPLoadFF->getChain();
14055 SDValue BasePtr = VPLoadFF->getBasePtr();
14056
14057 SDValue Mask = VPLoadFF->getMask();
14058 SDValue VL = VPLoadFF->getVectorLength();
14059
14060 MVT XLenVT = Subtarget.getXLenVT();
14061
14062 MVT ContainerVT = VT;
14063 if (VT.isFixedLengthVector()) {
14064 ContainerVT = getContainerForFixedLengthVector(VT);
14065 MVT MaskVT = getMaskTypeFor(ContainerVT);
14066 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14067 }
14068
14069 unsigned IntID = Intrinsic::riscv_vleff_mask;
14070 SDValue Ops[] = {
14071 Chain,
14072 DAG.getTargetConstant(IntID, DL, XLenVT),
14073 DAG.getUNDEF(ContainerVT),
14074 BasePtr,
14075 Mask,
14076 VL,
14077 DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT)};
14078
14079 SDVTList VTs = DAG.getVTList({ContainerVT, Op->getValueType(1), MVT::Other});
14080
14081 SDValue Result =
14082 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
14083 SDValue OutVL = Result.getValue(1);
14084 Chain = Result.getValue(2);
14085
14086 if (VT.isFixedLengthVector())
14087 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
14088
14089 return DAG.getMergeValues({Result, OutVL, Chain}, DL);
14090}
14091
14092SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
14093 SelectionDAG &DAG) const {
14094 SDLoc DL(Op);
14095
14096 const auto *MemSD = cast<MemSDNode>(Op);
14097 EVT MemVT = MemSD->getMemoryVT();
14098 MachineMemOperand *MMO = MemSD->getMemOperand();
14099 SDValue Chain = MemSD->getChain();
14100 SDValue BasePtr = MemSD->getBasePtr();
14101 SDValue Val, Mask, VL;
14102
14103 bool IsCompressingStore = false;
14104 if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Op)) {
14105 Val = VPStore->getValue();
14106 Mask = VPStore->getMask();
14107 VL = VPStore->getVectorLength();
14108 } else {
14109 const auto *MStore = cast<MaskedStoreSDNode>(Op);
14110 Val = MStore->getValue();
14111 Mask = MStore->getMask();
14112 IsCompressingStore = MStore->isCompressingStore();
14113 }
14114
14115 bool IsUnmasked =
14116 ISD::isConstantSplatVectorAllOnes(Mask.getNode()) || IsCompressingStore;
14117
14118 MVT VT = Val.getSimpleValueType();
14119 MVT XLenVT = Subtarget.getXLenVT();
14120
14121 MVT ContainerVT = VT;
14122 if (VT.isFixedLengthVector()) {
14123 ContainerVT = getContainerForFixedLengthVector(VT);
14124
14125 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
14126 if (!IsUnmasked || IsCompressingStore) {
14127 MVT MaskVT = getMaskTypeFor(ContainerVT);
14128 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14129 }
14130 }
14131
14132 if (!VL)
14133 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
14134
14135 if (IsCompressingStore) {
14136 Val = DAG.getNode(
14137 ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
14138 DAG.getTargetConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
14139 DAG.getUNDEF(ContainerVT), Val, Mask, VL);
14140 VL =
14141 DAG.getNode(RISCVISD::VCPOP_VL, DL, XLenVT, Mask,
14142 getAllOnesMask(Mask.getSimpleValueType(), VL, DL, DAG), VL);
14143 }
14144
14145 unsigned IntID =
14146 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
14147 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
14148 Ops.push_back(Val);
14149 Ops.push_back(BasePtr);
14150 if (!IsUnmasked)
14151 Ops.push_back(Mask);
14152 Ops.push_back(VL);
14153
14154 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
14155 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
14156}
14157
14158SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
14159 SelectionDAG &DAG) const {
14160 SDLoc DL(Op);
14161 SDValue Val = Op.getOperand(0);
14162 SDValue Mask = Op.getOperand(1);
14163 SDValue Passthru = Op.getOperand(2);
14164
14165 MVT VT = Val.getSimpleValueType();
14166 MVT XLenVT = Subtarget.getXLenVT();
14167 MVT ContainerVT = VT;
14168 if (VT.isFixedLengthVector()) {
14169 ContainerVT = getContainerForFixedLengthVector(VT);
14170 MVT MaskVT = getMaskTypeFor(ContainerVT);
14171 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
14172 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14173 Passthru = convertToScalableVector(ContainerVT, Passthru, DAG, Subtarget);
14174 }
14175
14176 SDValue VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
14177 SDValue Res =
14178 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT,
14179 DAG.getTargetConstant(Intrinsic::riscv_vcompress, DL, XLenVT),
14180 Passthru, Val, Mask, VL);
14181
14182 if (VT.isFixedLengthVector())
14183 Res = convertFromScalableVector(VT, Res, DAG, Subtarget);
14184
14185 return Res;
14186}
14187
14188SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
14189 SelectionDAG &DAG) const {
14190 unsigned Opc = Op.getOpcode();
14191 SDLoc DL(Op);
14192 SDValue Chain = Op.getOperand(0);
14193 SDValue Op1 = Op.getOperand(1);
14194 SDValue Op2 = Op.getOperand(2);
14195 SDValue CC = Op.getOperand(3);
14196 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
14197 MVT VT = Op.getSimpleValueType();
14198 MVT InVT = Op1.getSimpleValueType();
14199
14200 // RVV VMFEQ/VMFNE ignores qNan, so we expand strict_fsetccs with OEQ/UNE
14201 // condition code.
14202 if (Opc == ISD::STRICT_FSETCCS) {
14203 // Expand strict_fsetccs(x, oeq) to
14204 // (and strict_fsetccs(x, y, oge), strict_fsetccs(x, y, ole))
14205 SDVTList VTList = Op->getVTList();
14206 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
14207 SDValue OLECCVal = DAG.getCondCode(ISD::SETOLE);
14208 SDValue Tmp1 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
14209 Op2, OLECCVal);
14210 SDValue Tmp2 = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op2,
14211 Op1, OLECCVal);
14212 SDValue OutChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
14213 Tmp1.getValue(1), Tmp2.getValue(1));
14214 // Tmp1 and Tmp2 might be the same node.
14215 if (Tmp1 != Tmp2)
14216 Tmp1 = DAG.getNode(ISD::AND, DL, VT, Tmp1, Tmp2);
14217 return DAG.getMergeValues({Tmp1, OutChain}, DL);
14218 }
14219
14220 // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
14221 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
14222 SDValue OEQCCVal = DAG.getCondCode(ISD::SETOEQ);
14223 SDValue OEQ = DAG.getNode(ISD::STRICT_FSETCCS, DL, VTList, Chain, Op1,
14224 Op2, OEQCCVal);
14225 SDValue Res = DAG.getNOT(DL, OEQ, VT);
14226 return DAG.getMergeValues({Res, OEQ.getValue(1)}, DL);
14227 }
14228 }
14229
14230 MVT ContainerInVT = InVT;
14231 if (InVT.isFixedLengthVector()) {
14232 ContainerInVT = getContainerForFixedLengthVector(InVT);
14233 Op1 = convertToScalableVector(ContainerInVT, Op1, DAG, Subtarget);
14234 Op2 = convertToScalableVector(ContainerInVT, Op2, DAG, Subtarget);
14235 }
14236 MVT MaskVT = getMaskTypeFor(ContainerInVT);
14237
14238 auto [Mask, VL] = getDefaultVLOps(InVT, ContainerInVT, DL, DAG, Subtarget);
14239
14240 SDValue Res;
14241 if (Opc == ISD::STRICT_FSETCC &&
14242 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
14243 CCVal == ISD::SETOLE)) {
14244 // VMFLT/VMFLE/VMFGT/VMFGE raise exception for qNan. Generate a mask to only
14245 // active when both input elements are ordered.
14246 SDValue True = getAllOnesMask(ContainerInVT, VL, DL, DAG);
14247 SDValue OrderMask1 = DAG.getNode(
14248 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
14249 {Chain, Op1, Op1, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
14250 True, VL});
14251 SDValue OrderMask2 = DAG.getNode(
14252 RISCVISD::STRICT_FSETCC_VL, DL, DAG.getVTList(MaskVT, MVT::Other),
14253 {Chain, Op2, Op2, DAG.getCondCode(ISD::SETOEQ), DAG.getUNDEF(MaskVT),
14254 True, VL});
14255 Mask =
14256 DAG.getNode(RISCVISD::VMAND_VL, DL, MaskVT, OrderMask1, OrderMask2, VL);
14257 // Use Mask as the passthru operand to let the result be 0 if either of the
14258 // inputs is unordered.
14259 Res = DAG.getNode(RISCVISD::STRICT_FSETCCS_VL, DL,
14260 DAG.getVTList(MaskVT, MVT::Other),
14261 {Chain, Op1, Op2, CC, Mask, Mask, VL});
14262 } else {
14263 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
14264 : RISCVISD::STRICT_FSETCCS_VL;
14265 Res = DAG.getNode(RVVOpc, DL, DAG.getVTList(MaskVT, MVT::Other),
14266 {Chain, Op1, Op2, CC, DAG.getUNDEF(MaskVT), Mask, VL});
14267 }
14268
14269 if (VT.isFixedLengthVector()) {
14270 SDValue SubVec = convertFromScalableVector(VT, Res, DAG, Subtarget);
14271 return DAG.getMergeValues({SubVec, Res.getValue(1)}, DL);
14272 }
14273 return Res;
14274}
14275
14276// Lower vector ABS to smax(X, sub(0, X)).
14277SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
14278 SDLoc DL(Op);
14279 MVT VT = Op.getSimpleValueType();
14280 SDValue X = Op.getOperand(0);
14281
14282 assert(VT.isFixedLengthVector() && "Unexpected type for ISD::ABS");
14283
14284 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14285 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
14286
14287 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
14288
14289 SDValue Result;
14290 if (Subtarget.hasStdExtZvabd()) {
14291 Result = DAG.getNode(RISCVISD::ABS_VL, DL, ContainerVT, X,
14292 DAG.getUNDEF(ContainerVT), Mask, VL);
14293 } else {
14294 SDValue SplatZero = DAG.getNode(
14295 RISCVISD::VMV_V_X_VL, DL, ContainerVT, DAG.getUNDEF(ContainerVT),
14296 DAG.getConstant(0, DL, Subtarget.getXLenVT()), VL);
14297 SDValue NegX = DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X,
14298 DAG.getUNDEF(ContainerVT), Mask, VL);
14299 Result = DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX,
14300 DAG.getUNDEF(ContainerVT), Mask, VL);
14301 }
14302 return convertFromScalableVector(VT, Result, DAG, Subtarget);
14303}
14304
14305SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op,
14306 SelectionDAG &DAG) const {
14307 const auto &TSInfo =
14308 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14309
14310 unsigned NewOpc = getRISCVVLOp(Op);
14311 bool HasPassthruOp = TSInfo.hasPassthruOp(NewOpc);
14312 bool HasMask = TSInfo.hasMaskOp(NewOpc);
14313
14314 MVT VT = Op.getSimpleValueType();
14315 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14316
14317 // Create list of operands by converting existing ones to scalable types.
14318 SmallVector<SDValue, 6> Ops;
14319 for (const SDValue &V : Op->op_values()) {
14320 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14321
14322 // Pass through non-vector operands.
14323 if (!V.getValueType().isVector()) {
14324 Ops.push_back(V);
14325 continue;
14326 }
14327
14328 // "cast" fixed length vector to a scalable vector.
14329 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
14330 "Only fixed length vectors are supported!");
14331 MVT VContainerVT = ContainerVT.changeVectorElementType(
14332 V.getSimpleValueType().getVectorElementType());
14333 Ops.push_back(convertToScalableVector(VContainerVT, V, DAG, Subtarget));
14334 }
14335
14336 SDLoc DL(Op);
14337 auto [Mask, VL] = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
14338 if (HasPassthruOp)
14339 Ops.push_back(DAG.getUNDEF(ContainerVT));
14340 if (HasMask)
14341 Ops.push_back(Mask);
14342 Ops.push_back(VL);
14343
14344 // StrictFP operations have two result values. Their lowered result should
14345 // have same result count.
14346 if (Op->isStrictFPOpcode()) {
14347 SDValue ScalableRes =
14348 DAG.getNode(NewOpc, DL, DAG.getVTList(ContainerVT, MVT::Other), Ops,
14349 Op->getFlags());
14350 SDValue SubVec = convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
14351 return DAG.getMergeValues({SubVec, ScalableRes.getValue(1)}, DL);
14352 }
14353
14354 SDValue ScalableRes =
14355 DAG.getNode(NewOpc, DL, ContainerVT, Ops, Op->getFlags());
14356 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
14357}
14358
14359// Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
14360// * Operands of each node are assumed to be in the same order.
14361// * The EVL operand is promoted from i32 to i64 on RV64.
14362// * Fixed-length vectors are converted to their scalable-vector container
14363// types.
14364SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
14365 const auto &TSInfo =
14366 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14367
14368 unsigned RISCVISDOpc = getRISCVVLOp(Op);
14369 bool HasPassthruOp = TSInfo.hasPassthruOp(RISCVISDOpc);
14370
14371 SDLoc DL(Op);
14372 MVT VT = Op.getSimpleValueType();
14373 SmallVector<SDValue, 4> Ops;
14374
14375 MVT ContainerVT = VT;
14376 if (VT.isFixedLengthVector())
14377 ContainerVT = getContainerForFixedLengthVector(VT);
14378
14379 for (const auto &OpIdx : enumerate(Op->ops())) {
14380 SDValue V = OpIdx.value();
14381 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14382 // Add dummy passthru value before the mask. Or if there isn't a mask,
14383 // before EVL.
14384 if (HasPassthruOp) {
14385 auto MaskIdx = ISD::getVPMaskIdx(Op.getOpcode());
14386 if (MaskIdx) {
14387 if (*MaskIdx == OpIdx.index())
14388 Ops.push_back(DAG.getUNDEF(ContainerVT));
14389 } else if (ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) ==
14390 OpIdx.index()) {
14391 assert(Op.getOpcode() == ISD::VP_MERGE);
14392 // For VP_MERGE, copy the false operand instead of an undef value.
14393 Ops.push_back(Ops.back());
14394 }
14395 }
14396 // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
14397 if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
14398 ISD::getVPExplicitVectorLengthIdx(Op.getOpcode()) == OpIdx.index())
14399 Ops.push_back(DAG.getTargetConstant(RISCVFPRndMode::DYN, DL,
14400 Subtarget.getXLenVT()));
14401 // Pass through operands which aren't fixed-length vectors.
14402 if (!V.getValueType().isFixedLengthVector()) {
14403 Ops.push_back(V);
14404 continue;
14405 }
14406 // "cast" fixed length vector to a scalable vector.
14407 MVT OpVT = V.getSimpleValueType();
14408 MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
14409 assert(useRVVForFixedLengthVectorVT(OpVT) &&
14410 "Only fixed length vectors are supported!");
14411 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
14412 }
14413
14414 if (!VT.isFixedLengthVector())
14415 return DAG.getNode(RISCVISDOpc, DL, VT, Ops, Op->getFlags());
14416
14417 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops, Op->getFlags());
14418
14419 return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
14420}
14421
14422SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
14423 SelectionDAG &DAG) const {
14424 SDLoc DL(Op);
14425 MVT VT = Op.getSimpleValueType();
14426 MVT XLenVT = Subtarget.getXLenVT();
14427
14428 SDValue Mask = Op.getOperand(0);
14429 SDValue TrueVal = Op.getOperand(1);
14430 SDValue FalseVal = Op.getOperand(2);
14431 SDValue VL = Op.getOperand(3);
14432
14433 // Use default legalization if a vector of EVL type would be legal.
14434 EVT EVLVecVT = EVT::getVectorVT(*DAG.getContext(), VL.getValueType(),
14435 VT.getVectorElementCount());
14436 if (isTypeLegal(EVLVecVT))
14437 return SDValue();
14438
14439 MVT ContainerVT = VT;
14440 if (VT.isFixedLengthVector()) {
14441 ContainerVT = getContainerForFixedLengthVector(VT);
14442 Mask = convertToScalableVector(ContainerVT, Mask, DAG, Subtarget);
14443 TrueVal = convertToScalableVector(ContainerVT, TrueVal, DAG, Subtarget);
14444 FalseVal = convertToScalableVector(ContainerVT, FalseVal, DAG, Subtarget);
14445 }
14446
14447 // Promote to a vector of i8.
14448 MVT PromotedVT = ContainerVT.changeVectorElementType(MVT::i8);
14449
14450 // Promote TrueVal and FalseVal using VLMax.
14451 // FIXME: Is there a better way to do this?
14452 SDValue VLMax = DAG.getRegister(RISCV::X0, XLenVT);
14453 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, PromotedVT,
14454 DAG.getUNDEF(PromotedVT),
14455 DAG.getConstant(1, DL, XLenVT), VLMax);
14456 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, PromotedVT,
14457 DAG.getUNDEF(PromotedVT),
14458 DAG.getConstant(0, DL, XLenVT), VLMax);
14459 TrueVal = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, TrueVal, SplatOne,
14460 SplatZero, DAG.getUNDEF(PromotedVT), VL);
14461 // Any element past VL uses FalseVal, so use VLMax
14462 FalseVal = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, FalseVal,
14463 SplatOne, SplatZero, DAG.getUNDEF(PromotedVT), VLMax);
14464
14465 // VP_MERGE the two promoted values.
14466 SDValue VPMerge = DAG.getNode(RISCVISD::VMERGE_VL, DL, PromotedVT, Mask,
14467 TrueVal, FalseVal, FalseVal, VL);
14468
14469 // Convert back to mask.
14470 SDValue TrueMask = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
14471 SDValue Result = DAG.getNode(
14472 RISCVISD::SETCC_VL, DL, ContainerVT,
14473 {VPMerge, DAG.getConstant(0, DL, PromotedVT), DAG.getCondCode(ISD::SETNE),
14474 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), TrueMask, VLMax});
14475
14476 if (VT.isFixedLengthVector())
14477 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
14478 return Result;
14479}
14480
14481SDValue
14482RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
14483 SelectionDAG &DAG) const {
14484 using namespace SDPatternMatch;
14485
14486 SDLoc DL(Op);
14487
14488 SDValue Op1 = Op.getOperand(0);
14489 SDValue Op2 = Op.getOperand(1);
14490 SDValue Offset = Op.getOperand(2);
14491 SDValue Mask = Op.getOperand(3);
14492 SDValue EVL1 = Op.getOperand(4);
14493 SDValue EVL2 = Op.getOperand(5);
14494
14495 const MVT XLenVT = Subtarget.getXLenVT();
14496 MVT VT = Op.getSimpleValueType();
14497 MVT ContainerVT = VT;
14498 if (VT.isFixedLengthVector()) {
14499 ContainerVT = getContainerForFixedLengthVector(VT);
14500 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
14501 Op2 = convertToScalableVector(ContainerVT, Op2, DAG, Subtarget);
14502 MVT MaskVT = getMaskTypeFor(ContainerVT);
14503 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14504 }
14505
14506 bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
14507 if (IsMaskVector) {
14508 ContainerVT = ContainerVT.changeVectorElementType(MVT::i8);
14509
14510 // Expand input operands
14511 SDValue SplatOneOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
14512 DAG.getUNDEF(ContainerVT),
14513 DAG.getConstant(1, DL, XLenVT), EVL1);
14514 SDValue SplatZeroOp1 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
14515 DAG.getUNDEF(ContainerVT),
14516 DAG.getConstant(0, DL, XLenVT), EVL1);
14517 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op1, SplatOneOp1,
14518 SplatZeroOp1, DAG.getUNDEF(ContainerVT), EVL1);
14519
14520 SDValue SplatOneOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
14521 DAG.getUNDEF(ContainerVT),
14522 DAG.getConstant(1, DL, XLenVT), EVL2);
14523 SDValue SplatZeroOp2 = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
14524 DAG.getUNDEF(ContainerVT),
14525 DAG.getConstant(0, DL, XLenVT), EVL2);
14526 Op2 = DAG.getNode(RISCVISD::VMERGE_VL, DL, ContainerVT, Op2, SplatOneOp2,
14527 SplatZeroOp2, DAG.getUNDEF(ContainerVT), EVL2);
14528 }
14529
14530 auto getVectorFirstEle = [](SDValue Vec) {
14531 SDValue FirstEle;
14532 if (sd_match(Vec, m_InsertElt(m_Value(), m_Value(FirstEle), m_Zero())))
14533 return FirstEle;
14534
14535 if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
14536 Vec.getOpcode() == ISD::BUILD_VECTOR)
14537 return Vec.getOperand(0);
14538
14539 return SDValue();
14540 };
14541
14542 if (!IsMaskVector && isNullConstant(Offset) && isOneConstant(EVL1))
14543 if (auto FirstEle = getVectorFirstEle(Op->getOperand(0))) {
14544 MVT EltVT = ContainerVT.getVectorElementType();
14545 SDValue Result;
14546 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
14547 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
14548 EltVT = EltVT.changeTypeToInteger();
14549 ContainerVT = ContainerVT.changeVectorElementType(EltVT);
14550 Op2 = DAG.getBitcast(ContainerVT, Op2);
14551 FirstEle =
14552 DAG.getAnyExtOrTrunc(DAG.getBitcast(EltVT, FirstEle), DL, XLenVT);
14553 }
14554 Result = DAG.getNode(EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
14555 : RISCVISD::VSLIDE1UP_VL,
14556 DL, ContainerVT, DAG.getUNDEF(ContainerVT), Op2,
14557 FirstEle, Mask, EVL2);
14558 Result = DAG.getBitcast(
14559 ContainerVT.changeVectorElementType(VT.getVectorElementType()),
14560 Result);
14561 return VT.isFixedLengthVector()
14562 ? convertFromScalableVector(VT, Result, DAG, Subtarget)
14563 : Result;
14564 }
14565
14566 int64_t ImmValue = cast<ConstantSDNode>(Offset)->getSExtValue();
14567 SDValue DownOffset, UpOffset;
14568 if (ImmValue >= 0) {
14569 // The operand is a TargetConstant, we need to rebuild it as a regular
14570 // constant.
14571 DownOffset = DAG.getConstant(ImmValue, DL, XLenVT);
14572 UpOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, DownOffset);
14573 } else {
14574 // The operand is a TargetConstant, we need to rebuild it as a regular
14575 // constant rather than negating the original operand.
14576 UpOffset = DAG.getConstant(-ImmValue, DL, XLenVT);
14577 DownOffset = DAG.getNode(ISD::SUB, DL, XLenVT, EVL1, UpOffset);
14578 }
14579
14580 if (ImmValue != 0)
14581 Op1 = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
14582 DAG.getUNDEF(ContainerVT), Op1, DownOffset, Mask,
14583 Subtarget.hasVLDependentLatency() ? UpOffset : EVL2);
14584 SDValue Result = getVSlideup(DAG, Subtarget, DL, ContainerVT, Op1, Op2,
14585 UpOffset, Mask, EVL2, RISCVVType::TAIL_AGNOSTIC);
14586
14587 if (IsMaskVector) {
14588 // Truncate Result back to a mask vector (Result has same EVL as Op2)
14589 Result = DAG.getNode(
14590 RISCVISD::SETCC_VL, DL, ContainerVT.changeVectorElementType(MVT::i1),
14591 {Result, DAG.getConstant(0, DL, ContainerVT),
14592 DAG.getCondCode(ISD::SETNE), DAG.getUNDEF(getMaskTypeFor(ContainerVT)),
14593 Mask, EVL2});
14594 }
14595
14596 if (!VT.isFixedLengthVector())
14597 return Result;
14598 return convertFromScalableVector(VT, Result, DAG, Subtarget);
14599}
14600
14601SDValue
14602RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
14603 SelectionDAG &DAG) const {
14604 SDLoc DL(Op);
14605 MVT VT = Op.getSimpleValueType();
14606 MVT XLenVT = Subtarget.getXLenVT();
14607
14608 SDValue Op1 = Op.getOperand(0);
14609 SDValue Mask = Op.getOperand(1);
14610 SDValue EVL = Op.getOperand(2);
14611
14612 MVT ContainerVT = VT;
14613 if (VT.isFixedLengthVector()) {
14614 ContainerVT = getContainerForFixedLengthVector(VT);
14615 Op1 = convertToScalableVector(ContainerVT, Op1, DAG, Subtarget);
14616 MVT MaskVT = getMaskTypeFor(ContainerVT);
14617 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14618 }
14619
14620 MVT GatherVT = ContainerVT;
14621 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
14622 // Check if we are working with mask vectors
14623 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
14624 if (IsMaskVector) {
14625 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(MVT::i8);
14626
14627 // Expand input operand
14628 SDValue SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
14629 DAG.getUNDEF(IndicesVT),
14630 DAG.getConstant(1, DL, XLenVT), EVL);
14631 SDValue SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
14632 DAG.getUNDEF(IndicesVT),
14633 DAG.getConstant(0, DL, XLenVT), EVL);
14634 Op1 = DAG.getNode(RISCVISD::VMERGE_VL, DL, IndicesVT, Op1, SplatOne,
14635 SplatZero, DAG.getUNDEF(IndicesVT), EVL);
14636 }
14637
14638 unsigned EltSize = GatherVT.getScalarSizeInBits();
14639 unsigned MinSize = GatherVT.getSizeInBits().getKnownMinValue();
14640 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
14641 unsigned MaxVLMAX =
14642 RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
14643
14644 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
14645 // If this is SEW=8 and VLMAX is unknown or more than 256, we need
14646 // to use vrgatherei16.vv.
14647 // TODO: It's also possible to use vrgatherei16.vv for other types to
14648 // decrease register width for the index calculation.
14649 // NOTE: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
14650 if (MaxVLMAX > 256 && EltSize == 8) {
14651 // If this is LMUL=8, we have to split before using vrgatherei16.vv.
14652 // Split the vector in half and reverse each half using a full register
14653 // reverse.
14654 // Swap the halves and concatenate them.
14655 // Slide the concatenated result by (VLMax - VL).
14656 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
14657 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(GatherVT);
14658 auto [Lo, Hi] = DAG.SplitVector(Op1, DL);
14659
14660 SDValue LoRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
14661 SDValue HiRev = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
14662
14663 // Reassemble the low and high pieces reversed.
14664 // NOTE: this Result is unmasked (because we do not need masks for
14665 // shuffles). If in the future this has to change, we can use a SELECT_VL
14666 // between Result and UNDEF using the mask originally passed to VP_REVERSE
14667 SDValue Result =
14668 DAG.getNode(ISD::CONCAT_VECTORS, DL, GatherVT, HiRev, LoRev);
14669
14670 // Slide off any elements from past EVL that were reversed into the low
14671 // elements.
14672 SDValue VLMax =
14673 DAG.getElementCount(DL, XLenVT, GatherVT.getVectorElementCount());
14674 SDValue Diff = DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, EVL);
14675
14676 Result = getVSlidedown(DAG, Subtarget, DL, GatherVT,
14677 DAG.getUNDEF(GatherVT), Result, Diff, Mask, EVL);
14678
14679 if (IsMaskVector) {
14680 // Truncate Result back to a mask vector
14681 Result =
14682 DAG.getNode(RISCVISD::SETCC_VL, DL, ContainerVT,
14683 {Result, DAG.getConstant(0, DL, GatherVT),
14684 DAG.getCondCode(ISD::SETNE),
14685 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
14686 }
14687
14688 if (!VT.isFixedLengthVector())
14689 return Result;
14690 return convertFromScalableVector(VT, Result, DAG, Subtarget);
14691 }
14692
14693 // Just promote the int type to i16 which will double the LMUL.
14694 IndicesVT = MVT::getVectorVT(MVT::i16, IndicesVT.getVectorElementCount());
14695 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
14696 }
14697
14698 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IndicesVT, Mask, EVL);
14699 SDValue VecLen =
14700 DAG.getNode(ISD::SUB, DL, XLenVT, EVL, DAG.getConstant(1, DL, XLenVT));
14701 SDValue VecLenSplat = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, IndicesVT,
14702 DAG.getUNDEF(IndicesVT), VecLen, EVL);
14703 SDValue VRSUB = DAG.getNode(RISCVISD::SUB_VL, DL, IndicesVT, VecLenSplat, VID,
14704 DAG.getUNDEF(IndicesVT), Mask, EVL);
14705 SDValue Result = DAG.getNode(GatherOpc, DL, GatherVT, Op1, VRSUB,
14706 DAG.getUNDEF(GatherVT), Mask, EVL);
14707
14708 if (IsMaskVector) {
14709 // Truncate Result back to a mask vector
14710 Result = DAG.getNode(
14711 RISCVISD::SETCC_VL, DL, ContainerVT,
14712 {Result, DAG.getConstant(0, DL, GatherVT), DAG.getCondCode(ISD::SETNE),
14713 DAG.getUNDEF(getMaskTypeFor(ContainerVT)), Mask, EVL});
14714 }
14715
14716 if (!VT.isFixedLengthVector())
14717 return Result;
14718 return convertFromScalableVector(VT, Result, DAG, Subtarget);
14719}
14720
14721SDValue RISCVTargetLowering::lowerVPStridedLoad(SDValue Op,
14722 SelectionDAG &DAG) const {
14723 SDLoc DL(Op);
14724 MVT XLenVT = Subtarget.getXLenVT();
14725 MVT VT = Op.getSimpleValueType();
14726 MVT ContainerVT = VT;
14727 if (VT.isFixedLengthVector())
14728 ContainerVT = getContainerForFixedLengthVector(VT);
14729
14730 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
14731
14732 auto *VPNode = cast<VPStridedLoadSDNode>(Op);
14733 // Check if the mask is known to be all ones
14734 SDValue Mask = VPNode->getMask();
14735 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
14736
14737 SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vlse
14738 : Intrinsic::riscv_vlse_mask,
14739 DL, XLenVT);
14740 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
14741 DAG.getUNDEF(ContainerVT), VPNode->getBasePtr(),
14742 VPNode->getStride()};
14743 if (!IsUnmasked) {
14744 if (VT.isFixedLengthVector()) {
14745 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
14746 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14747 }
14748 Ops.push_back(Mask);
14749 }
14750 Ops.push_back(VPNode->getVectorLength());
14751 if (!IsUnmasked) {
14752 SDValue Policy =
14753 DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT);
14754 Ops.push_back(Policy);
14755 }
14756
14757 SDValue Result =
14758 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
14759 VPNode->getMemoryVT(), VPNode->getMemOperand());
14760 SDValue Chain = Result.getValue(1);
14761
14762 if (VT.isFixedLengthVector())
14763 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
14764
14765 return DAG.getMergeValues({Result, Chain}, DL);
14766}
14767
14768SDValue RISCVTargetLowering::lowerVPStridedStore(SDValue Op,
14769 SelectionDAG &DAG) const {
14770 SDLoc DL(Op);
14771 MVT XLenVT = Subtarget.getXLenVT();
14772
14773 auto *VPNode = cast<VPStridedStoreSDNode>(Op);
14774 SDValue StoreVal = VPNode->getValue();
14775 MVT VT = StoreVal.getSimpleValueType();
14776 MVT ContainerVT = VT;
14777 if (VT.isFixedLengthVector()) {
14778 ContainerVT = getContainerForFixedLengthVector(VT);
14779 StoreVal = convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
14780 }
14781
14782 // Check if the mask is known to be all ones
14783 SDValue Mask = VPNode->getMask();
14784 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
14785
14786 SDValue IntID = DAG.getTargetConstant(IsUnmasked ? Intrinsic::riscv_vsse
14787 : Intrinsic::riscv_vsse_mask,
14788 DL, XLenVT);
14789 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
14790 VPNode->getBasePtr(), VPNode->getStride()};
14791 if (!IsUnmasked) {
14792 if (VT.isFixedLengthVector()) {
14793 MVT MaskVT = ContainerVT.changeVectorElementType(MVT::i1);
14794 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14795 }
14796 Ops.push_back(Mask);
14797 }
14798 Ops.push_back(VPNode->getVectorLength());
14799
14800 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, VPNode->getVTList(),
14801 Ops, VPNode->getMemoryVT(),
14802 VPNode->getMemOperand());
14803}
14804
14805// Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
14806// matched to a RVV indexed load. The RVV indexed load instructions only
14807// support the "unsigned unscaled" addressing mode; indices are implicitly
14808// zero-extended or truncated to XLEN and are treated as byte offsets. Any
14809// signed or scaled indexing is extended to the XLEN value type and scaled
14810// accordingly.
14811SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
14812 SelectionDAG &DAG) const {
14813 SDLoc DL(Op);
14814 MVT VT = Op.getSimpleValueType();
14815
14816 const auto *MemSD = cast<MemSDNode>(Op.getNode());
14817 EVT MemVT = MemSD->getMemoryVT();
14818 MachineMemOperand *MMO = MemSD->getMemOperand();
14819 SDValue Chain = MemSD->getChain();
14820 SDValue BasePtr = MemSD->getBasePtr();
14821
14822 [[maybe_unused]] ISD::LoadExtType LoadExtType;
14823 SDValue Index, Mask, PassThru, VL;
14824
14825 if (auto *VPGN = dyn_cast<VPGatherSDNode>(Op.getNode())) {
14826 Index = VPGN->getIndex();
14827 Mask = VPGN->getMask();
14828 PassThru = DAG.getUNDEF(VT);
14829 VL = VPGN->getVectorLength();
14830 // VP doesn't support extending loads.
14831 LoadExtType = ISD::NON_EXTLOAD;
14832 } else {
14833 // Else it must be a MGATHER.
14834 auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
14835 Index = MGN->getIndex();
14836 Mask = MGN->getMask();
14837 PassThru = MGN->getPassThru();
14838 LoadExtType = MGN->getExtensionType();
14839 }
14840
14841 MVT IndexVT = Index.getSimpleValueType();
14842 MVT XLenVT = Subtarget.getXLenVT();
14843
14844 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
14845 "Unexpected VTs!");
14846 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
14847 // Targets have to explicitly opt-in for extending vector loads.
14848 assert(LoadExtType == ISD::NON_EXTLOAD &&
14849 "Unexpected extending MGATHER/VP_GATHER");
14850
14851 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
14852 // the selection of the masked intrinsics doesn't do this for us.
14853 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
14854
14855 MVT ContainerVT = VT;
14856 if (VT.isFixedLengthVector()) {
14857 ContainerVT = getContainerForFixedLengthVector(VT);
14858 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
14859 ContainerVT.getVectorElementCount());
14860
14861 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
14862
14863 if (!IsUnmasked) {
14864 MVT MaskVT = getMaskTypeFor(ContainerVT);
14865 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14866 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
14867 }
14868 }
14869
14870 if (!VL)
14871 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
14872
14873 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
14874 IndexVT = IndexVT.changeVectorElementType(XLenVT);
14875 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
14876 }
14877
14878 unsigned IntID =
14879 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
14880 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
14881 if (IsUnmasked)
14882 Ops.push_back(DAG.getUNDEF(ContainerVT));
14883 else
14884 Ops.push_back(PassThru);
14885 Ops.push_back(BasePtr);
14886 Ops.push_back(Index);
14887 if (!IsUnmasked)
14888 Ops.push_back(Mask);
14889 Ops.push_back(VL);
14890 if (!IsUnmasked)
14891 Ops.push_back(DAG.getTargetConstant(RISCVVType::TAIL_AGNOSTIC, DL, XLenVT));
14892
14893 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
14894 SDValue Result =
14895 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, MemVT, MMO);
14896 Chain = Result.getValue(1);
14897
14898 if (VT.isFixedLengthVector())
14899 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
14900
14901 return DAG.getMergeValues({Result, Chain}, DL);
14902}
14903
14904// Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
14905// matched to a RVV indexed store. The RVV indexed store instructions only
14906// support the "unsigned unscaled" addressing mode; indices are implicitly
14907// zero-extended or truncated to XLEN and are treated as byte offsets. Any
14908// signed or scaled indexing is extended to the XLEN value type and scaled
14909// accordingly.
14910SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
14911 SelectionDAG &DAG) const {
14912 SDLoc DL(Op);
14913 const auto *MemSD = cast<MemSDNode>(Op.getNode());
14914 EVT MemVT = MemSD->getMemoryVT();
14915 MachineMemOperand *MMO = MemSD->getMemOperand();
14916 SDValue Chain = MemSD->getChain();
14917 SDValue BasePtr = MemSD->getBasePtr();
14918
14919 [[maybe_unused]] bool IsTruncatingStore = false;
14920 SDValue Index, Mask, Val, VL;
14921
14922 if (auto *VPSN = dyn_cast<VPScatterSDNode>(Op.getNode())) {
14923 Index = VPSN->getIndex();
14924 Mask = VPSN->getMask();
14925 Val = VPSN->getValue();
14926 VL = VPSN->getVectorLength();
14927 // VP doesn't support truncating stores.
14928 IsTruncatingStore = false;
14929 } else {
14930 // Else it must be a MSCATTER.
14931 auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
14932 Index = MSN->getIndex();
14933 Mask = MSN->getMask();
14934 Val = MSN->getValue();
14935 IsTruncatingStore = MSN->isTruncatingStore();
14936 }
14937
14938 MVT VT = Val.getSimpleValueType();
14939 MVT IndexVT = Index.getSimpleValueType();
14940 MVT XLenVT = Subtarget.getXLenVT();
14941
14942 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
14943 "Unexpected VTs!");
14944 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
14945 // Targets have to explicitly opt-in for extending vector loads and
14946 // truncating vector stores.
14947 assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
14948
14949 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
14950 // the selection of the masked intrinsics doesn't do this for us.
14951 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
14952
14953 MVT ContainerVT = VT;
14954 if (VT.isFixedLengthVector()) {
14955 ContainerVT = getContainerForFixedLengthVector(VT);
14956 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
14957 ContainerVT.getVectorElementCount());
14958
14959 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
14960 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
14961
14962 if (!IsUnmasked) {
14963 MVT MaskVT = getMaskTypeFor(ContainerVT);
14964 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
14965 }
14966 }
14967
14968 if (!VL)
14969 VL = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget).second;
14970
14971 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(XLenVT)) {
14972 IndexVT = IndexVT.changeVectorElementType(XLenVT);
14973 Index = DAG.getNode(ISD::TRUNCATE, DL, IndexVT, Index);
14974 }
14975
14976 unsigned IntID =
14977 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
14978 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(IntID, DL, XLenVT)};
14979 Ops.push_back(Val);
14980 Ops.push_back(BasePtr);
14981 Ops.push_back(Index);
14982 if (!IsUnmasked)
14983 Ops.push_back(Mask);
14984 Ops.push_back(VL);
14985
14986 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL,
14987 DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
14988}
14989
14990SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
14991 SelectionDAG &DAG) const {
14992 const MVT XLenVT = Subtarget.getXLenVT();
14993 SDLoc DL(Op);
14994 SDValue Chain = Op->getOperand(0);
14995 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::frm, DL, XLenVT);
14996 SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
14997 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
14998
14999 // Encoding used for rounding mode in RISC-V differs from that used in
15000 // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
15001 // table, which consists of a sequence of 4-bit fields, each representing
15002 // corresponding FLT_ROUNDS mode.
15003 static const int Table =
15004 (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
15005 (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
15006 (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
15007 (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
15008 (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
15009
15010 SDValue Shift =
15011 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
15012 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
15013 DAG.getConstant(Table, DL, XLenVT), Shift);
15014 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
15015 DAG.getConstant(7, DL, XLenVT));
15016
15017 return DAG.getMergeValues({Masked, Chain}, DL);
15018}
15019
15020SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
15021 SelectionDAG &DAG) const {
15022 const MVT XLenVT = Subtarget.getXLenVT();
15023 SDLoc DL(Op);
15024 SDValue Chain = Op->getOperand(0);
15025 SDValue RMValue = Op->getOperand(1);
15026 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::frm, DL, XLenVT);
15027
15028 // Encoding used for rounding mode in RISC-V differs from that used in
15029 // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
15030 // a table, which consists of a sequence of 4-bit fields, each representing
15031 // corresponding RISC-V mode.
15032 static const unsigned Table =
15033 (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
15034 (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
15035 (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
15036 (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
15037 (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
15038
15039 RMValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, RMValue);
15040
15041 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
15042 DAG.getConstant(2, DL, XLenVT));
15043 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
15044 DAG.getConstant(Table, DL, XLenVT), Shift);
15045 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
15046 DAG.getConstant(0x7, DL, XLenVT));
15047 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
15048 RMValue);
15049}
15050
15051SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
15052 SelectionDAG &DAG) const {
15053 const MVT XLenVT = Subtarget.getXLenVT();
15054 SDLoc DL(Op);
15055 SDValue Chain = Op->getOperand(0);
15056 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15057 SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
15058 return DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
15059}
15060
15061SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
15062 SelectionDAG &DAG) const {
15063 const MVT XLenVT = Subtarget.getXLenVT();
15064 SDLoc DL(Op);
15065 SDValue Chain = Op->getOperand(0);
15066 SDValue EnvValue = Op->getOperand(1);
15067 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15068
15069 EnvValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EnvValue);
15070 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
15071 EnvValue);
15072}
15073
15074SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
15075 SelectionDAG &DAG) const {
15076 const MVT XLenVT = Subtarget.getXLenVT();
15077 SDLoc DL(Op);
15078 SDValue Chain = Op->getOperand(0);
15079 SDValue EnvValue = DAG.getRegister(RISCV::X0, XLenVT);
15080 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15081
15082 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
15083 EnvValue);
15084}
15085
15086const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL;
15087const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL;
15088
15089SDValue RISCVTargetLowering::lowerGET_FPMODE(SDValue Op,
15090 SelectionDAG &DAG) const {
15091 const MVT XLenVT = Subtarget.getXLenVT();
15092 SDLoc DL(Op);
15093 SDValue Chain = Op->getOperand(0);
15094 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15095 SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
15096 SDValue Result = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
15097 Chain = Result.getValue(1);
15098 return DAG.getMergeValues({Result, Chain}, DL);
15099}
15100
15101SDValue RISCVTargetLowering::lowerSET_FPMODE(SDValue Op,
15102 SelectionDAG &DAG) const {
15103 const MVT XLenVT = Subtarget.getXLenVT();
15104 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15105 SDLoc DL(Op);
15106 SDValue Chain = Op->getOperand(0);
15107 SDValue EnvValue = Op->getOperand(1);
15108 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15109 SDValue ModeMask = DAG.getConstant(ModeMaskValue, DL, XLenVT);
15110
15111 EnvValue = DAG.getNode(ISD::ZERO_EXTEND, DL, XLenVT, EnvValue);
15112 EnvValue = DAG.getNode(ISD::AND, DL, XLenVT, EnvValue, ModeMask);
15113 Chain = DAG.getNode(RISCVISD::CLEAR_CSR, DL, MVT::Other, Chain, SysRegNo,
15114 ModeMask);
15115 return DAG.getNode(RISCVISD::SET_CSR, DL, MVT::Other, Chain, SysRegNo,
15116 EnvValue);
15117}
15118
15119SDValue RISCVTargetLowering::lowerRESET_FPMODE(SDValue Op,
15120 SelectionDAG &DAG) const {
15121 const MVT XLenVT = Subtarget.getXLenVT();
15122 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15123 SDLoc DL(Op);
15124 SDValue Chain = Op->getOperand(0);
15125 SDValue SysRegNo = DAG.getTargetConstant(RISCVSysReg::fcsr, DL, XLenVT);
15126 SDValue ModeMask = DAG.getConstant(ModeMaskValue, DL, XLenVT);
15127
15128 return DAG.getNode(RISCVISD::CLEAR_CSR, DL, MVT::Other, Chain, SysRegNo,
15129 ModeMask);
15130}
15131
15132SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
15133 SelectionDAG &DAG) const {
15134 MachineFunction &MF = DAG.getMachineFunction();
15135
15136 bool isRISCV64 = Subtarget.is64Bit();
15137 EVT PtrVT = getPointerTy(DAG.getDataLayout());
15138
15139 int FI = MF.getFrameInfo().CreateFixedObject(isRISCV64 ? 8 : 4, 0, false);
15140 return DAG.getFrameIndex(FI, PtrVT);
15141}
15142
15143// Returns the opcode of the target-specific SDNode that implements the 32-bit
15144// form of the given Opcode.
15145static unsigned getRISCVWOpcode(unsigned Opcode) {
15146 switch (Opcode) {
15147 default:
15148 llvm_unreachable("Unexpected opcode");
15149 case ISD::SHL:
15150 return RISCVISD::SLLW;
15151 case ISD::SRA:
15152 return RISCVISD::SRAW;
15153 case ISD::SRL:
15154 return RISCVISD::SRLW;
15155 case ISD::SDIV:
15156 return RISCVISD::DIVW;
15157 case ISD::UDIV:
15158 return RISCVISD::DIVUW;
15159 case ISD::UREM:
15160 return RISCVISD::REMUW;
15161 case ISD::ROTL:
15162 return RISCVISD::ROLW;
15163 case ISD::ROTR:
15164 return RISCVISD::RORW;
15165 }
15166}
15167
15168// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
15169// node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
15170// otherwise be promoted to i64, making it difficult to select the
15171// SLLW/DIVUW/.../*W later one because the fact the operation was originally of
15172// type i8/i16/i32 is lost.
15173static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
15174 unsigned ExtOpc = ISD::ANY_EXTEND) {
15175 SDLoc DL(N);
15176 unsigned WOpcode = getRISCVWOpcode(N->getOpcode());
15177 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
15178 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
15179 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
15180 // ReplaceNodeResults requires we maintain the same type for the return value.
15181 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
15182}
15183
15184// Converts the given 32-bit operation to a i64 operation with signed extension
15185// semantic to reduce the signed extension instructions.
15186static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
15187 SDLoc DL(N);
15188 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15189 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
15190 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
15191 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
15192 DAG.getValueType(MVT::i32));
15193 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
15194}
15195
15196// Zero-extend a 32-bit packed vector to the 64-bit packed type WideVT,
15197// clearing the upper lanes.
15198static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
15199 SDValue V, MVT WideVT) {
15200 SDValue Wide =
15201 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, DAG.getBitcast(MVT::i32, V));
15202 return DAG.getBitcast(WideVT, Wide);
15203}
15204
15205void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
15206 SmallVectorImpl<SDValue> &Results,
15207 SelectionDAG &DAG) const {
15208 SDLoc DL(N);
15209 switch (N->getOpcode()) {
15210 default:
15211 llvm_unreachable("Don't know how to custom type legalize this operation!");
15212 case ISD::STRICT_FP_TO_SINT:
15213 case ISD::STRICT_FP_TO_UINT:
15214 case ISD::FP_TO_SINT:
15215 case ISD::FP_TO_UINT: {
15216 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15217 "Unexpected custom legalisation");
15218 bool IsStrict = N->isStrictFPOpcode();
15219 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
15220 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
15221 SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
15222 if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
15223 TargetLowering::TypeSoftenFloat) {
15224 if (!isTypeLegal(Op0.getValueType()))
15225 return;
15226 if (IsStrict) {
15227 SDValue Chain = N->getOperand(0);
15228 // In absence of Zfh, promote f16 to f32, then convert.
15229 if (Op0.getValueType() == MVT::f16 &&
15230 !Subtarget.hasStdExtZfhOrZhinx()) {
15231 Op0 = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {MVT::f32, MVT::Other},
15232 {Chain, Op0});
15233 Chain = Op0.getValue(1);
15234 }
15235 unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
15236 : RISCVISD::STRICT_FCVT_WU_RV64;
15237 SDVTList VTs = DAG.getVTList(MVT::i64, MVT::Other);
15238 SDValue Res = DAG.getNode(
15239 Opc, DL, VTs, Chain, Op0,
15240 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
15241 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15242 Results.push_back(Res.getValue(1));
15243 return;
15244 }
15245 // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
15246 // convert.
15247 if ((Op0.getValueType() == MVT::f16 &&
15248 !Subtarget.hasStdExtZfhOrZhinx()) ||
15249 Op0.getValueType() == MVT::bf16)
15250 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
15251
15252 unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
15253 SDValue Res =
15254 DAG.getNode(Opc, DL, MVT::i64, Op0,
15255 DAG.getTargetConstant(RISCVFPRndMode::RTZ, DL, MVT::i64));
15256 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15257 return;
15258 }
15259 // If the FP type needs to be softened, emit a library call using the 'si'
15260 // version. If we left it to default legalization we'd end up with 'di'. If
15261 // the FP type doesn't need to be softened just let generic type
15262 // legalization promote the result type.
15263 RTLIB::Libcall LC;
15264 if (IsSigned)
15265 LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
15266 else
15267 LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
15268 MakeLibCallOptions CallOptions;
15269 EVT OpVT = Op0.getValueType();
15270 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0));
15271 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
15272 SDValue Result;
15273 std::tie(Result, Chain) =
15274 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
15275 Results.push_back(Result);
15276 if (IsStrict)
15277 Results.push_back(Chain);
15278 break;
15279 }
15280 case ISD::LROUND: {
15281 SDValue Op0 = N->getOperand(0);
15282 EVT Op0VT = Op0.getValueType();
15283 if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
15284 TargetLowering::TypeSoftenFloat) {
15285 if (!isTypeLegal(Op0VT))
15286 return;
15287
15288 // In absence of Zfh, promote f16 to f32, then convert.
15289 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
15290 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
15291
15292 SDValue Res =
15293 DAG.getNode(RISCVISD::FCVT_W_RV64, DL, MVT::i64, Op0,
15294 DAG.getTargetConstant(RISCVFPRndMode::RMM, DL, MVT::i64));
15295 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15296 return;
15297 }
15298 // If the FP type needs to be softened, emit a library call to lround. We'll
15299 // need to truncate the result. We assume any value that doesn't fit in i32
15300 // is allowed to return an unspecified value.
15301 RTLIB::Libcall LC =
15302 Op0.getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
15303 MakeLibCallOptions CallOptions;
15304 EVT OpVT = Op0.getValueType();
15305 CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64);
15306 SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
15307 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
15308 Results.push_back(Result);
15309 break;
15310 }
15311 case ISD::READCYCLECOUNTER:
15312 case ISD::READSTEADYCOUNTER: {
15313 assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
15314 "has custom type legalization on riscv32");
15315
15316 SDValue LoCounter, HiCounter;
15317 MVT XLenVT = Subtarget.getXLenVT();
15318 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
15319 LoCounter = DAG.getTargetConstant(RISCVSysReg::cycle, DL, XLenVT);
15320 HiCounter = DAG.getTargetConstant(RISCVSysReg::cycleh, DL, XLenVT);
15321 } else {
15322 LoCounter = DAG.getTargetConstant(RISCVSysReg::time, DL, XLenVT);
15323 HiCounter = DAG.getTargetConstant(RISCVSysReg::timeh, DL, XLenVT);
15324 }
15325 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
15326 SDValue RCW = DAG.getNode(RISCVISD::READ_COUNTER_WIDE, DL, VTs,
15327 N->getOperand(0), LoCounter, HiCounter);
15328
15329 Results.push_back(
15330 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
15331 Results.push_back(RCW.getValue(2));
15332 break;
15333 }
15334 case ISD::LOAD: {
15335 if (!ISD::isNON_EXTLoad(N))
15336 return;
15337
15338 // Use a SEXTLOAD instead of the default EXTLOAD. Similar to the
15339 // sext_inreg we emit for ADD/SUB/MUL/SLLI.
15340 LoadSDNode *Ld = cast<LoadSDNode>(N);
15341
15342 if (N->getValueType(0) == MVT::i64) {
15343 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
15344 "Unexpected custom legalisation");
15345
15346 if (Ld->getAlign() < Subtarget.getZilsdAlign())
15347 return;
15348
15349 SDLoc DL(N);
15350 SDValue Result = DAG.getMemIntrinsicNode(
15351 RISCVISD::LD_RV32, DL,
15352 DAG.getVTList({MVT::i32, MVT::i32, MVT::Other}),
15353 {Ld->getChain(), Ld->getBasePtr()}, MVT::i64, Ld->getMemOperand());
15354 SDValue Lo = Result.getValue(0);
15355 SDValue Hi = Result.getValue(1);
15356 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
15357 Results.append({Pair, Result.getValue(2)});
15358 return;
15359 }
15360
15361 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15362 "Unexpected custom legalisation");
15363
15364 SDLoc dl(N);
15365 SDValue Res = DAG.getExtLoad(ISD::SEXTLOAD, dl, MVT::i64, Ld->getChain(),
15366 Ld->getBasePtr(), Ld->getMemoryVT(),
15367 Ld->getMemOperand());
15368 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Res));
15369 Results.push_back(Res.getValue(1));
15370 return;
15371 }
15372 case ISD::MUL: {
15373 unsigned Size = N->getSimpleValueType(0).getSizeInBits();
15374 unsigned XLen = Subtarget.getXLen();
15375 if (Size > XLen) {
15376 // This multiply needs to be expanded, try to use MULH+MUL or WMUL if
15377 // possible. We duplicate the default legalization to
15378 // MULHU/MULHS/UMUL_LOHI/SMUL_LOHI to minimize the number of calls to
15379 // MaskedValueIsZero and ComputeNumSignBits
15380 // FIXME: Should we have a target independent MULHSU/WMULSU node? Are
15381 // there are other targets that could use it?
15382 assert(Size == (XLen * 2) && "Unexpected custom legalisation");
15383
15384 auto MakeMULPair = [&](SDValue L, SDValue R, unsigned HighOpc,
15385 unsigned LoHiOpc) {
15386 MVT XLenVT = Subtarget.getXLenVT();
15387 L = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, L);
15388 R = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, R);
15389 SDValue Lo, Hi;
15390 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
15391 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
15392 Lo = DAG.getNode(LoHiOpc, DL, VTs, L, R);
15393 Hi = Lo.getValue(1);
15394 } else {
15395 Lo = DAG.getNode(ISD::MUL, DL, XLenVT, L, R);
15396 Hi = DAG.getNode(HighOpc, DL, XLenVT, L, R);
15397 }
15398 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
15399 };
15400
15401 SDValue LHS = N->getOperand(0);
15402 SDValue RHS = N->getOperand(1);
15403
15404 APInt HighMask = APInt::getHighBitsSet(Size, XLen);
15405 bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
15406 bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
15407 if (LHSIsU && RHSIsU) {
15408 Results.push_back(MakeMULPair(LHS, RHS, ISD::MULHU, ISD::UMUL_LOHI));
15409 return;
15410 }
15411
15412 bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
15413 bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
15414 if (LHSIsS && RHSIsS)
15415 Results.push_back(MakeMULPair(LHS, RHS, ISD::MULHS, ISD::SMUL_LOHI));
15416 else if (RHSIsU && LHSIsS)
15417 Results.push_back(
15418 MakeMULPair(LHS, RHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15419 else if (LHSIsU && RHSIsS)
15420 Results.push_back(
15421 MakeMULPair(RHS, LHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15422
15423 return;
15424 }
15425 [[fallthrough]];
15426 }
15427 case ISD::ADD:
15428 case ISD::SUB:
15429 if (N->getValueType(0) == MVT::i64) {
15430 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15431 "Unexpected custom legalisation");
15432
15433 // Expand to ADDD/SUBD.
15434 auto [LHSLo, LHSHi] =
15435 DAG.SplitScalar(N->getOperand(0), DL, MVT::i32, MVT::i32);
15436 auto [RHSLo, RHSHi] =
15437 DAG.SplitScalar(N->getOperand(1), DL, MVT::i32, MVT::i32);
15438 unsigned Opc =
15439 N->getOpcode() == ISD::ADD ? RISCVISD::ADDD : RISCVISD::SUBD;
15440 SDValue Res = DAG.getNode(Opc, DL, DAG.getVTList(MVT::i32, MVT::i32),
15441 LHSLo, LHSHi, RHSLo, RHSHi);
15442 Res = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Res, Res.getValue(1));
15443 Results.push_back(Res);
15444 return;
15445 }
15446
15447 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15448 "Unexpected custom legalisation");
15449 Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
15450 break;
15451 case ISD::SHL:
15452 case ISD::SRA:
15453 case ISD::SRL: {
15454 EVT VT = N->getValueType(0);
15455 if (VT.isFixedLengthVector() && Subtarget.hasStdExtP()) {
15456 assert(Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
15457 "Unexpected vector type for P-extension shift");
15458
15459 // If shift amount is a splat, don't scalarize - let normal widening
15460 // and SIMD patterns handle it (pslli.h, psrli.h, etc.)
15461 SDValue ShiftAmt = N->getOperand(1);
15462 if (DAG.isSplatValue(ShiftAmt, /*AllowUndefs=*/true))
15463 break;
15464
15465 EVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT);
15466 unsigned WidenNumElts = WidenVT.getVectorNumElements();
15467 // Unroll with OrigNumElts operations, padding result to WidenNumElts
15468 SDValue Res = DAG.UnrollVectorOp(N, WidenNumElts);
15469 Results.push_back(Res);
15470 break;
15471 }
15472
15473 if (VT == MVT::i64) {
15474 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15475 "Unexpected custom legalisation");
15476
15477 SDValue LHS = N->getOperand(0);
15478 SDValue ShAmt = N->getOperand(1);
15479
15480 unsigned WideOpc = 0;
15481 APInt HighMask = APInt::getHighBitsSet(64, 32);
15482 if (DAG.MaskedValueIsZero(LHS, HighMask))
15483 WideOpc = RISCVISD::WSLL;
15484 else if (DAG.ComputeMaxSignificantBits(LHS) <= 32)
15485 WideOpc = RISCVISD::WSLA;
15486
15487 if (WideOpc) {
15488 SDValue Res =
15489 DAG.getNode(WideOpc, DL, DAG.getVTList(MVT::i32, MVT::i32),
15490 DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, LHS),
15491 DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, ShAmt));
15492 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0),
15493 Res, Res.getValue(1)));
15494 return;
15495 }
15496
15497 // Only handle constant shifts < 32. Non-constant shifts are handled by
15498 // lowerShiftLeftParts/lowerShiftRightParts, and shifts >= 32 use default
15499 // legalization.
15500 auto *ShAmtC = dyn_cast<ConstantSDNode>(N->getOperand(1));
15501 if (!ShAmtC || ShAmtC->getZExtValue() >= 32)
15502 break;
15503
15504 auto [Lo, Hi] = DAG.SplitScalar(LHS, DL, MVT::i32, MVT::i32);
15505
15506 // If the shift amount operand is coming from a vector legalization it may
15507 // have an illegal type.
15508 if (ShAmt.getValueType() != MVT::i32)
15509 ShAmt = DAG.getZExtOrTrunc(ShAmt, DL, MVT::i32);
15510
15511 SDValue LoRes, HiRes;
15512 if (N->getOpcode() == ISD::SHL) {
15513 // Lo = slli Lo, shamt
15514 // Hi = nsrli {Hi, Lo}, (32 - shamt)
15515 uint64_t ShAmtVal = ShAmtC->getZExtValue();
15516 LoRes = DAG.getNode(ISD::SHL, DL, MVT::i32, Lo, ShAmt);
15517 HiRes = DAG.getNode(RISCVISD::NSRL, DL, MVT::i32, Lo, Hi,
15518 DAG.getConstant(32 - ShAmtVal, DL, MVT::i32));
15519 } else {
15520 bool IsSRA = N->getOpcode() == ISD::SRA;
15521 LoRes = DAG.getNode(IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL,
15522 MVT::i32, Lo, Hi, ShAmt);
15523 HiRes =
15524 DAG.getNode(IsSRA ? ISD::SRA : ISD::SRL, DL, MVT::i32, Hi, ShAmt);
15525 }
15526 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, LoRes, HiRes);
15527 Results.push_back(Res);
15528 return;
15529 }
15530
15531 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
15532 "Unexpected custom legalisation");
15533 if (N->getOperand(1).getOpcode() != ISD::Constant) {
15534 // If we can use a BSET instruction, allow default promotion to apply.
15535 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
15536 isOneConstant(N->getOperand(0)))
15537 break;
15538 Results.push_back(customLegalizeToWOp(N, DAG));
15539 break;
15540 }
15541
15542 // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
15543 // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
15544 // shift amount.
15545 if (N->getOpcode() == ISD::SHL) {
15546 SDLoc DL(N);
15547 SDValue NewOp0 =
15548 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15549 SDValue NewOp1 =
15550 DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
15551 SDValue NewWOp = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0, NewOp1);
15552 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
15553 DAG.getValueType(MVT::i32));
15554 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
15555 }
15556
15557 break;
15558 }
15559 case ISD::ROTL:
15560 case ISD::ROTR:
15561 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15562 "Unexpected custom legalisation");
15563 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
15564 Subtarget.hasVendorXTHeadBb()) &&
15565 "Unexpected custom legalization");
15566 if (!isa<ConstantSDNode>(N->getOperand(1)) &&
15567 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
15568 return;
15569 Results.push_back(customLegalizeToWOp(N, DAG));
15570 break;
15571 case ISD::CTTZ:
15572 case ISD::CTTZ_ZERO_POISON:
15573 case ISD::CTLZ:
15574 case ISD::CTLZ_ZERO_POISON:
15575 case ISD::CTLS: {
15576 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15577 "Unexpected custom legalisation");
15578
15579 SDValue NewOp0 =
15580 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15581 unsigned Opc;
15582 switch (N->getOpcode()) {
15583 default: llvm_unreachable("Unexpected opcode");
15584 case ISD::CTTZ:
15585 case ISD::CTTZ_ZERO_POISON:
15586 Opc = RISCVISD::CTZW;
15587 break;
15588 case ISD::CTLZ:
15589 case ISD::CTLZ_ZERO_POISON:
15590 Opc = RISCVISD::CLZW;
15591 break;
15592 case ISD::CTLS:
15593 Opc = RISCVISD::CLSW;
15594 break;
15595 }
15596
15597 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
15598 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15599 return;
15600 }
15601 case ISD::SDIV:
15602 case ISD::UDIV:
15603 case ISD::UREM: {
15604 MVT VT = N->getSimpleValueType(0);
15605 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
15606 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
15607 "Unexpected custom legalisation");
15608 // Don't promote division/remainder by constant since we should expand those
15609 // to multiply by magic constant.
15610 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
15611 if (N->getOperand(1).getOpcode() == ISD::Constant &&
15612 !isIntDivCheap(N->getValueType(0), Attr))
15613 return;
15614
15615 // If the input is i32, use ANY_EXTEND since the W instructions don't read
15616 // the upper 32 bits. For other types we need to sign or zero extend
15617 // based on the opcode.
15618 unsigned ExtOpc = ISD::ANY_EXTEND;
15619 if (VT != MVT::i32)
15620 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
15621 : ISD::ZERO_EXTEND;
15622
15623 Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
15624 break;
15625 }
15626 case ISD::SADDO:
15627 case ISD::SSUBO: {
15628 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15629 "Unexpected custom legalisation");
15630
15631 // This is similar to the default legalization, but we return the
15632 // sext_inreg instead of the add/sub.
15633 bool IsAdd = N->getOpcode() == ISD::SADDO;
15634 SDValue LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
15635 SDValue RHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
15636 SDValue Op =
15637 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
15638 SDValue Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Op,
15639 DAG.getValueType(MVT::i32));
15640
15641 SDValue Overflow;
15642
15643 // If the RHS is a constant, we can simplify ConditionRHS below. Otherwise
15644 // use the default legalization.
15645 if (IsAdd && isa<ConstantSDNode>(N->getOperand(1))) {
15646 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
15647
15648 // For an addition, the result should be less than one of the operands
15649 // (LHS) if and only if the other operand (RHS) is negative, otherwise
15650 // there will be overflow.
15651 EVT OType = N->getValueType(1);
15652 SDValue ResultLowerThanLHS =
15653 DAG.getSetCC(DL, OType, Res, LHS, ISD::SETLT);
15654 SDValue ConditionRHS = DAG.getSetCC(DL, OType, RHS, Zero, ISD::SETLT);
15655
15656 Overflow =
15657 DAG.getNode(ISD::XOR, DL, OType, ConditionRHS, ResultLowerThanLHS);
15658 } else {
15659 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, Op, ISD::SETNE);
15660 }
15661
15662 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15663 Results.push_back(Overflow);
15664 return;
15665 }
15666 case ISD::UADDO:
15667 case ISD::USUBO: {
15668 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15669 "Unexpected custom legalisation");
15670 bool IsAdd = N->getOpcode() == ISD::UADDO;
15671 // Create an ADDW or SUBW.
15672 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15673 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
15674 SDValue Res =
15675 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
15676 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
15677 DAG.getValueType(MVT::i32));
15678
15679 SDValue Overflow;
15680 if (IsAdd && isOneConstant(RHS)) {
15681 // Special case uaddo X, 1 overflowed if the addition result is 0.
15682 // The general case (X + C) < C is not necessarily beneficial. Although we
15683 // reduce the live range of X, we may introduce the materialization of
15684 // constant C, especially when the setcc result is used by branch. We have
15685 // no compare with constant and branch instructions.
15686 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res,
15687 DAG.getConstant(0, DL, MVT::i64), ISD::SETEQ);
15688 } else if (IsAdd && isAllOnesConstant(RHS)) {
15689 // Special case uaddo X, -1 overflowed if X != 0.
15690 Overflow = DAG.getSetCC(DL, N->getValueType(1), N->getOperand(0),
15691 DAG.getConstant(0, DL, MVT::i32), ISD::SETNE);
15692 } else {
15693 // Sign extend the LHS and perform an unsigned compare with the ADDW
15694 // result. Since the inputs are sign extended from i32, this is equivalent
15695 // to comparing the lower 32 bits.
15696 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
15697 Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
15698 IsAdd ? ISD::SETULT : ISD::SETUGT);
15699 }
15700
15701 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15702 Results.push_back(Overflow);
15703 return;
15704 }
15705 case ISD::UADDSAT:
15706 case ISD::USUBSAT:
15707 case ISD::SADDSAT:
15708 case ISD::SSUBSAT: {
15709 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15710 "Unexpected custom legalisation");
15711
15712 if (Subtarget.hasStdExtP()) {
15713 // On RV64, map scalar i32 saturating add/sub through lane 0 of a packed
15714 // v2i32 operation so we can select ps*.w instructions.
15715 SDValue LHS = DAG.getNode(
15716 ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
15717 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0)));
15718 SDValue RHS = DAG.getNode(
15719 ISD::SCALAR_TO_VECTOR, DL, MVT::v2i32,
15720 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)));
15721 SDValue VecRes = DAG.getNode(N->getOpcode(), DL, MVT::v2i32, LHS, RHS);
15722 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
15723 Results.push_back(
15724 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i32, VecRes, Zero));
15725 return;
15726 }
15727
15728 assert(!Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
15729 Results.push_back(expandAddSubSat(N, DAG));
15730 return;
15731 }
15732 case ISD::ABS:
15733 case ISD::ABS_MIN_POISON: {
15734 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15735 "Unexpected custom legalisation");
15736
15737 if (Subtarget.hasStdExtP()) {
15738 SDValue Src =
15739 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15740 SDValue Abs = DAG.getNode(RISCVISD::ABSW, DL, MVT::i64, Src);
15741 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
15742 return;
15743 }
15744
15745 if (Subtarget.hasStdExtZbb()) {
15746 // Emit a special node that will be expanded to NEGW+MAX at isel.
15747 // This allows us to remember that the result is sign extended. Expanding
15748 // to NEGW+MAX here requires a Freeze which breaks ComputeNumSignBits.
15749 SDValue Src = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64,
15750 N->getOperand(0));
15751 SDValue Abs = DAG.getNode(RISCVISD::NEGW_MAX, DL, MVT::i64, Src);
15752 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Abs));
15753 return;
15754 }
15755
15756 // Expand abs to Y = (sraiw X, 31); subw(xor(X, Y), Y)
15757 SDValue Src = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
15758
15759 // Freeze the source so we can increase it's use count.
15760 Src = DAG.getFreeze(Src);
15761
15762 // Copy sign bit to all bits using the sraiw pattern.
15763 SDValue SignFill = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Src,
15764 DAG.getValueType(MVT::i32));
15765 SignFill = DAG.getNode(ISD::SRA, DL, MVT::i64, SignFill,
15766 DAG.getConstant(31, DL, MVT::i64));
15767
15768 SDValue NewRes = DAG.getNode(ISD::XOR, DL, MVT::i64, Src, SignFill);
15769 NewRes = DAG.getNode(ISD::SUB, DL, MVT::i64, NewRes, SignFill);
15770
15771 // NOTE: The result is only required to be anyextended, but sext is
15772 // consistent with type legalization of sub.
15773 NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewRes,
15774 DAG.getValueType(MVT::i32));
15775 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
15776 return;
15777 }
15778 case ISD::BITCAST: {
15779 EVT VT = N->getValueType(0);
15780 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
15781 SDValue Op0 = N->getOperand(0);
15782 EVT Op0VT = Op0.getValueType();
15783 MVT XLenVT = Subtarget.getXLenVT();
15784 if (VT == MVT::i16 &&
15785 ((Op0VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
15786 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
15787 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
15788 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
15789 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
15790 Subtarget.hasStdExtFOrZfinx()) {
15791 SDValue FPConv =
15792 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
15793 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
15794 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
15795 Subtarget.hasStdExtDOrZdinx()) {
15796 SDValue NewReg = DAG.getNode(RISCVISD::SplitF64, DL,
15797 DAG.getVTList(MVT::i32, MVT::i32), Op0);
15798 SDValue Lo = NewReg.getValue(0);
15799 SDValue Hi = NewReg.getValue(1);
15800 // For big-endian, swap the order when building the i64 pair.
15801 if (!Subtarget.isLittleEndian())
15802 std::swap(Lo, Hi);
15803 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Lo, Hi);
15804 Results.push_back(RetReg);
15805 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
15806 isTypeLegal(Op0VT)) {
15807 // Custom-legalize bitcasts from fixed-length vector types to illegal
15808 // scalar types in order to improve codegen. Bitcast the vector to a
15809 // one-element vector type whose element type is the same as the result
15810 // type, and extract the first element.
15811 EVT BVT = EVT::getVectorVT(*DAG.getContext(), VT, 1);
15812 if (isTypeLegal(BVT)) {
15813 SDValue BVec = DAG.getBitcast(BVT, Op0);
15814 Results.push_back(DAG.getExtractVectorElt(DL, VT, BVec, 0));
15815 }
15816 }
15817 break;
15818 }
15819 case ISD::BITREVERSE: {
15820 assert(N->getValueType(0) == MVT::i8 && Subtarget.hasStdExtZbkb() &&
15821 "Unexpected custom legalisation");
15822 MVT XLenVT = Subtarget.getXLenVT();
15823 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
15824 SDValue NewRes = DAG.getNode(RISCVISD::BREV8, DL, XLenVT, NewOp);
15825 // ReplaceNodeResults requires we maintain the same type for the return
15826 // value.
15827 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, NewRes));
15828 break;
15829 }
15830 case RISCVISD::BREV8:
15831 case RISCVISD::ORC_B: {
15832 MVT VT = N->getSimpleValueType(0);
15833 MVT XLenVT = Subtarget.getXLenVT();
15834 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
15835 "Unexpected custom legalisation");
15836 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
15837 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
15838 "Unexpected extension");
15839 SDValue NewOp = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
15840 SDValue NewRes = DAG.getNode(N->getOpcode(), DL, XLenVT, NewOp);
15841 // ReplaceNodeResults requires we maintain the same type for the return
15842 // value.
15843 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NewRes));
15844 break;
15845 }
15846 case RISCVISD::ASUB:
15847 case RISCVISD::ASUBU:
15848 case RISCVISD::MULHSU:
15849 case RISCVISD::MULHR:
15850 case RISCVISD::MULHRU:
15851 case RISCVISD::MULHRSU: {
15852 MVT VT = N->getSimpleValueType(0);
15853 SDValue Op0 = N->getOperand(0);
15854 SDValue Op1 = N->getOperand(1);
15855 unsigned Opcode = N->getOpcode();
15856 // PMULH* variants don't support i8
15857 [[maybe_unused]] bool IsMulH =
15858 Opcode == RISCVISD::MULHSU || Opcode == RISCVISD::MULHR ||
15859 Opcode == RISCVISD::MULHRU || Opcode == RISCVISD::MULHRSU;
15860 assert(VT == MVT::v2i16 || (!IsMulH && VT == MVT::v4i8));
15861 MVT NewVT = MVT::v4i16;
15862 if (VT == MVT::v4i8)
15863 NewVT = MVT::v8i8;
15864 SDValue Undef = DAG.getUNDEF(VT);
15865 Op0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, {Op0, Undef});
15866 Op1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, NewVT, {Op1, Undef});
15867 Results.push_back(DAG.getNode(Opcode, DL, NewVT, {Op0, Op1}));
15868 return;
15869 }
15870 case ISD::EXTRACT_VECTOR_ELT: {
15871 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
15872 // type is illegal (currently only vXi64 RV32).
15873 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
15874 // transferred to the destination register. We issue two of these from the
15875 // upper- and lower- halves of the SEW-bit vector element, slid down to the
15876 // first element.
15877 SDValue Vec = N->getOperand(0);
15878 SDValue Idx = N->getOperand(1);
15879
15880 // The vector type hasn't been legalized yet so we can't issue target
15881 // specific nodes if it needs legalization.
15882 // FIXME: We would manually legalize if it's important.
15883 if (!isTypeLegal(Vec.getValueType()))
15884 return;
15885
15886 MVT VecVT = Vec.getSimpleValueType();
15887
15888 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
15889 VecVT.getVectorElementType() == MVT::i64 &&
15890 "Unexpected EXTRACT_VECTOR_ELT legalization");
15891
15892 // If this is a fixed vector, we need to convert it to a scalable vector.
15893 MVT ContainerVT = VecVT;
15894 if (VecVT.isFixedLengthVector()) {
15895 ContainerVT = getContainerForFixedLengthVector(VecVT);
15896 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
15897 }
15898
15899 MVT XLenVT = Subtarget.getXLenVT();
15900
15901 // Use a VL of 1 to avoid processing more elements than we need.
15902 auto [Mask, VL] = getDefaultVLOps(1, ContainerVT, DL, DAG, Subtarget);
15903
15904 // Unless the index is known to be 0, we must slide the vector down to get
15905 // the desired element into index 0.
15906 if (!isNullConstant(Idx)) {
15907 Vec = getVSlidedown(DAG, Subtarget, DL, ContainerVT,
15908 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
15909 }
15910
15911 // Extract the lower XLEN bits of the correct vector element.
15912 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
15913
15914 // To extract the upper XLEN bits of the vector element, shift the first
15915 // element right by 32 bits and re-extract the lower XLEN bits.
15916 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
15917 DAG.getUNDEF(ContainerVT),
15918 DAG.getConstant(32, DL, XLenVT), VL);
15919 SDValue LShr32 =
15920 DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec, ThirtyTwoV,
15921 DAG.getUNDEF(ContainerVT), Mask, VL);
15922
15923 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
15924
15925 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
15926 break;
15927 }
15928 case ISD::INTRINSIC_WO_CHAIN: {
15929 unsigned IntNo = N->getConstantOperandVal(0);
15930 switch (IntNo) {
15931 default:
15932 llvm_unreachable(
15933 "Don't know how to custom type legalize this intrinsic!");
15934 case Intrinsic::experimental_get_vector_length: {
15935 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
15936 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
15937 return;
15938 }
15939 case Intrinsic::riscv_paadd:
15940 case Intrinsic::riscv_paaddu:
15941 case Intrinsic::riscv_pasub:
15942 case Intrinsic::riscv_pasubu:
15943 case Intrinsic::riscv_pabd:
15944 case Intrinsic::riscv_pabdu:
15945 case Intrinsic::riscv_pas:
15946 case Intrinsic::riscv_psa:
15947 case Intrinsic::riscv_psas:
15948 case Intrinsic::riscv_pssa:
15949 case Intrinsic::riscv_paas:
15950 case Intrinsic::riscv_pasa:
15951 case Intrinsic::riscv_pmerge:
15952 case Intrinsic::riscv_psabs: {
15953 EVT VT = N->getValueType(0);
15954 if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
15955 return;
15956
15957 unsigned Opc;
15958 switch (IntNo) {
15959 case Intrinsic::riscv_paadd:
15960 Opc = ISD::AVGFLOORS;
15961 break;
15962 case Intrinsic::riscv_paaddu:
15963 Opc = ISD::AVGFLOORU;
15964 break;
15965 case Intrinsic::riscv_pasub:
15966 Opc = RISCVISD::ASUB;
15967 break;
15968 case Intrinsic::riscv_pasubu:
15969 Opc = RISCVISD::ASUBU;
15970 break;
15971 case Intrinsic::riscv_pabd:
15972 Opc = ISD::ABDS;
15973 break;
15974 case Intrinsic::riscv_pabdu:
15975 Opc = ISD::ABDU;
15976 break;
15977 case Intrinsic::riscv_psabs:
15978 Opc = RISCVISD::PSABS;
15979 break;
15980 default:
15981 // pas/psa/psas/pssa/paas/pasa and pmerge: re-emit at the widened type
15982 // rather than lowering to a generic node.
15983 Opc = ISD::INTRINSIC_WO_CHAIN;
15984 break;
15985 }
15986
15987 EVT WideVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
15988 SDValue Undef = DAG.getUNDEF(VT);
15989 SmallVector<SDValue, 4> Ops(N->ops());
15990 for (SDValue &Op : Ops) {
15991 if (Op.getValueType() == VT)
15992 Op = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideVT, Op, Undef);
15993 }
15994 SDValue Res;
15995 if (Opc == ISD::INTRINSIC_WO_CHAIN)
15996 Res = DAG.getNode(Opc, DL, WideVT, Ops);
15997 else
15998 Res = DAG.getNode(Opc, DL, WideVT, ArrayRef(Ops).slice(1));
15999 Results.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
16000 DAG.getVectorIdxConstant(0, DL)));
16001 return;
16002 }
16003 case Intrinsic::riscv_predsum:
16004 case Intrinsic::riscv_predsumu: {
16005 bool IsSigned = IntNo == Intrinsic::riscv_predsum;
16006 SDValue Vec = N->getOperand(1);
16007 MVT VecVT = Vec.getSimpleValueType();
16008 auto Ext = [&](SDValue V) {
16009 return DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
16010 MVT::i64, V);
16011 };
16012 auto RedSum = [&](MVT VT, SDValue V, SDValue Acc) {
16013 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, VT, N->getOperand(0), V,
16014 Acc);
16015 };
16016
16017 // RV32: i64 accumulator. Reduce to a 32-bit partial sum, then
16018 // widening-accumulate into i64 via wadda/waddau (v2i32 uses wadda alone).
16019 if (!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64) {
16020 SDValue Acc = N->getOperand(2);
16021 SDValue Res;
16022 if (VecVT == MVT::v2i32) {
16023 Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc,
16024 Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 0)));
16025 Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Res,
16026 Ext(DAG.getExtractVectorElt(DL, MVT::i32, Vec, 1)));
16027 } else {
16028 // The paired predsum.dbs/dhs computes the 32-bit element sum.
16029 SDValue Partial =
16030 RedSum(MVT::i32, Vec, DAG.getConstant(0, DL, MVT::i32));
16031 Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Acc, Ext(Partial));
16032 }
16033 Results.push_back(Res);
16034 return;
16035 }
16036
16037 // RV64: i32 accumulator. Reduce at i64 (XLEN), then truncate.
16038 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16039 return;
16040
16041 // Zero the upper lanes (zext.w) so they don't contribute to the sum.
16042 if (VecVT == MVT::v4i8 || VecVT == MVT::v2i16)
16043 Vec = widenPackedVectorWithZeros(
16044 DAG, DL, Vec, VecVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16);
16045
16046 // The result is truncated to i32, so the accumulator's upper bits are
16047 // unused and need no sign/zero extension.
16048 SDValue Acc =
16049 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
16050 SDValue Res = RedSum(MVT::i64, Vec, Acc);
16051 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16052 return;
16053 }
16054 case Intrinsic::riscv_pabdsumu:
16055 case Intrinsic::riscv_pabdsumau: {
16056 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
16057 // The two packed sources (rs1, rs2) are the last two operands.
16058 SDValue Rs1 = N->getOperand(N->getNumOperands() - 2);
16059 SDValue Rs2 = N->getOperand(N->getNumOperands() - 1);
16060 MVT VecVT = Rs1.getSimpleValueType();
16061
16062 // RV32: i64 result, always from a v8i8 source. The accumulator, if any,
16063 // folds into the widening add below.
16064 if (!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64) {
16065 // Sum of absolute differences of two v4i8 halves.
16066 auto Sad = [&](SDValue A, SDValue B) {
16067 SDValue S = DAG.getNode(
16068 ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
16069 DAG.getTargetConstant(Intrinsic::riscv_pabdsumu, DL, MVT::i32), A,
16070 B);
16071 return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, S);
16072 };
16073 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(Rs1, DL);
16074 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(Rs2, DL);
16075 SDValue Lo = Sad(Rs1Lo, Rs2Lo);
16076 SDValue Hi = Sad(Rs1Hi, Rs2Hi);
16077 // (acc + lo) + hi keeps the accumulate chained so it folds into a
16078 // single waddau; without an accumulator lo + hi folds into waddu.
16079 SDValue Res =
16080 IsAcc ? DAG.getNode(ISD::ADD, DL, MVT::i64, N->getOperand(1), Lo)
16081 : Lo;
16082 Res = DAG.getNode(ISD::ADD, DL, MVT::i64, Res, Hi);
16083 Results.push_back(Res);
16084 return;
16085 }
16086
16087 // RV64: i32 result, so reduce at i64 and truncate. The source is v4i8 or
16088 // v8i8; widen a v4i8 to v8i8, zeroing the upper bytes (v8i8 is legal).
16089 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16090 return;
16091 if (VecVT == MVT::v4i8) {
16092 Rs1 = widenPackedVectorWithZeros(DAG, DL, Rs1, MVT::v8i8);
16093 Rs2 = widenPackedVectorWithZeros(DAG, DL, Rs2, MVT::v8i8);
16094 }
16095 SmallVector<SDValue, 4> Ops = {N->getOperand(0)};
16096 if (IsAcc)
16097 Ops.push_back(
16098 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1)));
16099 Ops.push_back(Rs1);
16100 Ops.push_back(Rs2);
16101 SDValue Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, Ops);
16102 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16103 return;
16104 }
16105 case Intrinsic::riscv_orc_b:
16106 case Intrinsic::riscv_brev8:
16107 case Intrinsic::riscv_sha256sig0:
16108 case Intrinsic::riscv_sha256sig1:
16109 case Intrinsic::riscv_sha256sum0:
16110 case Intrinsic::riscv_sha256sum1:
16111 case Intrinsic::riscv_sm3p0:
16112 case Intrinsic::riscv_sm3p1: {
16113 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16114 return;
16115 unsigned Opc;
16116 switch (IntNo) {
16117 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
16118 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
16119 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
16120 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
16121 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
16122 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
16123 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
16124 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
16125 }
16126
16127 SDValue NewOp =
16128 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
16129 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp);
16130 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16131 return;
16132 }
16133 case Intrinsic::riscv_sm4ks:
16134 case Intrinsic::riscv_sm4ed: {
16135 unsigned Opc =
16136 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
16137 SDValue NewOp0 =
16138 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
16139 SDValue NewOp1 =
16140 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
16141 SDValue Res =
16142 DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, N->getOperand(3));
16143 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16144 return;
16145 }
16146 case Intrinsic::riscv_mopr: {
16147 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16148 return;
16149 SDValue NewOp =
16150 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
16151 SDValue Res = DAG.getNode(
16152 RISCVISD::MOP_R, DL, MVT::i64, NewOp,
16153 DAG.getTargetConstant(N->getConstantOperandVal(2), DL, MVT::i64));
16154 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16155 return;
16156 }
16157 case Intrinsic::riscv_moprr: {
16158 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16159 return;
16160 SDValue NewOp0 =
16161 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
16162 SDValue NewOp1 =
16163 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
16164 SDValue Res = DAG.getNode(
16165 RISCVISD::MOP_RR, DL, MVT::i64, NewOp0, NewOp1,
16166 DAG.getTargetConstant(N->getConstantOperandVal(3), DL, MVT::i64));
16167 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16168 return;
16169 }
16170 case Intrinsic::riscv_clmulh:
16171 case Intrinsic::riscv_clmulr: {
16172 if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
16173 return;
16174
16175 // Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
16176 // to the full 128-bit clmul result of multiplying two xlen values.
16177 // Perform clmulr or clmulh on the shifted values. Finally, extract the
16178 // upper 32 bits.
16179 //
16180 // The alternative is to mask the inputs to 32 bits and use clmul, but
16181 // that requires two shifts to mask each input without zext.w.
16182 // FIXME: If the inputs are known zero extended or could be freely
16183 // zero extended, the mask form would be better.
16184 SDValue NewOp0 =
16185 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
16186 SDValue NewOp1 =
16187 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
16188 NewOp0 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp0,
16189 DAG.getConstant(32, DL, MVT::i64));
16190 NewOp1 = DAG.getNode(ISD::SHL, DL, MVT::i64, NewOp1,
16191 DAG.getConstant(32, DL, MVT::i64));
16192 unsigned Opc =
16193 IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
16194 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1);
16195 Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
16196 DAG.getConstant(32, DL, MVT::i64));
16197 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
16198 return;
16199 }
16200 case Intrinsic::riscv_vmv_x_s: {
16201 EVT VT = N->getValueType(0);
16202 MVT XLenVT = Subtarget.getXLenVT();
16203 if (VT.bitsLT(XLenVT)) {
16204 // Simple case just extract using vmv.x.s and truncate.
16205 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
16206 Subtarget.getXLenVT(), N->getOperand(1));
16207 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
16208 return;
16209 }
16210
16211 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
16212 "Unexpected custom legalization");
16213
16214 // We need to do the move in two steps.
16215 SDValue Vec = N->getOperand(1);
16216 MVT VecVT = Vec.getSimpleValueType();
16217
16218 // First extract the lower XLEN bits of the element.
16219 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
16220
16221 // To extract the upper XLEN bits of the vector element, shift the first
16222 // element right by 32 bits and re-extract the lower XLEN bits.
16223 auto [Mask, VL] = getDefaultVLOps(1, VecVT, DL, DAG, Subtarget);
16224
16225 SDValue ThirtyTwoV =
16226 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT, DAG.getUNDEF(VecVT),
16227 DAG.getConstant(32, DL, XLenVT), VL);
16228 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV,
16229 DAG.getUNDEF(VecVT), Mask, VL);
16230 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
16231
16232 Results.push_back(
16233 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
16234 break;
16235 }
16236 }
16237 break;
16238 }
16239 case ISD::VECREDUCE_ADD:
16240 case ISD::VECREDUCE_AND:
16241 case ISD::VECREDUCE_OR:
16242 case ISD::VECREDUCE_XOR:
16243 case ISD::VECREDUCE_SMAX:
16244 case ISD::VECREDUCE_UMAX:
16245 case ISD::VECREDUCE_SMIN:
16246 case ISD::VECREDUCE_UMIN:
16247 if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
16248 Results.push_back(V);
16249 break;
16250 case ISD::VP_REDUCE_ADD:
16251 case ISD::VP_REDUCE_AND:
16252 case ISD::VP_REDUCE_OR:
16253 case ISD::VP_REDUCE_XOR:
16254 case ISD::VP_REDUCE_SMAX:
16255 case ISD::VP_REDUCE_UMAX:
16256 case ISD::VP_REDUCE_SMIN:
16257 case ISD::VP_REDUCE_UMIN:
16258 if (SDValue V = lowerVPREDUCE(SDValue(N, 0), DAG))
16259 Results.push_back(V);
16260 break;
16261 case ISD::GET_ROUNDING: {
16262 SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
16263 SDValue Res = DAG.getNode(ISD::GET_ROUNDING, DL, VTs, N->getOperand(0));
16264 Results.push_back(Res.getValue(0));
16265 Results.push_back(Res.getValue(1));
16266 break;
16267 }
16268 }
16269}
16270
16271/// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
16272/// which corresponds to it.
16273static unsigned getVecReduceOpcode(unsigned Opc) {
16274 switch (Opc) {
16275 default:
16276 llvm_unreachable("Unhandled binary to transform reduction");
16277 case ISD::ADD:
16278 return ISD::VECREDUCE_ADD;
16279 case ISD::UMAX:
16280 return ISD::VECREDUCE_UMAX;
16281 case ISD::SMAX:
16282 return ISD::VECREDUCE_SMAX;
16283 case ISD::UMIN:
16284 return ISD::VECREDUCE_UMIN;
16285 case ISD::SMIN:
16286 return ISD::VECREDUCE_SMIN;
16287 case ISD::AND:
16288 return ISD::VECREDUCE_AND;
16289 case ISD::OR:
16290 return ISD::VECREDUCE_OR;
16291 case ISD::XOR:
16292 return ISD::VECREDUCE_XOR;
16293 case ISD::FADD:
16294 // Note: This is the associative form of the generic reduction opcode.
16295 return ISD::VECREDUCE_FADD;
16296 case ISD::FMAXNUM:
16297 return ISD::VECREDUCE_FMAX;
16298 case ISD::FMINNUM:
16299 return ISD::VECREDUCE_FMIN;
16300 }
16301}
16302
16303/// Perform two related transforms whose purpose is to incrementally recognize
16304/// an explode_vector followed by scalar reduction as a vector reduction node.
16305/// This exists to recover from a deficiency in SLP which can't handle
16306/// forests with multiple roots sharing common nodes. In some cases, one
16307/// of the trees will be vectorized, and the other will remain (unprofitably)
16308/// scalarized.
16309static SDValue
16310combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
16311 const RISCVSubtarget &Subtarget) {
16312
16313 // This transforms need to run before all integer types have been legalized
16314 // to i64 (so that the vector element type matches the add type), and while
16315 // it's safe to introduce odd sized vector types.
16316 if (DAG.NewNodesMustHaveLegalTypes)
16317 return SDValue();
16318
16319 // Without V, this transform isn't useful. We could form the (illegal)
16320 // operations and let them be scalarized again, but there's really no point.
16321 if (!Subtarget.hasVInstructions())
16322 return SDValue();
16323
16324 const SDLoc DL(N);
16325 const EVT VT = N->getValueType(0);
16326 const unsigned Opc = N->getOpcode();
16327
16328 if (!VT.isInteger()) {
16329 switch (Opc) {
16330 default:
16331 return SDValue();
16332 case ISD::FADD:
16333 // For FADD, we only handle the case with reassociation allowed. We
16334 // could handle strict reduction order, but at the moment, there's no
16335 // known reason to, and the complexity isn't worth it.
16336 if (!N->getFlags().hasAllowReassociation())
16337 return SDValue();
16338 break;
16339 case ISD::FMAXNUM:
16340 case ISD::FMINNUM:
16341 break;
16342 }
16343 }
16344
16345 const unsigned ReduceOpc = getVecReduceOpcode(Opc);
16346 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
16347 "Inconsistent mappings");
16348 SDValue LHS = N->getOperand(0);
16349 SDValue RHS = N->getOperand(1);
16350
16351 if (!LHS.hasOneUse() || !RHS.hasOneUse())
16352 return SDValue();
16353
16354 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16355 std::swap(LHS, RHS);
16356
16357 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16358 !isa<ConstantSDNode>(RHS.getOperand(1)))
16359 return SDValue();
16360
16361 uint64_t RHSIdx = cast<ConstantSDNode>(RHS.getOperand(1))->getLimitedValue();
16362 SDValue SrcVec = RHS.getOperand(0);
16363 EVT SrcVecVT = SrcVec.getValueType();
16364 assert(SrcVecVT.getVectorElementType() == VT);
16365 if (SrcVecVT.isScalableVector())
16366 return SDValue();
16367
16368 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen())
16369 return SDValue();
16370
16371 // match binop (extract_vector_elt V, 0), (extract_vector_elt V, 1) to
16372 // reduce_op (extract_subvector [2 x VT] from V). This will form the
16373 // root of our reduction tree. TODO: We could extend this to any two
16374 // adjacent aligned constant indices if desired.
16375 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16376 LHS.getOperand(0) == SrcVec && isa<ConstantSDNode>(LHS.getOperand(1))) {
16377 uint64_t LHSIdx =
16378 cast<ConstantSDNode>(LHS.getOperand(1))->getLimitedValue();
16379 if (0 == std::min(LHSIdx, RHSIdx) && 1 == std::max(LHSIdx, RHSIdx)) {
16380 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, 2);
16381 SDValue Vec = DAG.getExtractSubvector(DL, ReduceVT, SrcVec, 0);
16382 return DAG.getNode(ReduceOpc, DL, VT, Vec, N->getFlags());
16383 }
16384 }
16385
16386 // Match (binop (reduce (extract_subvector V, 0),
16387 // (extract_vector_elt V, sizeof(SubVec))))
16388 // into a reduction of one more element from the original vector V.
16389 if (LHS.getOpcode() != ReduceOpc)
16390 return SDValue();
16391
16392 SDValue ReduceVec = LHS.getOperand(0);
16393 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16394 ReduceVec.hasOneUse() && ReduceVec.getOperand(0) == RHS.getOperand(0) &&
16395 isNullConstant(ReduceVec.getOperand(1)) &&
16396 ReduceVec.getValueType().getVectorNumElements() == RHSIdx) {
16397 // For illegal types (e.g. 3xi32), most will be combined again into a
16398 // wider (hopefully legal) type. If this is a terminal state, we are
16399 // relying on type legalization here to produce something reasonable
16400 // and this lowering quality could probably be improved. (TODO)
16401 EVT ReduceVT = EVT::getVectorVT(*DAG.getContext(), VT, RHSIdx + 1);
16402 SDValue Vec = DAG.getExtractSubvector(DL, ReduceVT, SrcVec, 0);
16403 return DAG.getNode(ReduceOpc, DL, VT, Vec,
16404 ReduceVec->getFlags() & N->getFlags());
16405 }
16406
16407 return SDValue();
16408}
16409
16410
16411// Try to fold (<bop> x, (reduction.<bop> vec, start))
16412static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
16413 const RISCVSubtarget &Subtarget) {
16414 auto BinOpToRVVReduce = [](unsigned Opc) {
16415 switch (Opc) {
16416 default:
16417 llvm_unreachable("Unhandled binary to transform reduction");
16418 case ISD::ADD:
16419 return RISCVISD::VECREDUCE_ADD_VL;
16420 case ISD::UMAX:
16421 return RISCVISD::VECREDUCE_UMAX_VL;
16422 case ISD::SMAX:
16423 return RISCVISD::VECREDUCE_SMAX_VL;
16424 case ISD::UMIN:
16425 return RISCVISD::VECREDUCE_UMIN_VL;
16426 case ISD::SMIN:
16427 return RISCVISD::VECREDUCE_SMIN_VL;
16428 case ISD::AND:
16429 return RISCVISD::VECREDUCE_AND_VL;
16430 case ISD::OR:
16431 return RISCVISD::VECREDUCE_OR_VL;
16432 case ISD::XOR:
16433 return RISCVISD::VECREDUCE_XOR_VL;
16434 case ISD::FADD:
16435 return RISCVISD::VECREDUCE_FADD_VL;
16436 case ISD::FMAXNUM:
16437 return RISCVISD::VECREDUCE_FMAX_VL;
16438 case ISD::FMINNUM:
16439 return RISCVISD::VECREDUCE_FMIN_VL;
16440 }
16441 };
16442
16443 auto IsReduction = [&BinOpToRVVReduce](SDValue V, unsigned Opc) {
16444 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16445 isNullConstant(V.getOperand(1)) &&
16446 V.getOperand(0).getOpcode() == BinOpToRVVReduce(Opc);
16447 };
16448
16449 unsigned Opc = N->getOpcode();
16450 unsigned ReduceIdx;
16451 if (IsReduction(N->getOperand(0), Opc))
16452 ReduceIdx = 0;
16453 else if (IsReduction(N->getOperand(1), Opc))
16454 ReduceIdx = 1;
16455 else
16456 return SDValue();
16457
16458 // Skip if FADD disallows reassociation but the combiner needs.
16459 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
16460 return SDValue();
16461
16462 SDValue Extract = N->getOperand(ReduceIdx);
16463 SDValue Reduce = Extract.getOperand(0);
16464 if (!Extract.hasOneUse() || !Reduce.hasOneUse())
16465 return SDValue();
16466
16467 SDValue ScalarV = Reduce.getOperand(2);
16468 EVT ScalarVT = ScalarV.getValueType();
16469 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
16470 ScalarV.getOperand(0)->isUndef() &&
16471 isNullConstant(ScalarV.getOperand(2)))
16472 ScalarV = ScalarV.getOperand(1);
16473
16474 // Make sure that ScalarV is a splat with VL=1.
16475 if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
16476 ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
16477 ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
16478 return SDValue();
16479
16480 if (!isNonZeroAVL(ScalarV.getOperand(2)))
16481 return SDValue();
16482
16483 // Check the scalar of ScalarV is neutral element
16484 // TODO: Deal with value other than neutral element.
16485 if (!DAG.isIdentityElement(N->getOpcode(), N->getFlags(),
16486 ScalarV.getOperand(1), 0))
16487 return SDValue();
16488
16489 // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
16490 // FIXME: We might be able to improve this if operand 0 is undef.
16491 if (!isNonZeroAVL(Reduce.getOperand(5)))
16492 return SDValue();
16493
16494 SDValue NewStart = N->getOperand(1 - ReduceIdx);
16495
16496 SDLoc DL(N);
16497 SDValue NewScalarV =
16498 lowerScalarInsert(NewStart, ScalarV.getOperand(2),
16499 ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
16500
16501 // If we looked through an INSERT_SUBVECTOR we need to restore it.
16502 if (ScalarVT != ScalarV.getValueType())
16503 NewScalarV =
16504 DAG.getInsertSubvector(DL, DAG.getUNDEF(ScalarVT), NewScalarV, 0);
16505
16506 SDValue Ops[] = {Reduce.getOperand(0), Reduce.getOperand(1),
16507 NewScalarV, Reduce.getOperand(3),
16508 Reduce.getOperand(4), Reduce.getOperand(5)};
16509 SDValue NewReduce =
16510 DAG.getNode(Reduce.getOpcode(), DL, Reduce.getValueType(), Ops);
16511 return DAG.getNode(Extract.getOpcode(), DL, Extract.getValueType(), NewReduce,
16512 Extract.getOperand(1));
16513}
16514
16515// Optimize (add (shl x, c0), (shl y, c1)) ->
16516// (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
16517// or
16518// (SLLI (QC.SHLADD x, y, c1 - c0), c0), if 4 <= (c1-c0) <=31.
16519static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
16520 const RISCVSubtarget &Subtarget) {
16521 // Perform this optimization only in the zba/xandesperf/xqciac/xtheadba
16522 // extension.
16523 if (!Subtarget.hasShlAdd(3))
16524 return SDValue();
16525
16526 // Skip for vector types and larger types.
16527 EVT VT = N->getValueType(0);
16528 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
16529 return SDValue();
16530
16531 // The two operand nodes must be SHL and have no other use.
16532 SDValue N0 = N->getOperand(0);
16533 SDValue N1 = N->getOperand(1);
16534 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
16535 !N0->hasOneUse() || !N1->hasOneUse())
16536 return SDValue();
16537
16538 // Check c0 and c1.
16539 auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
16540 auto *N1C = dyn_cast<ConstantSDNode>(N1->getOperand(1));
16541 if (!N0C || !N1C)
16542 return SDValue();
16543 int64_t C0 = N0C->getSExtValue();
16544 int64_t C1 = N1C->getSExtValue();
16545 if (C0 <= 0 || C1 <= 0)
16546 return SDValue();
16547
16548 int64_t Diff = std::abs(C0 - C1);
16549 if (!Subtarget.hasShlAdd(Diff))
16550 return SDValue();
16551
16552 // Build nodes.
16553 SDLoc DL(N);
16554 int64_t Bits = std::min(C0, C1);
16555 SDValue NS = (C0 < C1) ? N0->getOperand(0) : N1->getOperand(0);
16556 SDValue NL = (C0 > C1) ? N0->getOperand(0) : N1->getOperand(0);
16557 SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, NL,
16558 DAG.getTargetConstant(Diff, DL, VT), NS);
16559 return DAG.getNode(ISD::SHL, DL, VT, SHADD, DAG.getConstant(Bits, DL, VT));
16560}
16561
16562// Check if this SDValue is an add immediate that is fed by a shift of 1, 2,
16563// or 3.
16564static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
16565 SelectionDAG &DAG) {
16566 using namespace llvm::SDPatternMatch;
16567
16568 // Looking for a reg-reg add and not an addi.
16569 if (isa<ConstantSDNode>(N->getOperand(1)))
16570 return SDValue();
16571
16572 // Based on testing it seems that performance degrades if the ADDI has
16573 // more than 2 uses.
16574 if (AddI->use_size() > 2)
16575 return SDValue();
16576
16577 APInt AddVal;
16578 SDValue SHLVal;
16579 if (!sd_match(AddI, m_Add(m_Value(SHLVal), m_ConstInt(AddVal))))
16580 return SDValue();
16581
16582 APInt VShift;
16583 if (!sd_match(SHLVal, m_OneUse(m_Shl(m_Value(), m_ConstInt(VShift)))))
16584 return SDValue();
16585
16586 if (VShift.slt(1) || VShift.sgt(3))
16587 return SDValue();
16588
16589 SDLoc DL(N);
16590 EVT VT = N->getValueType(0);
16591 // The shift must be positive but the add can be signed.
16592 uint64_t ShlConst = VShift.getZExtValue();
16593 int64_t AddConst = AddVal.getSExtValue();
16594
16595 SDValue SHADD = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, SHLVal->getOperand(0),
16596 DAG.getTargetConstant(ShlConst, DL, VT), Other);
16597 return DAG.getNode(ISD::ADD, DL, VT, SHADD,
16598 DAG.getSignedConstant(AddConst, DL, VT));
16599}
16600
16601// Optimize (add (add (shl x, c0), c1), y) ->
16602// (ADDI (SH*ADD y, x), c1), if c0 equals to [1|2|3].
16603static SDValue combineShlAddIAdd(SDNode *N, SelectionDAG &DAG,
16604 const RISCVSubtarget &Subtarget) {
16605 // Perform this optimization only in the zba extension.
16606 if (!ReassocShlAddiAdd || !Subtarget.hasShlAdd(3))
16607 return SDValue();
16608
16609 // Skip for vector types and larger types.
16610 EVT VT = N->getValueType(0);
16611 if (VT != Subtarget.getXLenVT())
16612 return SDValue();
16613
16614 SDValue AddI = N->getOperand(0);
16615 SDValue Other = N->getOperand(1);
16616 if (SDValue V = combineShlAddIAddImpl(N, AddI, Other, DAG))
16617 return V;
16618 if (SDValue V = combineShlAddIAddImpl(N, Other, AddI, DAG))
16619 return V;
16620 return SDValue();
16621}
16622
16623// Combine a constant select operand into its use:
16624//
16625// (and (select cond, -1, c), x)
16626// -> (select cond, x, (and x, c)) [AllOnes=1]
16627// (or (select cond, 0, c), x)
16628// -> (select cond, x, (or x, c)) [AllOnes=0]
16629// (xor (select cond, 0, c), x)
16630// -> (select cond, x, (xor x, c)) [AllOnes=0]
16631// (add (select cond, 0, c), x)
16632// -> (select cond, x, (add x, c)) [AllOnes=0]
16633// (sub x, (select cond, 0, c))
16634// -> (select cond, x, (sub x, c)) [AllOnes=0]
16635static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
16636 SelectionDAG &DAG, bool AllOnes,
16637 const RISCVSubtarget &Subtarget) {
16638 EVT VT = N->getValueType(0);
16639
16640 // Skip vectors.
16641 if (VT.isVector())
16642 return SDValue();
16643
16644 if (!Subtarget.hasConditionalMoveFusion()) {
16645 // (select cond, x, (and x, c)) has custom lowering with Zicond.
16646 if (!Subtarget.hasCZEROLike() || N->getOpcode() != ISD::AND)
16647 return SDValue();
16648
16649 // Maybe harmful when condition code has multiple use.
16650 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(0).hasOneUse())
16651 return SDValue();
16652
16653 // Maybe harmful when VT is wider than XLen.
16654 if (VT.getSizeInBits() > Subtarget.getXLen())
16655 return SDValue();
16656 }
16657
16658 if ((Slct.getOpcode() != ISD::SELECT &&
16659 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
16660 !Slct.hasOneUse())
16661 return SDValue();
16662
16663 auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
16664 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
16665 };
16666
16667 bool SwapSelectOps;
16668 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
16669 SDValue TrueVal = Slct.getOperand(1 + OpOffset);
16670 SDValue FalseVal = Slct.getOperand(2 + OpOffset);
16671 SDValue NonConstantVal;
16672 if (isZeroOrAllOnes(TrueVal, AllOnes)) {
16673 SwapSelectOps = false;
16674 NonConstantVal = FalseVal;
16675 } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
16676 SwapSelectOps = true;
16677 NonConstantVal = TrueVal;
16678 } else
16679 return SDValue();
16680
16681 // Slct is now know to be the desired identity constant when CC is true.
16682 TrueVal = OtherOp;
16683 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
16684 // Unless SwapSelectOps says the condition should be false.
16685 if (SwapSelectOps)
16686 std::swap(TrueVal, FalseVal);
16687
16688 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
16689 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
16690 {Slct.getOperand(0), Slct.getOperand(1),
16691 Slct.getOperand(2), TrueVal, FalseVal});
16692
16693 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
16694 {Slct.getOperand(0), TrueVal, FalseVal});
16695}
16696
16697// Attempt combineSelectAndUse on each operand of a commutative operator N.
16698static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
16699 bool AllOnes,
16700 const RISCVSubtarget &Subtarget) {
16701 SDValue N0 = N->getOperand(0);
16702 SDValue N1 = N->getOperand(1);
16703 if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes, Subtarget))
16704 return Result;
16705 if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes, Subtarget))
16706 return Result;
16707 return SDValue();
16708}
16709
16710// Transform (add (mul x, c0), c1) ->
16711// (add (mul (add x, c1/c0), c0), c1%c0).
16712// if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
16713// that should be excluded is when c0*(c1/c0) is simm12, which will lead
16714// to an infinite loop in DAGCombine if transformed.
16715// Or transform (add (mul x, c0), c1) ->
16716// (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
16717// if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
16718// case that should be excluded is when c0*(c1/c0+1) is simm12, which will
16719// lead to an infinite loop in DAGCombine if transformed.
16720// Or transform (add (mul x, c0), c1) ->
16721// (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
16722// if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
16723// case that should be excluded is when c0*(c1/c0-1) is simm12, which will
16724// lead to an infinite loop in DAGCombine if transformed.
16725// Or transform (add (mul x, c0), c1) ->
16726// (mul (add x, c1/c0), c0).
16727// if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
16728static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
16729 const RISCVSubtarget &Subtarget) {
16730 // Skip for vector types and larger types.
16731 EVT VT = N->getValueType(0);
16732 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
16733 return SDValue();
16734 // The first operand node must be a MUL and has no other use.
16735 SDValue N0 = N->getOperand(0);
16736 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
16737 return SDValue();
16738 // Check if c0 and c1 match above conditions.
16739 auto *N0C = dyn_cast<ConstantSDNode>(N0->getOperand(1));
16740 auto *N1C = dyn_cast<ConstantSDNode>(N->getOperand(1));
16741 if (!N0C || !N1C)
16742 return SDValue();
16743 // If N0C has multiple uses it's possible one of the cases in
16744 // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
16745 // in an infinite loop.
16746 if (!N0C->hasOneUse())
16747 return SDValue();
16748 int64_t C0 = N0C->getSExtValue();
16749 int64_t C1 = N1C->getSExtValue();
16750 int64_t CA, CB;
16751 // If C1 already fits in an add immediate, there is nothing to split out: the
16752 // (add (mul x, c0), c1) form is already canonical/cheap. Splitting it would
16753 // fight the generic DAGCombiner fold add(mul(add(A, CA), CM), CB) ->
16754 // add(mul(A, CM), CM*CA+CB) (which is gated on isLegalAddImmediate) and cause
16755 // an infinite loop.
16756 if (C0 == -1 || C0 == 0 || C0 == 1 ||
16757 Subtarget.getTargetLowering()->isLegalAddImmediate(C1))
16758 return SDValue();
16759 // Search for proper CA (non-zero) and CB that both are simm12.
16760 if ((C1 / C0) != 0 && isInt<12>(C1 / C0) && isInt<12>(C1 % C0) &&
16761 !isInt<12>(C0 * (C1 / C0))) {
16762 CA = C1 / C0;
16763 CB = C1 % C0;
16764 } else if ((C1 / C0 + 1) != 0 && isInt<12>(C1 / C0 + 1) &&
16765 isInt<12>(C1 % C0 - C0) && !isInt<12>(C0 * (C1 / C0 + 1))) {
16766 CA = C1 / C0 + 1;
16767 CB = C1 % C0 - C0;
16768 } else if ((C1 / C0 - 1) != 0 && isInt<12>(C1 / C0 - 1) &&
16769 isInt<12>(C1 % C0 + C0) && !isInt<12>(C0 * (C1 / C0 - 1))) {
16770 CA = C1 / C0 - 1;
16771 CB = C1 % C0 + C0;
16772 } else
16773 return SDValue();
16774 // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
16775 SDLoc DL(N);
16776 SDValue New0 = DAG.getNode(ISD::ADD, DL, VT, N0->getOperand(0),
16777 DAG.getSignedConstant(CA, DL, VT));
16778 SDValue New1 =
16779 DAG.getNode(ISD::MUL, DL, VT, New0, DAG.getSignedConstant(C0, DL, VT));
16780 return DAG.getNode(ISD::ADD, DL, VT, New1, DAG.getSignedConstant(CB, DL, VT));
16781}
16782
16783// add (zext, zext) -> zext (add (zext, zext))
16784// sub (zext, zext) -> sext (sub (zext, zext))
16785// mul (zext, zext) -> zext (mul (zext, zext))
16786// sdiv (zext, zext) -> zext (sdiv (zext, zext))
16787// udiv (zext, zext) -> zext (udiv (zext, zext))
16788// srem (zext, zext) -> zext (srem (zext, zext))
16789// urem (zext, zext) -> zext (urem (zext, zext))
16790//
16791// where the sum of the extend widths match, and the the range of the bin op
16792// fits inside the width of the narrower bin op. (For profitability on rvv, we
16793// use a power of two for both inner and outer extend.)
16794static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
16795
16796 EVT VT = N->getValueType(0);
16797 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
16798 return SDValue();
16799
16800 SDValue N0 = N->getOperand(0);
16801 SDValue N1 = N->getOperand(1);
16802 if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
16803 return SDValue();
16804 if (!N0.hasOneUse() || !N1.hasOneUse())
16805 return SDValue();
16806
16807 SDValue Src0 = N0.getOperand(0);
16808 SDValue Src1 = N1.getOperand(0);
16809 EVT SrcVT = Src0.getValueType();
16810 if (!DAG.getTargetLoweringInfo().isTypeLegal(SrcVT) ||
16811 SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
16812 SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
16813 return SDValue();
16814
16815 LLVMContext &C = *DAG.getContext();
16816 EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(C);
16817 EVT NarrowVT = EVT::getVectorVT(C, ElemVT, VT.getVectorElementCount());
16818
16819 Src0 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src0), NarrowVT, Src0);
16820 Src1 = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Src1), NarrowVT, Src1);
16821
16822 // Src0 and Src1 are zero extended, so they're always positive if signed.
16823 //
16824 // sub can produce a negative from two positive operands, so it needs sign
16825 // extended. Other nodes produce a positive from two positive operands, so
16826 // zero extend instead.
16827 unsigned OuterExtend =
16828 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
16829
16830 return DAG.getNode(
16831 OuterExtend, SDLoc(N), VT,
16832 DAG.getNode(N->getOpcode(), SDLoc(N), NarrowVT, Src0, Src1));
16833}
16834
16835// Try to turn (add (xor bool, 1) -1) into (neg bool).
16836static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
16837 SDValue N0 = N->getOperand(0);
16838 SDValue N1 = N->getOperand(1);
16839 EVT VT = N->getValueType(0);
16840 SDLoc DL(N);
16841
16842 // RHS should be -1.
16843 if (!isAllOnesConstant(N1))
16844 return SDValue();
16845
16846 // Look for (xor X, 1).
16847 if (N0.getOpcode() != ISD::XOR || !isOneConstant(N0.getOperand(1)))
16848 return SDValue();
16849
16850 // First xor input should be 0 or 1.
16851 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
16852 if (!DAG.MaskedValueIsZero(N0.getOperand(0), Mask))
16853 return SDValue();
16854
16855 // Emit a negate of the setcc.
16856 return DAG.getNegative(N0.getOperand(0), DL, VT);
16857}
16858
16859// Fold (add X, (mulhs X, C)) -> (mulhsu X, C) if C is negative. This occurs
16860// in the expansion of sdiv i32 X, 7 using magic multiply.
16861//
16862// mulhs returns the hi from X * C = hi * 2^32 + lo.
16863//
16864// Since C<0, u(C) as an unsigned constant is 2^32 + C = u(C).
16865// mulhsu computes
16866// X * u(C0) = X * (C + 2^32)
16867// = X * 2^32 + C * X // C * X is the same as mulhs
16868// = X * 2^32 + hi * 2^32 + lo
16869// = (X + hi) * 2^32 + lo
16870// So mulhsu computes (X + hi).
16871static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
16872 const RISCVSubtarget &Subtarget) {
16873 EVT VT = N->getValueType(0);
16874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16875 bool IsPExtPackedDoubleType =
16876 VT.isSimple() && Subtarget.isPExtPackedDoubleType(VT.getSimpleVT());
16877 if (!TLI.isOperationLegal(ISD::MULHS, VT) && !IsPExtPackedDoubleType &&
16878 !(Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8))
16879 return SDValue();
16880
16881 using namespace SDPatternMatch;
16882 SDValue X, Mulh;
16883 APInt C;
16884 if (!sd_match(N,
16885 m_Add(m_Value(X),
16886 m_OneUse(m_Value(Mulh, m_BinOp(ISD::MULHS, m_Deferred(X),
16887 m_ConstInt(C)))))) ||
16888 !C.isNegative())
16889 return SDValue();
16890
16891 SDLoc DL(N);
16892
16893 // We don't have a v4i8 MULHSU instruction, use a WMULSU+SRL+TRUNC.
16894 auto MakePWMulSU = [&](SDValue A, SDValue B) -> SDValue {
16895 SDValue WMul = DAG.getNode(RISCVISD::PWMULSU, DL, MVT::v4i16, A, B);
16896 SDValue Shifted = DAG.getNode(ISD::SRL, DL, MVT::v4i16, WMul,
16897 DAG.getConstant(8, DL, MVT::v4i16));
16898 return DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i8, Shifted);
16899 };
16900
16901 // We need to split double wide vectors ourselves, op legalization won't
16902 // run for custom nodes.
16903 if (IsPExtPackedDoubleType) {
16904 MVT HalfVT = VT.getSimpleVT().getHalfNumVectorElementsVT();
16905 auto [XLo, XHi] = DAG.SplitVector(X, DL, HalfVT, HalfVT);
16906 auto [CLo, CHi] = DAG.SplitVector(Mulh.getOperand(1), DL, HalfVT, HalfVT);
16907 SDValue ResLo, ResHi;
16908 if (HalfVT == MVT::v4i8) {
16909 ResLo = MakePWMulSU(XLo, CLo);
16910 ResHi = MakePWMulSU(XHi, CHi);
16911 } else {
16912 ResLo = DAG.getNode(RISCVISD::MULHSU, DL, HalfVT, XLo, CLo);
16913 ResHi = DAG.getNode(RISCVISD::MULHSU, DL, HalfVT, XHi, CHi);
16914 }
16915 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
16916 }
16917
16918 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8)
16919 return MakePWMulSU(X, Mulh.getOperand(1));
16920
16921 return DAG.getNode(RISCVISD::MULHSU, DL, VT, X, Mulh.getOperand(1));
16922}
16923
16924static SDValue performADDCombine(SDNode *N,
16925 TargetLowering::DAGCombinerInfo &DCI,
16926 const RISCVSubtarget &Subtarget) {
16927 SelectionDAG &DAG = DCI.DAG;
16928 if (SDValue V = combineAddOfBooleanXor(N, DAG))
16929 return V;
16930 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
16931 return V;
16932 if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) {
16933 if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
16934 return V;
16935 if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
16936 return V;
16937 }
16938 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
16939 return V;
16940 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
16941 return V;
16942 if (SDValue V = combineBinOpOfZExt(N, DAG))
16943 return V;
16944 if (SDValue V = combineAddMulh(N, DAG, Subtarget))
16945 return V;
16946
16947 // fold (add (select lhs, rhs, cc, 0, y), x) ->
16948 // (select lhs, rhs, cc, x, (add x, y))
16949 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
16950}
16951
16952// Try to turn a sub boolean RHS and constant LHS into an addi.
16953static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
16954 SDValue N0 = N->getOperand(0);
16955 SDValue N1 = N->getOperand(1);
16956 EVT VT = N->getValueType(0);
16957 SDLoc DL(N);
16958
16959 // Require a constant LHS.
16960 auto *N0C = dyn_cast<ConstantSDNode>(N0);
16961 if (!N0C)
16962 return SDValue();
16963
16964 // All our optimizations involve subtracting 1 from the immediate and forming
16965 // an ADDI. Make sure the new immediate is valid for an ADDI.
16966 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
16967 if (!ImmValMinus1.isSignedIntN(12))
16968 return SDValue();
16969
16970 SDValue NewLHS;
16971 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
16972 // (sub constant, (setcc x, y, eq/neq)) ->
16973 // (add (setcc x, y, neq/eq), constant - 1)
16974 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
16975 EVT SetCCOpVT = N1.getOperand(0).getValueType();
16976 if (!isIntEqualitySetCC(CCVal) || !SetCCOpVT.isInteger())
16977 return SDValue();
16978 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
16979 NewLHS =
16980 DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal);
16981 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(N1.getOperand(1)) &&
16982 N1.getOperand(0).getOpcode() == ISD::SETCC) {
16983 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
16984 // Since setcc returns a bool the xor is equivalent to 1-setcc.
16985 NewLHS = N1.getOperand(0);
16986 } else
16987 return SDValue();
16988
16989 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
16990 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
16991}
16992
16993// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
16994// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
16995// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
16996// valid with Y=3, while 0b0000_1000_0000_0100 is not.
16997static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
16998 const RISCVSubtarget &Subtarget) {
16999 if (!Subtarget.hasStdExtZbb())
17000 return SDValue();
17001
17002 EVT VT = N->getValueType(0);
17003
17004 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
17005 return SDValue();
17006
17007 SDValue N0 = N->getOperand(0);
17008 SDValue N1 = N->getOperand(1);
17009
17010 if (N0->getOpcode() != ISD::SHL)
17011 return SDValue();
17012
17013 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(N0.getOperand(1));
17014 if (!ShAmtCLeft)
17015 return SDValue();
17016 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
17017
17018 if (ShiftedAmount >= 8)
17019 return SDValue();
17020
17021 SDValue LeftShiftOperand = N0->getOperand(0);
17022 SDValue RightShiftOperand = N1;
17023
17024 if (ShiftedAmount != 0) { // Right operand must be a right shift.
17025 if (N1->getOpcode() != ISD::SRL)
17026 return SDValue();
17027 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(N1.getOperand(1));
17028 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
17029 return SDValue();
17030 RightShiftOperand = N1.getOperand(0);
17031 }
17032
17033 // At least one shift should have a single use.
17034 if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
17035 return SDValue();
17036
17037 if (LeftShiftOperand != RightShiftOperand)
17038 return SDValue();
17039
17040 APInt Mask = APInt::getSplat(VT.getSizeInBits(), APInt(8, 0x1));
17041 Mask <<= ShiftedAmount;
17042 // Check that X has indeed the right shape (only the Y-th bit can be set in
17043 // every byte).
17044 if (!DAG.MaskedValueIsZero(LeftShiftOperand, ~Mask))
17045 return SDValue();
17046
17047 return DAG.getNode(RISCVISD::ORC_B, SDLoc(N), VT, LeftShiftOperand);
17048}
17049
17050static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
17051 const RISCVSubtarget &Subtarget) {
17052 if (SDValue V = combineSubOfBoolean(N, DAG))
17053 return V;
17054
17055 EVT VT = N->getValueType(0);
17056 SDValue N0 = N->getOperand(0);
17057 SDValue N1 = N->getOperand(1);
17058 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
17059 if (isNullConstant(N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
17060 isNullConstant(N1.getOperand(1)) &&
17061 N1.getValueType() == N1.getOperand(0).getValueType()) {
17062 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
17063 if (CCVal == ISD::SETLT) {
17064 SDLoc DL(N);
17065 unsigned ShAmt = N0.getValueSizeInBits() - 1;
17066 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
17067 DAG.getConstant(ShAmt, DL, VT));
17068 }
17069 }
17070
17071 if (SDValue V = combineBinOpOfZExt(N, DAG))
17072 return V;
17073 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
17074 return V;
17075
17076 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
17077 // (select lhs, rhs, cc, x, (sub x, y))
17078 return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false, Subtarget);
17079}
17080
17081// Apply DeMorgan's law to (and/or (xor X, 1), (xor Y, 1)) if X and Y are 0/1.
17082// Legalizing setcc can introduce xors like this. Doing this transform reduces
17083// the number of xors and may allow the xor to fold into a branch condition.
17084static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
17085 SDValue N0 = N->getOperand(0);
17086 SDValue N1 = N->getOperand(1);
17087 bool IsAnd = N->getOpcode() == ISD::AND;
17088
17089 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
17090 return SDValue();
17091
17092 if (!N0.hasOneUse() || !N1.hasOneUse())
17093 return SDValue();
17094
17095 SDValue N01 = N0.getOperand(1);
17096 SDValue N11 = N1.getOperand(1);
17097
17098 // For AND, SimplifyDemandedBits may have turned one of the (xor X, 1) into
17099 // (xor X, -1) based on the upper bits of the other operand being 0. If the
17100 // operation is And, allow one of the Xors to use -1.
17101 if (isOneConstant(N01)) {
17102 if (!isOneConstant(N11) && !(IsAnd && isAllOnesConstant(N11)))
17103 return SDValue();
17104 } else if (isOneConstant(N11)) {
17105 // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
17106 if (!(IsAnd && isAllOnesConstant(N01)))
17107 return SDValue();
17108 } else
17109 return SDValue();
17110
17111 EVT VT = N->getValueType(0);
17112
17113 SDValue N00 = N0.getOperand(0);
17114 SDValue N10 = N1.getOperand(0);
17115
17116 // The LHS of the xors needs to be 0/1.
17117 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
17118 if (!DAG.MaskedValueIsZero(N00, Mask) || !DAG.MaskedValueIsZero(N10, Mask))
17119 return SDValue();
17120
17121 // Invert the opcode and insert a new xor.
17122 SDLoc DL(N);
17123 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
17124 SDValue Logic = DAG.getNode(Opc, DL, VT, N00, N10);
17125 return DAG.getNode(ISD::XOR, DL, VT, Logic, DAG.getConstant(1, DL, VT));
17126}
17127
17128// Fold (vXi8 (trunc (vselect (setltu, X, 256), X, (sext (setgt X, 0))))) to
17129// (vXi8 (trunc (smin (smax X, 0), 255))). This represents saturating a signed
17130// value to an unsigned value. This will be lowered to vmax and series of
17131// vnclipu instructions later. This can be extended to other truncated types
17132// other than i8 by replacing 256 and 255 with the equivalent constants for the
17133// type.
17134static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
17135 EVT VT = N->getValueType(0);
17136 SDValue N0 = N->getOperand(0);
17137 EVT SrcVT = N0.getValueType();
17138
17139 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17140 if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(SrcVT))
17141 return SDValue();
17142
17143 if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
17144 return SDValue();
17145
17146 SDValue Cond = N0.getOperand(0);
17147 SDValue True = N0.getOperand(1);
17148 SDValue False = N0.getOperand(2);
17149
17150 if (Cond.getOpcode() != ISD::SETCC)
17151 return SDValue();
17152
17153 // FIXME: Support the version of this pattern with the select operands
17154 // swapped.
17155 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
17156 if (CCVal != ISD::SETULT)
17157 return SDValue();
17158
17159 SDValue CondLHS = Cond.getOperand(0);
17160 SDValue CondRHS = Cond.getOperand(1);
17161
17162 if (CondLHS != True)
17163 return SDValue();
17164
17165 unsigned ScalarBits = VT.getScalarSizeInBits();
17166
17167 // FIXME: Support other constants.
17168 ConstantSDNode *CondRHSC = isConstOrConstSplat(CondRHS);
17169 if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
17170 return SDValue();
17171
17172 if (False.getOpcode() != ISD::SIGN_EXTEND)
17173 return SDValue();
17174
17175 False = False.getOperand(0);
17176
17177 if (False.getOpcode() != ISD::SETCC || False.getOperand(0) != True)
17178 return SDValue();
17179
17180 ConstantSDNode *FalseRHSC = isConstOrConstSplat(False.getOperand(1));
17181 if (!FalseRHSC || !FalseRHSC->isZero())
17182 return SDValue();
17183
17184 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(False.getOperand(2))->get();
17185 if (CCVal2 != ISD::SETGT)
17186 return SDValue();
17187
17188 // Emit the signed to unsigned saturation pattern.
17189 SDLoc DL(N);
17190 SDValue Max =
17191 DAG.getNode(ISD::SMAX, DL, SrcVT, True, DAG.getConstant(0, DL, SrcVT));
17192 SDValue Min =
17193 DAG.getNode(ISD::SMIN, DL, SrcVT, Max,
17194 DAG.getConstant((1ULL << ScalarBits) - 1, DL, SrcVT));
17195 return DAG.getNode(ISD::TRUNCATE, DL, VT, Min);
17196}
17197
17198// Handle P extension truncate patterns, both on packed vectors and on scalar
17199// i32 (the RV32-only asub/asubu and mulhr* instructions):
17200// ASUB/ASUBU: (trunc (srl (sub ([s|z]ext a), ([s|z]ext b)), 1))
17201// MULHSU: (trunc (srl (mul (sext a), (zext b)), EltBits))
17202// MULHR*: (trunc (srl (add (mul (sext a), (zext b)), round_const), EltBits))
17203static SDValue combinePExtTruncate(SDNode *N, SelectionDAG &DAG,
17204 const RISCVSubtarget &Subtarget) {
17205 SDValue N0 = N->getOperand(0);
17206 EVT VT = N->getValueType(0);
17207 if (N0.getOpcode() != ISD::SRL)
17208 return SDValue();
17209
17210 if (VT != MVT::v4i16 && VT != MVT::v2i16 && VT != MVT::v8i8 &&
17211 VT != MVT::v4i8 && VT != MVT::v2i32 && VT != MVT::i32)
17212 return SDValue();
17213
17214 ConstantSDNode *C = isConstOrConstSplat(N0.getOperand(1));
17215 if (!C)
17216 return SDValue();
17217
17218 SDValue Op = N0.getOperand(0);
17219 unsigned ShAmtVal = C->getZExtValue();
17220 unsigned EltBits = VT.getScalarSizeInBits();
17221
17222 // Check for rounding pattern: (add (mul ...), round_const)
17223 bool IsRounding = false;
17224 if (Op.getOpcode() == ISD::ADD && (EltBits == 16 || EltBits == 32)) {
17225 ConstantSDNode *RndC = isConstOrConstSplat(Op.getOperand(1));
17226 if (RndC && RndC->getZExtValue() == (1ULL << (EltBits - 1)) &&
17227 Op.getOperand(0).getOpcode() == ISD::MUL) {
17228 Op = Op.getOperand(0);
17229 IsRounding = true;
17230 }
17231 }
17232
17233 // Ensure Op is a binary operation before accessing its operands.
17234 if (Op.getNumOperands() != 2)
17235 return SDValue();
17236
17237 SDValue LHS = Op.getOperand(0);
17238 SDValue RHS = Op.getOperand(1);
17239
17240 bool LHSIsSExt = LHS.getOpcode() == ISD::SIGN_EXTEND;
17241 bool LHSIsZExt = LHS.getOpcode() == ISD::ZERO_EXTEND;
17242 bool RHSIsSExt = RHS.getOpcode() == ISD::SIGN_EXTEND;
17243 bool RHSIsZExt = RHS.getOpcode() == ISD::ZERO_EXTEND;
17244
17245 if (!(LHSIsSExt || LHSIsZExt) || !(RHSIsSExt || RHSIsZExt))
17246 return SDValue();
17247
17248 SDValue A = LHS.getOperand(0);
17249 SDValue B = RHS.getOperand(0);
17250
17251 if (A.getValueType() != VT || B.getValueType() != VT)
17252 return SDValue();
17253
17254 unsigned Opc;
17255 switch (Op.getOpcode()) {
17256 default:
17257 return SDValue();
17258 case ISD::SUB:
17259 // PASUB/PASUBU: shift amount must be 1
17260 if (ShAmtVal != 1)
17261 return SDValue();
17262 if (LHSIsSExt && RHSIsSExt)
17263 Opc = RISCVISD::ASUB;
17264 else if (LHSIsZExt && RHSIsZExt)
17265 Opc = RISCVISD::ASUBU;
17266 else
17267 return SDValue();
17268 break;
17269 case ISD::MUL:
17270 // MULH*/MULHR*: shift amount must be element size, only for i16/i32
17271 if (ShAmtVal != EltBits || (EltBits != 16 && EltBits != 32))
17272 return SDValue();
17273 if (!Subtarget.is64Bit() && (VT == MVT::v2i32 || VT == MVT::v4i16))
17274 return SDValue();
17275 if (IsRounding) {
17276 if (LHSIsSExt && RHSIsSExt) {
17277 Opc = RISCVISD::MULHR;
17278 } else if (LHSIsZExt && RHSIsZExt) {
17279 Opc = RISCVISD::MULHRU;
17280 } else if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17281 Opc = RISCVISD::MULHRSU;
17282 // commuted case
17283 if (LHSIsZExt && RHSIsSExt)
17284 std::swap(A, B);
17285 } else {
17286 return SDValue();
17287 }
17288 } else {
17289 // Scalar mulhsu is handled elsewhere, only match the packed MULHSU here.
17290 if (!VT.isVector())
17291 return SDValue();
17292 if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17293 Opc = RISCVISD::MULHSU;
17294 // commuted case
17295 if (LHSIsZExt && RHSIsSExt)
17296 std::swap(A, B);
17297 } else
17298 return SDValue();
17299 }
17300 break;
17301 }
17302
17303 return DAG.getNode(Opc, SDLoc(N), VT, {A, B});
17304}
17305
17306static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
17307 const RISCVSubtarget &Subtarget) {
17308 SDValue N0 = N->getOperand(0);
17309 EVT VT = N->getValueType(0);
17310
17311 // P truncate patterns: packed vectors, plus RV32-only scalar i32.
17312 if (Subtarget.hasStdExtP() &&
17313 (VT.isFixedLengthVector() || (VT == MVT::i32 && !Subtarget.is64Bit())))
17314 return combinePExtTruncate(N, DAG, Subtarget);
17315
17316 // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
17317 // extending X. This is safe since we only need the LSB after the shift and
17318 // shift amounts larger than 31 would produce poison. If we wait until
17319 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17320 // to use a BEXT instruction.
17321 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
17322 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
17323 !isa<ConstantSDNode>(N0.getOperand(1)) && N0.hasOneUse()) {
17324 SDLoc DL(N0);
17325 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
17326 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
17327 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
17328 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Srl);
17329 }
17330
17331 return combineTruncSelectToSMaxUSat(N, DAG);
17332}
17333
17334// InstCombinerImpl::transformZExtICmp will narrow a zext of an icmp with a
17335// truncation. But RVV doesn't have truncation instructions for more than twice
17336// the bitwidth.
17337//
17338// E.g. trunc <vscale x 1 x i64> %x to <vscale x 1 x i8> will generate:
17339//
17340// vsetvli a0, zero, e32, m2, ta, ma
17341// vnsrl.wi v12, v8, 0
17342// vsetvli zero, zero, e16, m1, ta, ma
17343// vnsrl.wi v8, v12, 0
17344// vsetvli zero, zero, e8, mf2, ta, ma
17345// vnsrl.wi v8, v8, 0
17346//
17347// So reverse the combine so we generate an vmseq/vmsne again:
17348//
17349// and (lshr (trunc X), ShAmt), 1
17350// -->
17351// zext (icmp ne (and X, (1 << ShAmt)), 0)
17352//
17353// and (lshr (not (trunc X)), ShAmt), 1
17354// -->
17355// zext (icmp eq (and X, (1 << ShAmt)), 0)
17356static SDValue reverseZExtICmpCombine(SDNode *N, SelectionDAG &DAG,
17357 const RISCVSubtarget &Subtarget) {
17358 using namespace SDPatternMatch;
17359 SDLoc DL(N);
17360
17361 if (!Subtarget.hasVInstructions())
17362 return SDValue();
17363
17364 EVT VT = N->getValueType(0);
17365 if (!VT.isVector())
17366 return SDValue();
17367
17368 APInt ShAmt;
17369 SDValue Inner;
17370 if (!sd_match(N, m_And(m_OneUse(m_Srl(m_Value(Inner), m_ConstInt(ShAmt))),
17371 m_One())))
17372 return SDValue();
17373
17374 SDValue X;
17375 bool IsNot;
17376 if (sd_match(Inner, m_Not(m_Trunc(m_Value(X)))))
17377 IsNot = true;
17378 else if (sd_match(Inner, m_Trunc(m_Value(X))))
17379 IsNot = false;
17380 else
17381 return SDValue();
17382
17383 EVT WideVT = X.getValueType();
17384 if (VT.getScalarSizeInBits() >= WideVT.getScalarSizeInBits() / 2)
17385 return SDValue();
17386
17387 SDValue Res =
17388 DAG.getNode(ISD::AND, DL, WideVT, X,
17389 DAG.getConstant(1ULL << ShAmt.getZExtValue(), DL, WideVT));
17390 Res = DAG.getSetCC(DL,
17391 EVT::getVectorVT(*DAG.getContext(), MVT::i1,
17392 WideVT.getVectorElementCount()),
17393 Res, DAG.getConstant(0, DL, WideVT),
17394 IsNot ? ISD::SETEQ : ISD::SETNE);
17395 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Res);
17396}
17397
17398// (and (i1) f, (setcc c, 0, ne)) -> (czero.nez f, c)
17399// (and (i1) f, (setcc c, 0, eq)) -> (czero.eqz f, c)
17400// (and (setcc c, 0, ne), (i1) g) -> (czero.nez g, c)
17401// (and (setcc c, 0, eq), (i1) g) -> (czero.eqz g, c)
17402static SDValue combineANDOfSETCCToCZERO(SDNode *N, SelectionDAG &DAG,
17403 const RISCVSubtarget &Subtarget) {
17404 if (!Subtarget.hasCZEROLike())
17405 return SDValue();
17406
17407 SDValue N0 = N->getOperand(0);
17408 SDValue N1 = N->getOperand(1);
17409
17410 auto IsEqualCompZero = [](SDValue &V) -> bool {
17411 if (V.getOpcode() == ISD::SETCC && isNullConstant(V.getOperand(1))) {
17412 ISD::CondCode CC = cast<CondCodeSDNode>(V.getOperand(2))->get();
17413 if (ISD::isIntEqualitySetCC(CC))
17414 return true;
17415 }
17416 return false;
17417 };
17418
17419 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17420 std::swap(N0, N1);
17421 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17422 return SDValue();
17423
17424 KnownBits Known = DAG.computeKnownBits(N1);
17425 if (Known.getMaxValue().ugt(1))
17426 return SDValue();
17427
17428 unsigned CzeroOpcode =
17429 (cast<CondCodeSDNode>(N0.getOperand(2))->get() == ISD::SETNE)
17430 ? RISCVISD::CZERO_EQZ
17431 : RISCVISD::CZERO_NEZ;
17432
17433 EVT VT = N->getValueType(0);
17434 SDLoc DL(N);
17435 return DAG.getNode(CzeroOpcode, DL, VT, N1, N0.getOperand(0));
17436}
17437
17438static SDValue reduceANDOfAtomicLoad(SDNode *N,
17439 TargetLowering::DAGCombinerInfo &DCI) {
17440 SelectionDAG &DAG = DCI.DAG;
17441 if (N->getOpcode() != ISD::AND)
17442 return SDValue();
17443
17444 SDValue N0 = N->getOperand(0);
17445 if (N0.getOpcode() != ISD::ATOMIC_LOAD)
17446 return SDValue();
17447 if (!N0.hasOneUse())
17448 return SDValue();
17449
17450 AtomicSDNode *ALoad = cast<AtomicSDNode>(N0.getNode());
17451 if (isStrongerThanMonotonic(ALoad->getSuccessOrdering()))
17452 return SDValue();
17453
17454 EVT LoadedVT = ALoad->getMemoryVT();
17455 ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(N->getOperand(1));
17456 if (!MaskConst)
17457 return SDValue();
17458 uint64_t Mask = MaskConst->getZExtValue();
17459 uint64_t ExpectedMask = maskTrailingOnes<uint64_t>(LoadedVT.getSizeInBits());
17460 if (Mask != ExpectedMask)
17461 return SDValue();
17462
17463 SDValue ZextLoad = DAG.getAtomicLoad(
17464 ISD::ZEXTLOAD, SDLoc(N), ALoad->getMemoryVT(), N->getValueType(0),
17465 ALoad->getChain(), ALoad->getBasePtr(), ALoad->getMemOperand());
17466 DCI.CombineTo(N, ZextLoad);
17467 DAG.ReplaceAllUsesOfValueWith(SDValue(N0.getNode(), 1), ZextLoad.getValue(1));
17468 DCI.recursivelyDeleteUnusedNodes(N0.getNode());
17469 return SDValue(N, 0);
17470}
17471
17472// Sometimes a mask is applied after a shift. If that shift was fed by a
17473// load, there is sometimes the opportunity to narrow the load, which is
17474// hidden by the intermediate shift. Detect that case and commute the
17475// shift/and in order to enable load narrowing.
17476static SDValue combineNarrowableShiftedLoad(SDNode *N, SelectionDAG &DAG) {
17477 EVT VT = N->getValueType(0);
17478 if (!VT.isScalarInteger())
17479 return SDValue();
17480
17481 using namespace SDPatternMatch;
17482 SDValue LoadNode;
17483 APInt MaskVal, ShiftVal;
17484 // (and (shl (load ...), ShiftAmt), Mask)
17485 if (!sd_match(
17486 N, m_And(m_OneUse(m_Shl(m_Value(LoadNode, m_SpecificOpc(ISD::LOAD)),
17487 m_ConstInt(ShiftVal))),
17488 m_ConstInt(MaskVal)))) {
17489 return SDValue();
17490 }
17491
17492 uint64_t ShiftAmt = ShiftVal.getZExtValue();
17493
17494 if (ShiftAmt >= VT.getSizeInBits())
17495 return SDValue();
17496
17497 // Calculate the appropriate mask if it were applied before the shift.
17498 APInt InnerMask = MaskVal.lshr(ShiftAmt);
17499 bool IsNarrowable =
17500 InnerMask == 0xff || InnerMask == 0xffff || InnerMask == 0xffffffff;
17501
17502 if (!IsNarrowable)
17503 return SDValue();
17504
17505 // AND the loaded value and change the shift appropriately, allowing
17506 // the load to be narrowed.
17507 SDLoc DL(N);
17508 SDValue InnerAnd = DAG.getNode(ISD::AND, DL, VT, LoadNode,
17509 DAG.getConstant(InnerMask, DL, VT));
17510 return DAG.getNode(ISD::SHL, DL, VT, InnerAnd,
17511 DAG.getShiftAmountConstant(ShiftAmt, VT, DL));
17512}
17513
17514// Combines two comparison operation and logic operation to one selection
17515// operation(min, max) and logic operation. Returns new constructed Node if
17516// conditions for optimization are satisfied.
17517static SDValue performANDCombine(SDNode *N,
17518 TargetLowering::DAGCombinerInfo &DCI,
17519 const RISCVSubtarget &Subtarget) {
17520 SelectionDAG &DAG = DCI.DAG;
17521 SDValue N0 = N->getOperand(0);
17522
17523 // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
17524 // extending X. This is safe since we only need the LSB after the shift and
17525 // shift amounts larger than 31 would produce poison. If we wait until
17526 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17527 // to use a BEXT instruction.
17528 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
17529 N->getValueType(0) == MVT::i32 && isOneConstant(N->getOperand(1)) &&
17530 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(N0.getOperand(1)) &&
17531 N0.hasOneUse()) {
17532 SDLoc DL(N);
17533 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
17534 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
17535 SDValue Srl = DAG.getNode(ISD::SRL, DL, MVT::i64, Op0, Op1);
17536 SDValue And = DAG.getNode(ISD::AND, DL, MVT::i64, Srl,
17537 DAG.getConstant(1, DL, MVT::i64));
17538 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, And);
17539 }
17540
17541 if (SDValue V = combineNarrowableShiftedLoad(N, DAG))
17542 return V;
17543 if (SDValue V = reverseZExtICmpCombine(N, DAG, Subtarget))
17544 return V;
17545 if (DCI.isAfterLegalizeDAG())
17546 if (SDValue V = combineANDOfSETCCToCZERO(N, DAG, Subtarget))
17547 return V;
17548 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17549 return V;
17550 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17551 return V;
17552 if (SDValue V = reduceANDOfAtomicLoad(N, DCI))
17553 return V;
17554
17555 if (DCI.isAfterLegalizeDAG())
17556 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
17557 return V;
17558
17559 // fold (and (select lhs, rhs, cc, -1, y), x) ->
17560 // (select lhs, rhs, cc, x, (and x, y))
17561 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
17562}
17563
17564// Try to pull an xor with 1 through a select idiom that uses czero_eqz/nez.
17565// FIXME: Generalize to other binary operators with same operand.
17566static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1,
17567 SelectionDAG &DAG) {
17568 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
17569
17570 if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
17571 N1.getOpcode() != RISCVISD::CZERO_NEZ ||
17572 !N0.hasOneUse() || !N1.hasOneUse())
17573 return SDValue();
17574
17575 // Should have the same condition.
17576 SDValue Cond = N0.getOperand(1);
17577 if (Cond != N1.getOperand(1))
17578 return SDValue();
17579
17580 SDValue TrueV = N0.getOperand(0);
17581 SDValue FalseV = N1.getOperand(0);
17582
17583 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
17584 TrueV.getOperand(1) != FalseV.getOperand(1) ||
17585 !isOneConstant(TrueV.getOperand(1)) ||
17586 !TrueV.hasOneUse() || !FalseV.hasOneUse())
17587 return SDValue();
17588
17589 EVT VT = N->getValueType(0);
17590 SDLoc DL(N);
17591
17592 SDValue NewN0 = DAG.getNode(RISCVISD::CZERO_EQZ, DL, VT, TrueV.getOperand(0),
17593 Cond);
17594 SDValue NewN1 =
17595 DAG.getNode(RISCVISD::CZERO_NEZ, DL, VT, FalseV.getOperand(0), Cond);
17596 SDValue NewOr =
17597 DAG.getNode(ISD::OR, DL, VT, NewN0, NewN1, SDNodeFlags::Disjoint);
17598 return DAG.getNode(ISD::XOR, DL, VT, NewOr, TrueV.getOperand(1));
17599}
17600
17601// (xor X, (xor (and X, C2), Y))
17602// ->(qc_insb X, (sra Y, ShAmt), Width, ShAmt)
17603// where C2 is a shifted mask with width = Width and shift = ShAmt
17604// qc_insb might become qc.insb or qc.insbi depending on the operands.
17605static SDValue combineXorToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17606 const RISCVSubtarget &Subtarget) {
17607 if (!Subtarget.hasVendorXqcibm())
17608 return SDValue();
17609
17610 using namespace SDPatternMatch;
17611 SDValue Base, Inserted;
17612 APInt CMask;
17613 if (!sd_match(N, m_Xor(m_Value(Base),
17614 m_OneUse(m_Xor(m_OneUse(m_And(m_Deferred(Base),
17615 m_ConstInt(CMask))),
17616 m_Value(Inserted))))))
17617 return SDValue();
17618
17619 if (N->getValueType(0) != MVT::i32)
17620 return SDValue();
17621 unsigned Width, ShAmt;
17622 if (!CMask.isShiftedMask(ShAmt, Width))
17623 return SDValue();
17624
17625 // Check if all zero bits in CMask are also zero in Inserted
17626 if (!DAG.MaskedValueIsZero(Inserted, ~CMask))
17627 return SDValue();
17628
17629 SDLoc DL(N);
17630
17631 // `Inserted` needs to be right shifted before it is put into the
17632 // instruction.
17633 Inserted = DAG.getNode(ISD::SRA, DL, MVT::i32, Inserted,
17634 DAG.getShiftAmountConstant(ShAmt, MVT::i32, DL));
17635
17636 SDValue Ops[] = {Base, Inserted, DAG.getConstant(Width, DL, MVT::i32),
17637 DAG.getConstant(ShAmt, DL, MVT::i32)};
17638 return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
17639}
17640
17641static SDValue combineOrToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17642 const RISCVSubtarget &Subtarget) {
17643 if (!Subtarget.hasVendorXqcibm())
17644 return SDValue();
17645
17646 using namespace SDPatternMatch;
17647
17648 SDValue X;
17649 APInt MaskImm;
17650 if (!sd_match(N, m_Or(m_OneUse(m_Value(X)), m_ConstInt(MaskImm))))
17651 return SDValue();
17652
17653 unsigned ShAmt, Width;
17654 if (!MaskImm.isShiftedMask(ShAmt, Width) || MaskImm.isSignedIntN(12))
17655 return SDValue();
17656
17657 if (N->getValueType(0) != MVT::i32)
17658 return SDValue();
17659
17660 // If Zbs is enabled and it is a single bit set we can use BSETI which
17661 // can be compressed to C_BSETI when Xqcibm in enabled.
17662 if (Width == 1 && Subtarget.hasStdExtZbs())
17663 return SDValue();
17664
17665 // If C1 is a shifted mask (but can't be formed as an ORI),
17666 // use a bitfield insert of -1.
17667 // Transform (or x, C1)
17668 // -> (qc.insbi x, -1, width, shift)
17669 SDLoc DL(N);
17670
17671 SDValue Ops[] = {X, DAG.getSignedConstant(-1, DL, MVT::i32),
17672 DAG.getConstant(Width, DL, MVT::i32),
17673 DAG.getConstant(ShAmt, DL, MVT::i32)};
17674 return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
17675}
17676
17677// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
17678// being inserted only sets known zero bits.
17679static SDValue combineOrAndToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17680 const RISCVSubtarget &Subtarget) {
17681 // Supported only in Xqcibm for now.
17682 if (!Subtarget.hasVendorXqcibm())
17683 return SDValue();
17684
17685 using namespace SDPatternMatch;
17686
17687 SDValue Inserted;
17688 APInt MaskImm, OrImm;
17689 if (!sd_match(
17690 N, m_SpecificVT(MVT::i32, m_Or(m_OneUse(m_And(m_Value(Inserted),
17691 m_ConstInt(MaskImm))),
17692 m_ConstInt(OrImm)))))
17693 return SDValue();
17694
17695 // Compute the Known Zero for the AND as this allows us to catch more general
17696 // cases than just looking for AND with imm.
17697 KnownBits Known = DAG.computeKnownBits(N->getOperand(0));
17698
17699 // The bits being inserted must only set those bits that are known to be
17700 // zero.
17701 if (!OrImm.isSubsetOf(Known.Zero)) {
17702 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
17703 // currently handle this case.
17704 return SDValue();
17705 }
17706
17707 unsigned ShAmt, Width;
17708 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
17709 if (!Known.Zero.isShiftedMask(ShAmt, Width))
17710 return SDValue();
17711
17712 // QC_INSB(I) dst, src, #width, #shamt.
17713 SDLoc DL(N);
17714
17715 SDValue ImmNode =
17716 DAG.getSignedConstant(OrImm.getSExtValue() >> ShAmt, DL, MVT::i32);
17717
17718 SDValue Ops[] = {Inserted, ImmNode, DAG.getConstant(Width, DL, MVT::i32),
17719 DAG.getConstant(ShAmt, DL, MVT::i32)};
17720 return DAG.getNode(RISCVISD::QC_INSB, DL, MVT::i32, Ops);
17721}
17722
17723static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17724 const RISCVSubtarget &Subtarget) {
17725 SelectionDAG &DAG = DCI.DAG;
17726
17727 if (SDValue V = combineOrAndToBitfieldInsert(N, DAG, Subtarget))
17728 return V;
17729 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17730 return V;
17731 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17732 return V;
17733
17734 if (DCI.isAfterLegalizeDAG()) {
17735 if (SDValue V = combineOrToBitfieldInsert(N, DAG, Subtarget))
17736 return V;
17737 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
17738 return V;
17739 }
17740
17741 // Look for Or of CZERO_EQZ/NEZ with same condition which is the select idiom.
17742 // We may be able to pull a common operation out of the true and false value.
17743 SDValue N0 = N->getOperand(0);
17744 SDValue N1 = N->getOperand(1);
17745 if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
17746 return V;
17747 if (SDValue V = combineOrOfCZERO(N, N1, N0, DAG))
17748 return V;
17749
17750 // fold (or (select cond, 0, y), x) ->
17751 // (select cond, x, (or x, y))
17752 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
17753}
17754
17755static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
17756 const RISCVSubtarget &Subtarget) {
17757 SDValue N0 = N->getOperand(0);
17758 SDValue N1 = N->getOperand(1);
17759
17760 // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
17761 // (ADDI (BSET X0, X), -1). If we wait until type legalization, we'll create
17762 // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
17763 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
17764 N->getValueType(0) == MVT::i32 && isAllOnesConstant(N1) &&
17765 N0.getOpcode() == ISD::SHL && isAllOnesConstant(N0.getOperand(0)) &&
17766 !isa<ConstantSDNode>(N0.getOperand(1)) && N0.hasOneUse()) {
17767 SDLoc DL(N);
17768 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N0.getOperand(0));
17769 SDValue Op1 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N0.getOperand(1));
17770 SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i64, Op0, Op1);
17771 SDValue Not = DAG.getNOT(DL, Shl, MVT::i64);
17772 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Not);
17773 }
17774
17775 // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
17776 // NOTE: Assumes ROL being legal means ROLW is legal.
17777 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17778 if (N0.getOpcode() == RISCVISD::SLLW &&
17779 isAllOnesConstant(N1) && isOneConstant(N0.getOperand(0)) &&
17780 TLI.isOperationLegal(ISD::ROTL, MVT::i64)) {
17781 SDLoc DL(N);
17782 return DAG.getNode(RISCVISD::ROLW, DL, MVT::i64,
17783 DAG.getConstant(~1, DL, MVT::i64), N0.getOperand(1));
17784 }
17785
17786 // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
17787 if (N0.getOpcode() == ISD::SETCC && isOneConstant(N1) && N0.hasOneUse()) {
17788 auto *ConstN00 = dyn_cast<ConstantSDNode>(N0.getOperand(0));
17789 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
17790 if (ConstN00 && CC == ISD::SETLT) {
17791 EVT VT = N0.getValueType();
17792 SDLoc DL(N0);
17793 const APInt &Imm = ConstN00->getAPIntValue();
17794 if ((Imm + 1).isSignedIntN(12))
17795 return DAG.getSetCC(DL, VT, N0.getOperand(1),
17796 DAG.getConstant(Imm + 1, DL, VT), CC);
17797 }
17798 }
17799
17800 if (SDValue V = combineXorToBitfieldInsert(N, DAG, Subtarget))
17801 return V;
17802
17803 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17804 return V;
17805 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17806 return V;
17807
17808 // fold (xor (select cond, 0, y), x) ->
17809 // (select cond, x, (xor x, y))
17810 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
17811}
17812
17813// Try to expand a multiply to a sequence of shifts and add/subs,
17814// for a machine without native mul instruction.
17815static SDValue expandMulToNAFSequence(SDNode *N, SelectionDAG &DAG,
17816 uint64_t MulAmt) {
17817 SDLoc DL(N);
17818 EVT VT = N->getValueType(0);
17819 const uint64_t BitWidth = VT.getFixedSizeInBits();
17820
17821 SDValue Result = DAG.getConstant(0, DL, N->getValueType(0));
17822 SDValue N0 = N->getOperand(0);
17823
17824 // Find the Non-adjacent form of the multiplier.
17825 for (uint64_t E = MulAmt, I = 0; E && I < BitWidth; ++I, E >>= 1) {
17826 if (E & 1) {
17827 bool IsAdd = (E & 3) == 1;
17828 E -= IsAdd ? 1 : -1;
17829 SDValue ShiftVal = DAG.getNode(ISD::SHL, DL, VT, N0,
17830 DAG.getShiftAmountConstant(I, VT, DL));
17831 ISD::NodeType AddSubOp = IsAdd ? ISD::ADD : ISD::SUB;
17832 Result = DAG.getNode(AddSubOp, DL, VT, Result, ShiftVal);
17833 }
17834 }
17835
17836 return Result;
17837}
17838
17839// X * (2^N +/- 2^M) -> (add/sub (shl X, C1), (shl X, C2))
17840static SDValue expandMulToAddOrSubOfShl(SDNode *N, SelectionDAG &DAG,
17841 uint64_t MulAmt) {
17842 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
17843 SDValue X = N->getOperand(0);
17844 ISD::NodeType Op;
17845 uint64_t ShiftAmt1;
17846 bool CanSub = isPowerOf2_64(MulAmt + MulAmtLowBit);
17847 auto PreferSub = [X, MulAmtLowBit]() {
17848 // For MulAmt == 3 << M both (X << M + 2) - (X << M)
17849 // and (X << M + 1) + (X << M) are valid expansions.
17850 // Prefer SUB if we can get (X << M + 2) for free,
17851 // because X is exact (Y >> M + 2).
17852 uint64_t ShAmt = Log2_64(MulAmtLowBit) + 2;
17853 using namespace SDPatternMatch;
17854 return sd_match(X, m_ExactSr(m_Value(), m_SpecificInt(ShAmt)));
17855 };
17856 if (isPowerOf2_64(MulAmt - MulAmtLowBit) && !(CanSub && PreferSub())) {
17857 Op = ISD::ADD;
17858 ShiftAmt1 = MulAmt - MulAmtLowBit;
17859 } else if (CanSub) {
17860 Op = ISD::SUB;
17861 ShiftAmt1 = MulAmt + MulAmtLowBit;
17862 } else {
17863 return SDValue();
17864 }
17865 EVT VT = N->getValueType(0);
17866 SDLoc DL(N);
17867 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, X,
17868 DAG.getConstant(Log2_64(ShiftAmt1), DL, VT));
17869 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, X,
17870 DAG.getConstant(Log2_64(MulAmtLowBit), DL, VT));
17871 return DAG.getNode(Op, DL, VT, Shift1, Shift2);
17872}
17873
17874static SDValue getShlAddShlAdd(SDNode *N, SelectionDAG &DAG, unsigned ShX,
17875 unsigned ShY, bool AddX, unsigned Shift) {
17876 SDLoc DL(N);
17877 EVT VT = N->getValueType(0);
17878 SDValue X = N->getOperand(0);
17879 // Put the shift first if we can fold:
17880 // a. a zext into the shift forming a slli.uw
17881 // b. an exact shift right forming one shorter shift or no shift at all
17882 using namespace SDPatternMatch;
17883 if (Shift != 0 &&
17884 sd_match(X, m_AnyOf(m_And(m_Value(), m_SpecificInt(UINT64_C(0xffffffff))),
17885 m_ExactSr(m_Value(), m_ConstInt())))) {
17886 X = DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(Shift, DL, VT));
17887 Shift = 0;
17888 }
17889 SDValue ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
17890 DAG.getTargetConstant(ShY, DL, VT), X);
17891 if (ShX != 0)
17892 ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, ShlAdd,
17893 DAG.getTargetConstant(ShX, DL, VT), AddX ? X : ShlAdd);
17894 if (Shift == 0)
17895 return ShlAdd;
17896 // Otherwise, put the shl last so that it can fold with following instructions
17897 // (e.g. sext or add).
17898 return DAG.getNode(ISD::SHL, DL, VT, ShlAdd, DAG.getConstant(Shift, DL, VT));
17899}
17900
17901static SDValue expandMulToShlAddShlAdd(SDNode *N, SelectionDAG &DAG,
17902 uint64_t MulAmt, unsigned Shift) {
17903 switch (MulAmt) {
17904 // 3/5/9 -> (shYadd X, X)
17905 case 3:
17906 return getShlAddShlAdd(N, DAG, 0, 1, /*AddX=*/false, Shift);
17907 case 5:
17908 return getShlAddShlAdd(N, DAG, 0, 2, /*AddX=*/false, Shift);
17909 case 9:
17910 return getShlAddShlAdd(N, DAG, 0, 3, /*AddX=*/false, Shift);
17911 // 3/5/9 * 3/5/9 -> (shXadd (shYadd X, X), (shYadd X, X))
17912 case 5 * 3:
17913 return getShlAddShlAdd(N, DAG, 2, 1, /*AddX=*/false, Shift);
17914 case 9 * 3:
17915 return getShlAddShlAdd(N, DAG, 3, 1, /*AddX=*/false, Shift);
17916 case 5 * 5:
17917 return getShlAddShlAdd(N, DAG, 2, 2, /*AddX=*/false, Shift);
17918 case 9 * 5:
17919 return getShlAddShlAdd(N, DAG, 3, 2, /*AddX=*/false, Shift);
17920 case 9 * 9:
17921 return getShlAddShlAdd(N, DAG, 3, 3, /*AddX=*/false, Shift);
17922 default:
17923 break;
17924 }
17925
17926 int ShX;
17927 if (int ShY = isShifted359(MulAmt - 1, ShX)) {
17928 assert(ShX != 0 && "MulAmt=4,6,10 handled before");
17929 // 2/4/8 * 3/5/9 + 1 -> (shXadd (shYadd X, X), X)
17930 if (ShX <= 3)
17931 return getShlAddShlAdd(N, DAG, ShX, ShY, /*AddX=*/true, Shift);
17932 // 2^N * 3/5/9 + 1 -> (add (shYadd (shl X, N), (shl X, N)), X)
17933 if (Shift == 0) {
17934 SDLoc DL(N);
17935 EVT VT = N->getValueType(0);
17936 SDValue X = N->getOperand(0);
17937 SDValue Shl =
17938 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShX, DL, VT));
17939 SDValue ShlAdd = DAG.getNode(RISCVISD::SHL_ADD, DL, VT, Shl,
17940 DAG.getTargetConstant(ShY, DL, VT), Shl);
17941 return DAG.getNode(ISD::ADD, DL, VT, ShlAdd, X);
17942 }
17943 }
17944 return SDValue();
17945}
17946
17947// Try to expand a scalar multiply to a faster sequence.
17948static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
17949 TargetLowering::DAGCombinerInfo &DCI,
17950 const RISCVSubtarget &Subtarget) {
17951
17952 EVT VT = N->getValueType(0);
17953
17954 // LI + MUL is usually smaller than the alternative sequence.
17955 if (DAG.getMachineFunction().getFunction().hasMinSize())
17956 return SDValue();
17957
17958 if (VT != Subtarget.getXLenVT())
17959 return SDValue();
17960
17961 bool ShouldExpandMul =
17962 (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) ||
17963 !Subtarget.hasStdExtZmmul();
17964 if (!ShouldExpandMul)
17965 return SDValue();
17966
17967 ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(N->getOperand(1));
17968 if (!CNode)
17969 return SDValue();
17970 uint64_t MulAmt = CNode->getZExtValue();
17971
17972 // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12.
17973 if (Subtarget.hasVendorXqciac() && isInt<12>(CNode->getSExtValue()))
17974 return SDValue();
17975
17976 // WARNING: The code below is knowingly incorrect with regards to undef
17977 // semantics. We're adding additional uses of X here, and in principle, we
17978 // should be freezing X before doing so. However, adding freeze here causes
17979 // real regressions, and no other target properly freezes X in these cases
17980 // either.
17981 if (Subtarget.hasShlAdd(3)) {
17982 // 3/5/9 * 2^N -> (shl (shXadd X, X), N)
17983 // 3/5/9 * 3/5/9 * 2^N - In particular, this covers multiples
17984 // of 25 which happen to be quite common.
17985 // (2/4/8 * 3/5/9 + 1) * 2^N
17986 unsigned Shift = llvm::countr_zero(MulAmt);
17987 if (SDValue V = expandMulToShlAddShlAdd(N, DAG, MulAmt >> Shift, Shift))
17988 return V;
17989
17990 // If this is a power 2 + 2/4/8, we can use a shift followed by a single
17991 // shXadd. First check if this a sum of two power of 2s because that's
17992 // easy. Then count how many zeros are up to the first bit.
17993 SDValue X = N->getOperand(0);
17994 if (Shift >= 1 && Shift <= 3 && isPowerOf2_64(MulAmt & (MulAmt - 1))) {
17995 unsigned ShiftAmt = llvm::countr_zero((MulAmt & (MulAmt - 1)));
17996 SDLoc DL(N);
17997 SDValue Shift1 =
17998 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
17999 return DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
18000 DAG.getTargetConstant(Shift, DL, VT), Shift1);
18001 }
18002
18003 // TODO: 2^(C1>3) * 3/5/9 - 1
18004
18005 // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
18006 if (MulAmt > 2 && isPowerOf2_64((MulAmt - 1) & (MulAmt - 2))) {
18007 unsigned ScaleShift = llvm::countr_zero(MulAmt - 1);
18008 if (ScaleShift >= 1 && ScaleShift < 4) {
18009 unsigned ShiftAmt = llvm::countr_zero((MulAmt - 1) & (MulAmt - 2));
18010 SDLoc DL(N);
18011 SDValue Shift1 =
18012 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShiftAmt, DL, VT));
18013 return DAG.getNode(
18014 ISD::ADD, DL, VT, Shift1,
18015 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
18016 DAG.getTargetConstant(ScaleShift, DL, VT), X));
18017 }
18018 }
18019
18020 // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
18021 for (uint64_t Offset : {3, 5, 9}) {
18022 if (isPowerOf2_64(MulAmt + Offset)) {
18023 unsigned ShAmt = llvm::countr_zero(MulAmt + Offset);
18024 if (ShAmt >= VT.getSizeInBits())
18025 continue;
18026 SDLoc DL(N);
18027 SDValue Shift1 =
18028 DAG.getNode(ISD::SHL, DL, VT, X, DAG.getConstant(ShAmt, DL, VT));
18029 SDValue Mul359 =
18030 DAG.getNode(RISCVISD::SHL_ADD, DL, VT, X,
18031 DAG.getTargetConstant(Log2_64(Offset - 1), DL, VT), X);
18032 return DAG.getNode(ISD::SUB, DL, VT, Shift1, Mul359);
18033 }
18034 }
18035 }
18036
18037 if (SDValue V = expandMulToAddOrSubOfShl(N, DAG, MulAmt))
18038 return V;
18039
18040 if (!Subtarget.hasStdExtZmmul())
18041 return expandMulToNAFSequence(N, DAG, MulAmt);
18042
18043 return SDValue();
18044}
18045
18046// Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
18047// (bitcast (sra (v2Xi16 (bitcast X)), 15))
18048// Same for other equivalent types with other equivalent constants.
18049static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
18050 EVT VT = N->getValueType(0);
18051 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18052
18053 // Do this for legal vectors unless they are i1 or i8 vectors.
18054 if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
18055 return SDValue();
18056
18057 if (N->getOperand(0).getOpcode() != ISD::AND ||
18058 N->getOperand(0).getOperand(0).getOpcode() != ISD::SRL)
18059 return SDValue();
18060
18061 SDValue And = N->getOperand(0);
18062 SDValue Srl = And.getOperand(0);
18063
18064 APInt V1, V2, V3;
18065 if (!ISD::isConstantSplatVector(N->getOperand(1).getNode(), V1) ||
18066 !ISD::isConstantSplatVector(And.getOperand(1).getNode(), V2) ||
18067 !ISD::isConstantSplatVector(Srl.getOperand(1).getNode(), V3))
18068 return SDValue();
18069
18070 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
18071 if (!V1.isMask(HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
18072 V3 != (HalfSize - 1))
18073 return SDValue();
18074
18075 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(),
18076 EVT::getIntegerVT(*DAG.getContext(), HalfSize),
18077 VT.getVectorElementCount() * 2);
18078 SDLoc DL(N);
18079 SDValue Cast = DAG.getNode(ISD::BITCAST, DL, HalfVT, Srl.getOperand(0));
18080 SDValue Sra = DAG.getNode(ISD::SRA, DL, HalfVT, Cast,
18081 DAG.getConstant(HalfSize - 1, DL, HalfVT));
18082 return DAG.getNode(ISD::BITCAST, DL, VT, Sra);
18083}
18084
18085static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
18086 TargetLowering::DAGCombinerInfo &DCI,
18087 const RISCVSubtarget &Subtarget) {
18088 EVT VT = N->getValueType(0);
18089 if (!VT.isVector())
18090 return expandMul(N, DAG, DCI, Subtarget);
18091
18092 SDLoc DL(N);
18093 SDValue N0 = N->getOperand(0);
18094 SDValue N1 = N->getOperand(1);
18095 SDValue MulOper;
18096 unsigned AddSubOpc;
18097
18098 // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
18099 // (mul x, add (y, 1)) -> (add x, (mul x, y))
18100 // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
18101 // (mul x, (sub 1, y)) -> (sub x, (mul x, y))
18102 auto IsAddSubWith1 = [&](SDValue V) -> bool {
18103 AddSubOpc = V->getOpcode();
18104 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
18105 SDValue Opnd = V->getOperand(1);
18106 MulOper = V->getOperand(0);
18107 if (AddSubOpc == ISD::SUB)
18108 std::swap(Opnd, MulOper);
18109 if (isOneOrOneSplat(Opnd))
18110 return true;
18111 }
18112 return false;
18113 };
18114
18115 if (IsAddSubWith1(N0)) {
18116 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N1, MulOper);
18117 return DAG.getNode(AddSubOpc, DL, VT, N1, MulVal);
18118 }
18119
18120 if (IsAddSubWith1(N1)) {
18121 SDValue MulVal = DAG.getNode(ISD::MUL, DL, VT, N0, MulOper);
18122 return DAG.getNode(AddSubOpc, DL, VT, N0, MulVal);
18123 }
18124
18125 if (SDValue V = combineBinOpOfZExt(N, DAG))
18126 return V;
18127
18128 if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
18129 return V;
18130
18131 return SDValue();
18132}
18133
18134/// According to the property that indexed load/store instructions zero-extend
18135/// their indices, try to narrow the type of index operand.
18136static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
18137 if (isIndexTypeSigned(IndexType))
18138 return false;
18139
18140 if (!N->hasOneUse())
18141 return false;
18142
18143 EVT VT = N.getValueType();
18144 SDLoc DL(N);
18145
18146 // In general, what we're doing here is seeing if we can sink a truncate to
18147 // a smaller element type into the expression tree building our index.
18148 // TODO: We can generalize this and handle a bunch more cases if useful.
18149
18150 // Narrow a buildvector to the narrowest element type. This requires less
18151 // work and less register pressure at high LMUL, and creates smaller constants
18152 // which may be cheaper to materialize.
18153 if (ISD::isBuildVectorOfConstantSDNodes(N.getNode())) {
18154 KnownBits Known = DAG.computeKnownBits(N);
18155 unsigned ActiveBits = std::max(8u, Known.countMaxActiveBits());
18156 LLVMContext &C = *DAG.getContext();
18157 EVT ResultVT = EVT::getIntegerVT(C, ActiveBits).getRoundIntegerType(C);
18158 if (ResultVT.bitsLT(VT.getVectorElementType())) {
18159 N = DAG.getNode(ISD::TRUNCATE, DL,
18160 VT.changeVectorElementType(C, ResultVT), N);
18161 return true;
18162 }
18163 }
18164
18165 // Handle the pattern (shl (zext x to ty), C) and bits(x) + C < bits(ty).
18166 if (N.getOpcode() != ISD::SHL)
18167 return false;
18168
18169 SDValue N0 = N.getOperand(0);
18170 if (N0.getOpcode() != ISD::ZERO_EXTEND &&
18171 N0.getOpcode() != RISCVISD::VZEXT_VL)
18172 return false;
18173 if (!N0->hasOneUse())
18174 return false;
18175
18176 APInt ShAmt;
18177 SDValue N1 = N.getOperand(1);
18178 if (!ISD::isConstantSplatVector(N1.getNode(), ShAmt))
18179 return false;
18180
18181 SDValue Src = N0.getOperand(0);
18182 EVT SrcVT = Src.getValueType();
18183 unsigned SrcElen = SrcVT.getScalarSizeInBits();
18184
18185 // Consider any leading zeros in the source.
18186 SrcElen -= DAG.computeKnownBits(Src).countMinLeadingZeros();
18187
18188 unsigned ShAmtV = ShAmt.getZExtValue();
18189 unsigned NewElen = PowerOf2Ceil(SrcElen + ShAmtV);
18190 NewElen = std::max(NewElen, 8U);
18191 // Make sure the new elen is at least as large as the original elen.
18192 NewElen = std::max<unsigned>(NewElen, SrcVT.getScalarSizeInBits());
18193
18194 // Skip if NewElen is not narrower than the original extended type.
18195 if (NewElen >= N0.getValueType().getScalarSizeInBits())
18196 return false;
18197
18198 EVT NewEltVT = EVT::getIntegerVT(*DAG.getContext(), NewElen);
18199 EVT NewVT = SrcVT.changeVectorElementType(*DAG.getContext(), NewEltVT);
18200
18201 SDValue NewExt = DAG.getNode(N0->getOpcode(), DL, NewVT, N0->ops());
18202 SDValue NewShAmtVec = DAG.getConstant(ShAmtV, DL, NewVT);
18203 N = DAG.getNode(ISD::SHL, DL, NewVT, NewExt, NewShAmtVec);
18204 return true;
18205}
18206
18207/// Try to map an integer comparison with size > XLEN to vector instructions
18208/// before type legalization splits it up into chunks.
18209static SDValue
18210combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
18211 const SDLoc &DL, SelectionDAG &DAG,
18212 const RISCVSubtarget &Subtarget) {
18213 assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
18214
18215 if (!Subtarget.useRVVForFixedLengthVectors())
18216 return SDValue();
18217
18218 MVT XLenVT = Subtarget.getXLenVT();
18219 EVT OpVT = X.getValueType();
18220 // We're looking for an oversized integer equality comparison.
18221 if (!OpVT.isScalarInteger())
18222 return SDValue();
18223
18224 unsigned OpSize = OpVT.getSizeInBits();
18225 // The size should be larger than XLen and smaller than the maximum vector
18226 // size.
18227 if (OpSize <= Subtarget.getXLen() ||
18228 OpSize > Subtarget.getRealMinVLen() *
18229 Subtarget.getMaxLMULForFixedLengthVectors())
18230 return SDValue();
18231
18232 // Don't perform this combine if constructing the vector will be expensive.
18233 auto IsVectorBitCastCheap = [](SDValue X) {
18234 X = peekThroughBitcasts(X);
18235 return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
18236 X.getOpcode() == ISD::LOAD;
18237 };
18238 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
18239 return SDValue();
18240
18241 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
18242 Attribute::NoImplicitFloat))
18243 return SDValue();
18244
18245 // Bail out for non-byte-sized types.
18246 if (!OpVT.isByteSized())
18247 return SDValue();
18248
18249 // Find a preferred vector element type by inspecting how the value is used.
18250 auto GetPreferredEltVT = [](SDValue V) -> MVT {
18251 // Look backwards: check if V itself is derived from a vector
18252 SDValue Peek = peekThroughBitcasts(V);
18253 EVT PeekVT = Peek.getValueType();
18254
18255 if (PeekVT.isVector() && PeekVT.isInteger()) {
18256 EVT EltVT = PeekVT.getVectorElementType();
18257 if (EltVT.isSimple())
18258 return EltVT.getSimpleVT();
18259 }
18260
18261 // Look forwards: check if V is bitcasted to a vector elsewhere in the DAG
18262 for (SDUse &Use : V->uses()) {
18263 // Ensure we are checking the use of the specific value result, not the
18264 // node's chain
18265 if (Use.getResNo() != V.getResNo())
18266 continue;
18267
18268 SDNode *User = Use.getUser();
18269 if (User->getOpcode() == ISD::BITCAST) {
18270 EVT CastVT = User->getValueType(0);
18271 if (CastVT.isVector() && CastVT.isInteger()) {
18272 EVT EltVT = CastVT.getVectorElementType();
18273 if (EltVT.isSimple())
18274 return EltVT.getSimpleVT();
18275 }
18276 }
18277 }
18278 return MVT::INVALID_SIMPLE_VALUE_TYPE;
18279 };
18280
18281 auto IsValidEltVT = [&](MVT VT) {
18282 if (VT == MVT::INVALID_SIMPLE_VALUE_TYPE || !VT.isInteger())
18283 return false;
18284
18285 // Make sure we don't try to create an impossible vector type where the
18286 // elements don't perfectly fill up the OpSize boundary.
18287 unsigned EltSize = VT.getSizeInBits();
18288 if (OpSize % EltSize != 0)
18289 return false;
18290
18291 // Construct the proposed vector type to check its legality
18292 unsigned NumElts = OpSize / EltSize;
18293 EVT TestVecVT = EVT::getVectorVT(*DAG.getContext(), VT, NumElts);
18294 return Subtarget.getTargetLowering()->isTypeLegal(TestVecVT);
18295 };
18296
18297 // Get preferred VT from either X or Y.
18298 MVT EltVT = GetPreferredEltVT(X);
18299 if (!IsValidEltVT(EltVT))
18300 EltVT = GetPreferredEltVT(Y);
18301
18302 // If both are unsuitable, use the safe default (i8)
18303 if (!IsValidEltVT(EltVT))
18304 EltVT = MVT::i8;
18305
18306 unsigned EltSize = EltVT.getSizeInBits();
18307 unsigned NumElts = OpSize / EltSize;
18308 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NumElts);
18309 EVT CmpVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
18310
18311 SDValue VecX = DAG.getBitcast(VecVT, X);
18312 SDValue VecY;
18313 // Constant fold the common case of comparing with zero. Later optimizations
18314 // might not do this for us.
18315 if (isNullConstant(Y))
18316 VecY = DAG.getConstant(0, DL, VecVT);
18317 else
18318 VecY = DAG.getBitcast(VecVT, Y);
18319
18320 SDValue Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
18321 return DAG.getSetCC(DL, VT, DAG.getNode(ISD::VECREDUCE_OR, DL, XLenVT, Cmp),
18322 DAG.getConstant(0, DL, XLenVT), CC);
18323}
18324
18325static SDValue performSETCCCombine(SDNode *N,
18326 TargetLowering::DAGCombinerInfo &DCI,
18327 const RISCVSubtarget &Subtarget) {
18328 SelectionDAG &DAG = DCI.DAG;
18329 SDLoc dl(N);
18330 SDValue N0 = N->getOperand(0);
18331 SDValue N1 = N->getOperand(1);
18332 EVT VT = N->getValueType(0);
18333 EVT OpVT = N0.getValueType();
18334
18335 ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
18336 // Looking for an equality compare.
18337 if (!isIntEqualitySetCC(Cond))
18338 return SDValue();
18339
18340 if (SDValue V =
18341 combineVectorSizedSetCCEquality(VT, N0, N1, Cond, dl, DAG, Subtarget))
18342 return V;
18343
18344 if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(N1) &&
18345 N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
18346 isa<ConstantSDNode>(N0.getOperand(1))) {
18347 const APInt &AndRHSC = N0.getConstantOperandAPInt(1);
18348 // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI.
18349 if (isNullConstant(N1) && !isInt<12>(AndRHSC.getSExtValue()) &&
18350 AndRHSC.isNegatedPowerOf2()) {
18351 unsigned ShiftBits = AndRHSC.countr_zero();
18352 SDValue Shift = DAG.getNode(ISD::SRL, dl, OpVT, N0.getOperand(0),
18353 DAG.getConstant(ShiftBits, dl, OpVT));
18354 return DAG.getSetCC(dl, VT, Shift, N1, Cond);
18355 }
18356
18357 // Similar to above but handling the lower 32 bits by using sraiw. Allow
18358 // comparing with constants other than 0 if the constant can be folded into
18359 // addi or xori after shifting.
18360 uint64_t N1Int = cast<ConstantSDNode>(N1)->getZExtValue();
18361 uint64_t AndRHSInt = AndRHSC.getZExtValue();
18362 if (OpVT == MVT::i64 && isUInt<32>(AndRHSInt) &&
18363 isPowerOf2_32(-uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) {
18364 unsigned ShiftBits = llvm::countr_zero(AndRHSInt);
18365 int64_t NewC = SignExtend64<32>(N1Int) >> ShiftBits;
18366 if (NewC >= -2048 && NewC <= 2048) {
18367 SDValue SExt =
18368 DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OpVT, N0.getOperand(0),
18369 DAG.getValueType(MVT::i32));
18370 SDValue Shift = DAG.getNode(ISD::SRA, dl, OpVT, SExt,
18371 DAG.getConstant(ShiftBits, dl, OpVT));
18372 return DAG.getSetCC(dl, VT, Shift,
18373 DAG.getSignedConstant(NewC, dl, OpVT), Cond);
18374 }
18375 }
18376
18377 // Fold (and X, Mask) ==/!= C -> X ==/!= sext(C, countr_one(Mask)) if the
18378 // Mask is only clearing redundant sign bits.
18379 if (isMask_64(AndRHSInt)) {
18380 unsigned TrailingOnes = llvm::countr_one(AndRHSInt);
18381 unsigned N1Width = llvm::bit_width(N1Int);
18382 int64_t N1SExt = SignExtend64(N1Int, TrailingOnes);
18383 if (N1Width <= TrailingOnes && isInt<12>(N1SExt) &&
18384 DAG.ComputeMaxSignificantBits(N0.getOperand(0)) <= TrailingOnes)
18385 return DAG.getSetCC(dl, VT, N0.getOperand(0),
18386 DAG.getSignedConstant(N1SExt, dl, OpVT), Cond);
18387 }
18388 }
18389
18390 // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
18391 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
18392 // bit 31. Same for setne. C1' may be cheaper to materialize and the
18393 // sext_inreg can become a sext.w instead of a shift pair.
18394 if (OpVT != MVT::i64 || !Subtarget.is64Bit())
18395 return SDValue();
18396
18397 // RHS needs to be a constant.
18398 auto *N1C = dyn_cast<ConstantSDNode>(N1);
18399 if (!N1C)
18400 return SDValue();
18401
18402 // LHS needs to be (and X, 0xffffffff).
18403 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
18404 !isa<ConstantSDNode>(N0.getOperand(1)) ||
18405 N0.getConstantOperandVal(1) != UINT64_C(0xffffffff))
18406 return SDValue();
18407
18408 // Don't do this if the sign bit is provably zero, it will be turned back into
18409 // an AND.
18410 APInt SignMask = APInt::getOneBitSet(64, 31);
18411 if (DAG.MaskedValueIsZero(N0.getOperand(0), SignMask))
18412 return SDValue();
18413
18414 const APInt &C1 = N1C->getAPIntValue();
18415
18416 // If the constant is larger than 2^32 - 1 it is impossible for both sides
18417 // to be equal.
18418 if (C1.getActiveBits() > 32)
18419 return DAG.getBoolConstant(Cond == ISD::SETNE, dl, VT, OpVT);
18420
18421 SDValue SExtOp = DAG.getNode(ISD::SIGN_EXTEND_INREG, N, OpVT,
18422 N0.getOperand(0), DAG.getValueType(MVT::i32));
18423 return DAG.getSetCC(dl, VT, SExtOp, DAG.getConstant(C1.trunc(32).sext(64),
18424 dl, OpVT), Cond);
18425}
18426
18427static SDValue
18428performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
18429 const RISCVSubtarget &Subtarget) {
18430 SelectionDAG &DAG = DCI.DAG;
18431 SDValue Src = N->getOperand(0);
18432 EVT VT = N->getValueType(0);
18433 EVT SrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
18434 unsigned Opc = Src.getOpcode();
18435 SDLoc DL(N);
18436
18437 // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
18438 // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
18439 if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(MVT::i16) &&
18440 Subtarget.hasStdExtZfhmin())
18441 return DAG.getNode(RISCVISD::FMV_X_SIGNEXTH, DL, VT, Src.getOperand(0));
18442
18443 // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
18444 if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
18445 VT == MVT::i64 && !isa<ConstantSDNode>(Src.getOperand(1)) &&
18446 DAG.computeKnownBits(Src.getOperand(1)).countMaxActiveBits() <= 5)
18447 return DAG.getNode(RISCVISD::SLLW, DL, VT, Src.getOperand(0),
18448 Src.getOperand(1));
18449
18450 // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
18451 if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
18452 return DAG.getNegative(Src, DL, VT);
18453
18454 // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
18455 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
18456 isAllOnesConstant(Src.getOperand(1)) &&
18457 Src.getOperand(0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
18458 return DAG.getNode(ISD::ADD, DL, VT, Src.getOperand(0),
18459 DAG.getAllOnesConstant(DL, VT));
18460
18461 return SDValue();
18462}
18463
18464namespace {
18465// Forward declaration of the structure holding the necessary information to
18466// apply a combine.
18467struct CombineResult;
18468
18469enum ExtKind : uint8_t {
18470 ZExt = 1 << 0,
18471 SExt = 1 << 1,
18472 FPExt = 1 << 2,
18473 BF16Ext = 1 << 3
18474};
18475/// Helper class for folding sign/zero extensions.
18476/// In particular, this class is used for the following combines:
18477/// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
18478/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
18479/// mul | mul_vl -> vwmul(u) | vwmul_su
18480/// shl | shl_vl -> vwsll
18481/// fadd -> vfwadd | vfwadd_w
18482/// fsub -> vfwsub | vfwsub_w
18483/// fmul -> vfwmul
18484/// An object of this class represents an operand of the operation we want to
18485/// combine.
18486/// E.g., when trying to combine `mul_vl a, b`, we will have one instance of
18487/// NodeExtensionHelper for `a` and one for `b`.
18488///
18489/// This class abstracts away how the extension is materialized and
18490/// how its number of users affect the combines.
18491///
18492/// In particular:
18493/// - VWADD_W is conceptually == add(op0, sext(op1))
18494/// - VWADDU_W == add(op0, zext(op1))
18495/// - VWSUB_W == sub(op0, sext(op1))
18496/// - VWSUBU_W == sub(op0, zext(op1))
18497/// - VFWADD_W == fadd(op0, fpext(op1))
18498/// - VFWSUB_W == fsub(op0, fpext(op1))
18499/// And VMV_V_X_VL, depending on the value, is conceptually equivalent to
18500/// zext|sext(smaller_value).
18501struct NodeExtensionHelper {
18502 /// Records if this operand is like being zero extended.
18503 bool SupportsZExt;
18504 /// Records if this operand is like being sign extended.
18505 /// Note: SupportsZExt and SupportsSExt are not mutually exclusive. For
18506 /// instance, a splat constant (e.g., 3), would support being both sign and
18507 /// zero extended.
18508 bool SupportsSExt;
18509 /// Records if this operand is like being floating point extended.
18510 bool SupportsFPExt;
18511 /// Records if this operand is extended from bf16.
18512 bool SupportsBF16Ext;
18513 /// This boolean captures whether we care if this operand would still be
18514 /// around after the folding happens.
18515 bool EnforceOneUse;
18516 /// Original value that this NodeExtensionHelper represents.
18517 SDValue OrigOperand;
18518
18519 /// Get the value feeding the extension or the value itself.
18520 /// E.g., for zext(a), this would return a.
18521 SDValue getSource() const {
18522 switch (OrigOperand.getOpcode()) {
18523 case ISD::ZERO_EXTEND:
18524 case ISD::SIGN_EXTEND:
18525 case RISCVISD::VSEXT_VL:
18526 case RISCVISD::VZEXT_VL:
18527 case RISCVISD::FP_EXTEND_VL:
18528 return OrigOperand.getOperand(0);
18529 default:
18530 return OrigOperand;
18531 }
18532 }
18533
18534 /// Check if this instance represents a splat.
18535 bool isSplat() const {
18536 return OrigOperand.getOpcode() == RISCVISD::VMV_V_X_VL ||
18537 OrigOperand.getOpcode() == ISD::SPLAT_VECTOR;
18538 }
18539
18540 /// Get the extended opcode.
18541 unsigned getExtOpc(ExtKind SupportsExt) const {
18542 switch (SupportsExt) {
18543 case ExtKind::SExt:
18544 return RISCVISD::VSEXT_VL;
18545 case ExtKind::ZExt:
18546 return RISCVISD::VZEXT_VL;
18547 case ExtKind::FPExt:
18548 case ExtKind::BF16Ext:
18549 return RISCVISD::FP_EXTEND_VL;
18550 }
18551 llvm_unreachable("Unknown ExtKind enum");
18552 }
18553
18554 /// Get or create a value that can feed \p Root with the given extension \p
18555 /// SupportsExt. If \p SExt is std::nullopt, this returns the source of this
18556 /// operand. \see ::getSource().
18557 SDValue getOrCreateExtendedOp(SDNode *Root, SelectionDAG &DAG,
18558 const RISCVSubtarget &Subtarget,
18559 std::optional<ExtKind> SupportsExt) const {
18560 if (!SupportsExt.has_value())
18561 return OrigOperand;
18562
18563 MVT NarrowVT = getNarrowType(Root, *SupportsExt);
18564
18565 SDValue Source = getSource();
18566 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType()));
18567 if (Source.getValueType() == NarrowVT)
18568 return Source;
18569
18570 unsigned ExtOpc = getExtOpc(*SupportsExt);
18571
18572 // If we need an extension, we should be changing the type.
18573 SDLoc DL(OrigOperand);
18574 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
18575 switch (OrigOperand.getOpcode()) {
18576 case ISD::ZERO_EXTEND:
18577 case ISD::SIGN_EXTEND:
18578 case RISCVISD::VSEXT_VL:
18579 case RISCVISD::VZEXT_VL:
18580 case RISCVISD::FP_EXTEND_VL:
18581 return DAG.getNode(ExtOpc, DL, NarrowVT, Source, Mask, VL);
18582 case ISD::SPLAT_VECTOR:
18583 return DAG.getSplat(NarrowVT, DL, Source.getOperand(0));
18584 case RISCVISD::VMV_V_X_VL:
18585 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, NarrowVT,
18586 DAG.getUNDEF(NarrowVT), Source.getOperand(1), VL);
18587 case RISCVISD::VFMV_V_F_VL:
18588 Source = Source.getOperand(1);
18589 assert(Source.getOpcode() == ISD::FP_EXTEND && "Unexpected source");
18590 Source = Source.getOperand(0);
18591 assert(Source.getValueType() == NarrowVT.getVectorElementType());
18592 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, NarrowVT,
18593 DAG.getUNDEF(NarrowVT), Source, VL);
18594 default:
18595 // Other opcodes can only come from the original LHS of VW(ADD|SUB)_W_VL
18596 // and that operand should already have the right NarrowVT so no
18597 // extension should be required at this point.
18598 llvm_unreachable("Unsupported opcode");
18599 }
18600 }
18601
18602 /// Helper function to get the narrow type for \p Root.
18603 /// The narrow type is the type of \p Root where we divided the size of each
18604 /// element by 2. E.g., if Root's type <2xi16> -> narrow type <2xi8>.
18605 /// \pre Both the narrow type and the original type should be legal.
18606 static MVT getNarrowType(const SDNode *Root, ExtKind SupportsExt) {
18607 MVT VT = Root->getSimpleValueType(0);
18608
18609 // Determine the narrow size.
18610 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
18611
18612 MVT EltVT = SupportsExt == ExtKind::BF16Ext ? MVT::bf16
18613 : SupportsExt == ExtKind::FPExt
18614 ? MVT::getFloatingPointVT(NarrowSize)
18615 : MVT::getIntegerVT(NarrowSize);
18616
18617 assert((int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
18618 "Trying to extend something we can't represent");
18619 MVT NarrowVT = MVT::getVectorVT(EltVT, VT.getVectorElementCount());
18620 return NarrowVT;
18621 }
18622
18623 /// Get the opcode to materialize:
18624 /// Opcode(sext(a), sext(b)) -> newOpcode(a, b)
18625 static unsigned getSExtOpcode(unsigned Opcode) {
18626 switch (Opcode) {
18627 case ISD::ADD:
18628 case RISCVISD::ADD_VL:
18629 case RISCVISD::VWADD_W_VL:
18630 case RISCVISD::VWADDU_W_VL:
18631 case ISD::OR:
18632 case RISCVISD::OR_VL:
18633 return RISCVISD::VWADD_VL;
18634 case ISD::SUB:
18635 case RISCVISD::SUB_VL:
18636 case RISCVISD::VWSUB_W_VL:
18637 case RISCVISD::VWSUBU_W_VL:
18638 return RISCVISD::VWSUB_VL;
18639 case ISD::MUL:
18640 case RISCVISD::MUL_VL:
18641 return RISCVISD::VWMUL_VL;
18642 default:
18643 llvm_unreachable("Unexpected opcode");
18644 }
18645 }
18646
18647 /// Get the opcode to materialize:
18648 /// Opcode(zext(a), zext(b)) -> newOpcode(a, b)
18649 static unsigned getZExtOpcode(unsigned Opcode) {
18650 switch (Opcode) {
18651 case ISD::ADD:
18652 case RISCVISD::ADD_VL:
18653 case RISCVISD::VWADD_W_VL:
18654 case RISCVISD::VWADDU_W_VL:
18655 case ISD::OR:
18656 case RISCVISD::OR_VL:
18657 return RISCVISD::VWADDU_VL;
18658 case ISD::SUB:
18659 case RISCVISD::SUB_VL:
18660 case RISCVISD::VWSUB_W_VL:
18661 case RISCVISD::VWSUBU_W_VL:
18662 return RISCVISD::VWSUBU_VL;
18663 case ISD::MUL:
18664 case RISCVISD::MUL_VL:
18665 return RISCVISD::VWMULU_VL;
18666 case ISD::SHL:
18667 case RISCVISD::SHL_VL:
18668 return RISCVISD::VWSLL_VL;
18669 default:
18670 llvm_unreachable("Unexpected opcode");
18671 }
18672 }
18673
18674 /// Get the opcode to materialize:
18675 /// Opcode(fpext(a), fpext(b)) -> newOpcode(a, b)
18676 static unsigned getFPExtOpcode(unsigned Opcode) {
18677 switch (Opcode) {
18678 case RISCVISD::FADD_VL:
18679 case RISCVISD::VFWADD_W_VL:
18680 return RISCVISD::VFWADD_VL;
18681 case RISCVISD::FSUB_VL:
18682 case RISCVISD::VFWSUB_W_VL:
18683 return RISCVISD::VFWSUB_VL;
18684 case RISCVISD::FMUL_VL:
18685 return RISCVISD::VFWMUL_VL;
18686 case RISCVISD::VFMADD_VL:
18687 return RISCVISD::VFWMADD_VL;
18688 case RISCVISD::VFMSUB_VL:
18689 return RISCVISD::VFWMSUB_VL;
18690 case RISCVISD::VFNMADD_VL:
18691 return RISCVISD::VFWNMADD_VL;
18692 case RISCVISD::VFNMSUB_VL:
18693 return RISCVISD::VFWNMSUB_VL;
18694 default:
18695 llvm_unreachable("Unexpected opcode");
18696 }
18697 }
18698
18699 /// Get the opcode to materialize \p Opcode(sext(a), zext(b)) ->
18700 /// newOpcode(a, b).
18701 static unsigned getSUOpcode(unsigned Opcode) {
18702 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
18703 "SU is only supported for MUL");
18704 return RISCVISD::VWMULSU_VL;
18705 }
18706
18707 /// Get the opcode to materialize
18708 /// \p Opcode(a, s|z|fpext(b)) -> newOpcode(a, b).
18709 static unsigned getWOpcode(unsigned Opcode, ExtKind SupportsExt) {
18710 switch (Opcode) {
18711 case ISD::ADD:
18712 case RISCVISD::ADD_VL:
18713 case ISD::OR:
18714 case RISCVISD::OR_VL:
18715 return SupportsExt == ExtKind::SExt ? RISCVISD::VWADD_W_VL
18716 : RISCVISD::VWADDU_W_VL;
18717 case ISD::SUB:
18718 case RISCVISD::SUB_VL:
18719 return SupportsExt == ExtKind::SExt ? RISCVISD::VWSUB_W_VL
18720 : RISCVISD::VWSUBU_W_VL;
18721 case RISCVISD::FADD_VL:
18722 return RISCVISD::VFWADD_W_VL;
18723 case RISCVISD::FSUB_VL:
18724 return RISCVISD::VFWSUB_W_VL;
18725 default:
18726 llvm_unreachable("Unexpected opcode");
18727 }
18728 }
18729
18730 using CombineToTry = std::function<std::optional<CombineResult>(
18731 SDNode * /*Root*/, const NodeExtensionHelper & /*LHS*/,
18732 const NodeExtensionHelper & /*RHS*/, SelectionDAG &,
18733 const RISCVSubtarget &)>;
18734
18735 /// Check if this node needs to be fully folded or extended for all users.
18736 bool needToPromoteOtherUsers() const { return EnforceOneUse; }
18737
18738 void fillUpExtensionSupportForSplat(SDNode *Root, SelectionDAG &DAG,
18739 const RISCVSubtarget &Subtarget) {
18740 unsigned Opc = OrigOperand.getOpcode();
18741 MVT VT = OrigOperand.getSimpleValueType();
18742
18743 assert((Opc == ISD::SPLAT_VECTOR || Opc == RISCVISD::VMV_V_X_VL) &&
18744 "Unexpected Opcode");
18745
18746 // The pasthru must be undef for tail agnostic.
18747 if (Opc == RISCVISD::VMV_V_X_VL && !OrigOperand.getOperand(0).isUndef())
18748 return;
18749
18750 // Get the scalar value.
18751 SDValue Op = Opc == ISD::SPLAT_VECTOR ? OrigOperand.getOperand(0)
18752 : OrigOperand.getOperand(1);
18753
18754 // See if we have enough sign bits or zero bits in the scalar to use a
18755 // widening opcode by splatting to smaller element size.
18756 unsigned EltBits = VT.getScalarSizeInBits();
18757 unsigned ScalarBits = Op.getValueSizeInBits();
18758 // If we're not getting all bits from the element, we need special handling.
18759 if (ScalarBits < EltBits) {
18760 // This should only occur on RV32.
18761 assert(Opc == RISCVISD::VMV_V_X_VL && EltBits == 64 && ScalarBits == 32 &&
18762 !Subtarget.is64Bit() && "Unexpected splat");
18763 // vmv.v.x sign extends narrow inputs.
18764 SupportsSExt = true;
18765
18766 // If the input is positive, then sign extend is also zero extend.
18767 if (DAG.SignBitIsZero(Op))
18768 SupportsZExt = true;
18769
18770 EnforceOneUse = false;
18771 return;
18772 }
18773
18774 unsigned NarrowSize = EltBits / 2;
18775 // If the narrow type cannot be expressed with a legal VMV,
18776 // this is not a valid candidate.
18777 if (NarrowSize < 8)
18778 return;
18779
18780 if (DAG.ComputeMaxSignificantBits(Op) <= NarrowSize)
18781 SupportsSExt = true;
18782
18783 if (DAG.MaskedValueIsZero(Op,
18784 APInt::getBitsSetFrom(ScalarBits, NarrowSize)))
18785 SupportsZExt = true;
18786
18787 EnforceOneUse = false;
18788 }
18789
18790 bool isSupportedFPExtend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
18791 return (NarrowEltVT == MVT::f32 ||
18792 (NarrowEltVT == MVT::f16 && Subtarget.hasVInstructionsF16()));
18793 }
18794
18795 bool isSupportedBF16Extend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
18796 return NarrowEltVT == MVT::bf16 &&
18797 (Subtarget.hasStdExtZvfbfwma() || Subtarget.hasVInstructionsBF16());
18798 }
18799
18800 /// Helper method to set the various fields of this struct based on the
18801 /// type of \p Root.
18802 void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
18803 const RISCVSubtarget &Subtarget) {
18804 SupportsZExt = false;
18805 SupportsSExt = false;
18806 SupportsFPExt = false;
18807 SupportsBF16Ext = false;
18808 EnforceOneUse = true;
18809 unsigned Opc = OrigOperand.getOpcode();
18810 // For the nodes we handle below, we end up using their inputs directly: see
18811 // getSource(). However since they either don't have a passthru or we check
18812 // that their passthru is undef, we can safely ignore their mask and VL.
18813 switch (Opc) {
18814 case ISD::ZERO_EXTEND:
18815 case ISD::SIGN_EXTEND: {
18816 MVT VT = OrigOperand.getSimpleValueType();
18817 if (!VT.isVector())
18818 break;
18819
18820 SDValue NarrowElt = OrigOperand.getOperand(0);
18821 MVT NarrowVT = NarrowElt.getSimpleValueType();
18822 // i1 types are legal but we can't select V{S,Z}EXT_VLs with them.
18823 if (NarrowVT.getVectorElementType() == MVT::i1)
18824 break;
18825
18826 SupportsZExt = Opc == ISD::ZERO_EXTEND;
18827 SupportsSExt = Opc == ISD::SIGN_EXTEND;
18828 break;
18829 }
18830 case RISCVISD::VZEXT_VL:
18831 SupportsZExt = true;
18832 break;
18833 case RISCVISD::VSEXT_VL:
18834 SupportsSExt = true;
18835 break;
18836 case RISCVISD::FP_EXTEND_VL: {
18837 MVT NarrowEltVT =
18838 OrigOperand.getOperand(0).getSimpleValueType().getVectorElementType();
18839 if (isSupportedFPExtend(NarrowEltVT, Subtarget))
18840 SupportsFPExt = true;
18841 if (isSupportedBF16Extend(NarrowEltVT, Subtarget))
18842 SupportsBF16Ext = true;
18843
18844 break;
18845 }
18846 case ISD::SPLAT_VECTOR:
18847 case RISCVISD::VMV_V_X_VL:
18848 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
18849 break;
18850 case RISCVISD::VFMV_V_F_VL: {
18851 MVT VT = OrigOperand.getSimpleValueType();
18852
18853 if (!OrigOperand.getOperand(0).isUndef())
18854 break;
18855
18856 SDValue Op = OrigOperand.getOperand(1);
18857 if (Op.getOpcode() != ISD::FP_EXTEND)
18858 break;
18859
18860 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
18861 unsigned ScalarBits = Op.getOperand(0).getValueSizeInBits();
18862 if (NarrowSize != ScalarBits)
18863 break;
18864
18865 if (isSupportedFPExtend(Op.getOperand(0).getSimpleValueType(), Subtarget))
18866 SupportsFPExt = true;
18867 if (isSupportedBF16Extend(Op.getOperand(0).getSimpleValueType(),
18868 Subtarget))
18869 SupportsBF16Ext = true;
18870 break;
18871 }
18872 default:
18873 break;
18874 }
18875 }
18876
18877 /// Check if \p Root supports any extension folding combines.
18878 static bool isSupportedRoot(const SDNode *Root,
18879 const RISCVSubtarget &Subtarget) {
18880 switch (Root->getOpcode()) {
18881 case ISD::ADD:
18882 case ISD::SUB:
18883 case ISD::MUL: {
18884 return Root->getValueType(0).isScalableVector();
18885 }
18886 case ISD::OR: {
18887 return Root->getValueType(0).isScalableVector() &&
18888 Root->getFlags().hasDisjoint();
18889 }
18890 // Vector Widening Integer Add/Sub/Mul Instructions
18891 case RISCVISD::ADD_VL:
18892 case RISCVISD::MUL_VL:
18893 case RISCVISD::VWADD_W_VL:
18894 case RISCVISD::VWADDU_W_VL:
18895 case RISCVISD::SUB_VL:
18896 case RISCVISD::VWSUB_W_VL:
18897 case RISCVISD::VWSUBU_W_VL:
18898 // Vector Widening Floating-Point Add/Sub/Mul Instructions
18899 case RISCVISD::FADD_VL:
18900 case RISCVISD::FSUB_VL:
18901 case RISCVISD::FMUL_VL:
18902 case RISCVISD::VFWADD_W_VL:
18903 case RISCVISD::VFWSUB_W_VL:
18904 return true;
18905 case RISCVISD::OR_VL:
18906 return Root->getFlags().hasDisjoint();
18907 case ISD::SHL:
18908 return Root->getValueType(0).isScalableVector() &&
18909 Subtarget.hasStdExtZvbb();
18910 case RISCVISD::SHL_VL:
18911 return Subtarget.hasStdExtZvbb();
18912 case RISCVISD::VFMADD_VL:
18913 case RISCVISD::VFNMSUB_VL:
18914 case RISCVISD::VFNMADD_VL:
18915 case RISCVISD::VFMSUB_VL:
18916 return true;
18917 default:
18918 return false;
18919 }
18920 }
18921
18922 /// Build a NodeExtensionHelper for \p Root.getOperand(\p OperandIdx).
18923 NodeExtensionHelper(SDNode *Root, unsigned OperandIdx, SelectionDAG &DAG,
18924 const RISCVSubtarget &Subtarget) {
18925 assert(isSupportedRoot(Root, Subtarget) &&
18926 "Trying to build an helper with an "
18927 "unsupported root");
18928 assert(OperandIdx < 2 && "Requesting something else than LHS or RHS");
18929 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
18930 OrigOperand = Root->getOperand(OperandIdx);
18931
18932 unsigned Opc = Root->getOpcode();
18933 switch (Opc) {
18934 // We consider
18935 // VW<ADD|SUB>_W(LHS, RHS) -> <ADD|SUB>(LHS, SEXT(RHS))
18936 // VW<ADD|SUB>U_W(LHS, RHS) -> <ADD|SUB>(LHS, ZEXT(RHS))
18937 // VFW<ADD|SUB>_W(LHS, RHS) -> F<ADD|SUB>(LHS, FPEXT(RHS))
18938 case RISCVISD::VWADD_W_VL:
18939 case RISCVISD::VWADDU_W_VL:
18940 case RISCVISD::VWSUB_W_VL:
18941 case RISCVISD::VWSUBU_W_VL:
18942 case RISCVISD::VFWADD_W_VL:
18943 case RISCVISD::VFWSUB_W_VL:
18944 // Operand 1 can't be changed.
18945 if (OperandIdx == 1)
18946 break;
18947 [[fallthrough]];
18948 default:
18949 fillUpExtensionSupport(Root, DAG, Subtarget);
18950 break;
18951 }
18952 }
18953
18954 /// Helper function to get the Mask and VL from \p Root.
18955 static std::pair<SDValue, SDValue>
18956 getMaskAndVL(const SDNode *Root, SelectionDAG &DAG,
18957 const RISCVSubtarget &Subtarget) {
18958 assert(isSupportedRoot(Root, Subtarget) && "Unexpected root");
18959 switch (Root->getOpcode()) {
18960 case ISD::ADD:
18961 case ISD::SUB:
18962 case ISD::MUL:
18963 case ISD::OR:
18964 case ISD::SHL: {
18965 SDLoc DL(Root);
18966 MVT VT = Root->getSimpleValueType(0);
18967 return getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
18968 }
18969 default:
18970 return std::make_pair(Root->getOperand(3), Root->getOperand(4));
18971 }
18972 }
18973
18974 /// Helper function to check if \p N is commutative with respect to the
18975 /// foldings that are supported by this class.
18976 static bool isCommutative(const SDNode *N) {
18977 switch (N->getOpcode()) {
18978 case ISD::ADD:
18979 case ISD::MUL:
18980 case ISD::OR:
18981 case RISCVISD::ADD_VL:
18982 case RISCVISD::MUL_VL:
18983 case RISCVISD::OR_VL:
18984 case RISCVISD::FADD_VL:
18985 case RISCVISD::FMUL_VL:
18986 case RISCVISD::VFMADD_VL:
18987 case RISCVISD::VFNMSUB_VL:
18988 case RISCVISD::VFNMADD_VL:
18989 case RISCVISD::VFMSUB_VL:
18990 return true;
18991 case RISCVISD::VWADD_W_VL:
18992 case RISCVISD::VWADDU_W_VL:
18993 case ISD::SUB:
18994 case RISCVISD::SUB_VL:
18995 case RISCVISD::VWSUB_W_VL:
18996 case RISCVISD::VWSUBU_W_VL:
18997 case RISCVISD::VFWADD_W_VL:
18998 case RISCVISD::FSUB_VL:
18999 case RISCVISD::VFWSUB_W_VL:
19000 case ISD::SHL:
19001 case RISCVISD::SHL_VL:
19002 return false;
19003 default:
19004 llvm_unreachable("Unexpected opcode");
19005 }
19006 }
19007
19008 /// Get a list of combine to try for folding extensions in \p Root.
19009 /// Note that each returned CombineToTry function doesn't actually modify
19010 /// anything. Instead they produce an optional CombineResult that if not None,
19011 /// need to be materialized for the combine to be applied.
19012 /// \see CombineResult::materialize.
19013 /// If the related CombineToTry function returns std::nullopt, that means the
19014 /// combine didn't match.
19015 static SmallVector<CombineToTry>
19016 getSupportedFoldings(const SDNode *Root, const RISCVSubtarget &Subtarget);
19017};
19018
19019/// Helper structure that holds all the necessary information to materialize a
19020/// combine that does some extension folding.
19021struct CombineResult {
19022 /// Opcode to be generated when materializing the combine.
19023 unsigned TargetOpcode;
19024 // No value means no extension is needed.
19025 std::optional<ExtKind> LHSExt;
19026 std::optional<ExtKind> RHSExt;
19027 /// Root of the combine.
19028 SDNode *Root;
19029 /// LHS of the TargetOpcode.
19030 NodeExtensionHelper LHS;
19031 /// RHS of the TargetOpcode.
19032 NodeExtensionHelper RHS;
19033
19034 CombineResult(unsigned TargetOpcode, SDNode *Root,
19035 const NodeExtensionHelper &LHS, std::optional<ExtKind> LHSExt,
19036 const NodeExtensionHelper &RHS, std::optional<ExtKind> RHSExt)
19037 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
19038 LHS(LHS), RHS(RHS) {}
19039
19040 /// Return a value that uses TargetOpcode and that can be used to replace
19041 /// Root.
19042 /// The actual replacement is *not* done in that method.
19043 SDValue materialize(SelectionDAG &DAG,
19044 const RISCVSubtarget &Subtarget) const {
19045 SDValue Mask, VL, Passthru;
19046 std::tie(Mask, VL) =
19047 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
19048 switch (Root->getOpcode()) {
19049 default:
19050 Passthru = Root->getOperand(2);
19051 break;
19052 case ISD::ADD:
19053 case ISD::SUB:
19054 case ISD::MUL:
19055 case ISD::OR:
19056 case ISD::SHL:
19057 Passthru = DAG.getUNDEF(Root->getValueType(0));
19058 break;
19059 }
19060 return DAG.getNode(TargetOpcode, SDLoc(Root), Root->getValueType(0),
19061 LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, LHSExt),
19062 RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, RHSExt),
19063 Passthru, Mask, VL);
19064 }
19065};
19066
19067/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
19068/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
19069/// are zext) and LHS and RHS can be folded into Root.
19070/// AllowExtMask define which form `ext` can take in this pattern.
19071///
19072/// \note If the pattern can match with both zext and sext, the returned
19073/// CombineResult will feature the zext result.
19074///
19075/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19076/// can be used to apply the pattern.
19077static std::optional<CombineResult>
19078canFoldToVWWithSameExtensionImpl(SDNode *Root, const NodeExtensionHelper &LHS,
19079 const NodeExtensionHelper &RHS,
19080 uint8_t AllowExtMask, SelectionDAG &DAG,
19081 const RISCVSubtarget &Subtarget) {
19082 if ((AllowExtMask & ExtKind::ZExt) && LHS.SupportsZExt && RHS.SupportsZExt)
19083 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->getOpcode()),
19084 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
19085 /*RHSExt=*/{ExtKind::ZExt});
19086 if ((AllowExtMask & ExtKind::SExt) && LHS.SupportsSExt && RHS.SupportsSExt)
19087 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->getOpcode()),
19088 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19089 /*RHSExt=*/{ExtKind::SExt});
19090 if ((AllowExtMask & ExtKind::FPExt) && LHS.SupportsFPExt && RHS.SupportsFPExt)
19091 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
19092 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
19093 /*RHSExt=*/{ExtKind::FPExt});
19094 if ((AllowExtMask & ExtKind::BF16Ext) && LHS.SupportsBF16Ext &&
19095 RHS.SupportsBF16Ext)
19096 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
19097 Root, LHS, /*LHSExt=*/{ExtKind::BF16Ext}, RHS,
19098 /*RHSExt=*/{ExtKind::BF16Ext});
19099 return std::nullopt;
19100}
19101
19102/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
19103/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
19104/// are zext) and LHS and RHS can be folded into Root.
19105///
19106/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19107/// can be used to apply the pattern.
19108static std::optional<CombineResult>
19109canFoldToVWWithSameExtension(SDNode *Root, const NodeExtensionHelper &LHS,
19110 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19111 const RISCVSubtarget &Subtarget) {
19112 return canFoldToVWWithSameExtensionImpl(
19113 Root, LHS, RHS, ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
19114 Subtarget);
19115}
19116
19117/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS))
19118///
19119/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19120/// can be used to apply the pattern.
19121static std::optional<CombineResult>
19122canFoldToVWWithSameExtZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19123 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19124 const RISCVSubtarget &Subtarget) {
19125 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::ZExt, DAG,
19126 Subtarget);
19127}
19128
19129/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS))
19130///
19131/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19132/// can be used to apply the pattern.
19133static std::optional<CombineResult>
19134canFoldToVWWithSameExtBF16(SDNode *Root, const NodeExtensionHelper &LHS,
19135 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19136 const RISCVSubtarget &Subtarget) {
19137 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, ExtKind::BF16Ext, DAG,
19138 Subtarget);
19139}
19140
19141/// Check if \p Root follows a pattern Root(LHS, ext(RHS))
19142///
19143/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19144/// can be used to apply the pattern.
19145static std::optional<CombineResult>
19146canFoldToVW_W(SDNode *Root, const NodeExtensionHelper &LHS,
19147 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19148 const RISCVSubtarget &Subtarget) {
19149 if (RHS.SupportsFPExt)
19150 return CombineResult(
19151 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::FPExt),
19152 Root, LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::FPExt});
19153
19154 // FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
19155 // sext/zext?
19156 // Control this behavior behind an option (AllowSplatInVW_W) for testing
19157 // purposes.
19158 if (RHS.SupportsZExt && (!RHS.isSplat() || AllowSplatInVW_W))
19159 return CombineResult(
19160 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::ZExt), Root,
19161 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::ZExt});
19162 if (RHS.SupportsSExt && (!RHS.isSplat() || AllowSplatInVW_W))
19163 return CombineResult(
19164 NodeExtensionHelper::getWOpcode(Root->getOpcode(), ExtKind::SExt), Root,
19165 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::SExt});
19166 return std::nullopt;
19167}
19168
19169/// Check if \p Root follows a pattern Root(sext(LHS), RHS)
19170///
19171/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19172/// can be used to apply the pattern.
19173static std::optional<CombineResult>
19174canFoldToVWWithSEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19175 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19176 const RISCVSubtarget &Subtarget) {
19177 if (LHS.SupportsSExt)
19178 return CombineResult(NodeExtensionHelper::getSExtOpcode(Root->getOpcode()),
19179 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19180 /*RHSExt=*/std::nullopt);
19181 return std::nullopt;
19182}
19183
19184/// Check if \p Root follows a pattern Root(zext(LHS), RHS)
19185///
19186/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19187/// can be used to apply the pattern.
19188static std::optional<CombineResult>
19189canFoldToVWWithZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19190 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19191 const RISCVSubtarget &Subtarget) {
19192 if (LHS.SupportsZExt)
19193 return CombineResult(NodeExtensionHelper::getZExtOpcode(Root->getOpcode()),
19194 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
19195 /*RHSExt=*/std::nullopt);
19196 return std::nullopt;
19197}
19198
19199/// Check if \p Root follows a pattern Root(fpext(LHS), RHS)
19200///
19201/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19202/// can be used to apply the pattern.
19203static std::optional<CombineResult>
19204canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19205 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19206 const RISCVSubtarget &Subtarget) {
19207 if (LHS.SupportsFPExt)
19208 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Root->getOpcode()),
19209 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
19210 /*RHSExt=*/std::nullopt);
19211 return std::nullopt;
19212}
19213
19214/// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS))
19215///
19216/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19217/// can be used to apply the pattern.
19218static std::optional<CombineResult>
19219canFoldToVW_SU(SDNode *Root, const NodeExtensionHelper &LHS,
19220 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19221 const RISCVSubtarget &Subtarget) {
19222
19223 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
19224 return std::nullopt;
19225 return CombineResult(NodeExtensionHelper::getSUOpcode(Root->getOpcode()),
19226 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19227 /*RHSExt=*/{ExtKind::ZExt});
19228}
19229
19230SmallVector<NodeExtensionHelper::CombineToTry>
19231NodeExtensionHelper::getSupportedFoldings(const SDNode *Root,
19232 const RISCVSubtarget &Subtarget) {
19233 SmallVector<CombineToTry> Strategies;
19234 switch (Root->getOpcode()) {
19235 case ISD::ADD:
19236 case ISD::SUB:
19237 case ISD::OR:
19238 case RISCVISD::ADD_VL:
19239 case RISCVISD::SUB_VL:
19240 case RISCVISD::OR_VL:
19241 case RISCVISD::FADD_VL:
19242 case RISCVISD::FSUB_VL:
19243 // add|sub|fadd|fsub-> vwadd(u)|vwsub(u)|vfwadd|vfwsub
19244 Strategies.push_back(canFoldToVWWithSameExtension);
19245 if (Subtarget.hasVInstructionsBF16())
19246 Strategies.push_back(canFoldToVWWithSameExtBF16);
19247 // add|sub|fadd|fsub -> vwadd(u)_w|vwsub(u)_w}|vfwadd_w|vfwsub_w
19248 Strategies.push_back(canFoldToVW_W);
19249 break;
19250 case RISCVISD::FMUL_VL:
19251 case RISCVISD::VFMADD_VL:
19252 case RISCVISD::VFMSUB_VL:
19253 case RISCVISD::VFNMADD_VL:
19254 case RISCVISD::VFNMSUB_VL:
19255 Strategies.push_back(canFoldToVWWithSameExtension);
19256 if (Subtarget.hasVInstructionsBF16() ||
19257 (Subtarget.hasStdExtZvfbfwma() &&
19258 Root->getOpcode() == RISCVISD::VFMADD_VL))
19259 Strategies.push_back(canFoldToVWWithSameExtBF16);
19260 break;
19261 case ISD::MUL:
19262 case RISCVISD::MUL_VL:
19263 // mul -> vwmul(u)
19264 Strategies.push_back(canFoldToVWWithSameExtension);
19265 // mul -> vwmulsu
19266 Strategies.push_back(canFoldToVW_SU);
19267 break;
19268 case ISD::SHL:
19269 case RISCVISD::SHL_VL:
19270 // shl -> vwsll
19271 Strategies.push_back(canFoldToVWWithSameExtZEXT);
19272 break;
19273 case RISCVISD::VWADD_W_VL:
19274 case RISCVISD::VWSUB_W_VL:
19275 // vwadd_w|vwsub_w -> vwadd|vwsub
19276 Strategies.push_back(canFoldToVWWithSEXT);
19277 break;
19278 case RISCVISD::VWADDU_W_VL:
19279 case RISCVISD::VWSUBU_W_VL:
19280 // vwaddu_w|vwsubu_w -> vwaddu|vwsubu
19281 Strategies.push_back(canFoldToVWWithZEXT);
19282 break;
19283 case RISCVISD::VFWADD_W_VL:
19284 case RISCVISD::VFWSUB_W_VL:
19285 // vfwadd_w|vfwsub_w -> vfwadd|vfwsub
19286 Strategies.push_back(canFoldToVWWithFPEXT);
19287 break;
19288 default:
19289 llvm_unreachable("Unexpected opcode");
19290 }
19291 return Strategies;
19292}
19293} // End anonymous namespace.
19294
19295static SDValue simplifyOp_VL(SDNode *N) {
19296 // TODO: Extend this to other binops using generic identity logic
19297 assert(N->getOpcode() == RISCVISD::ADD_VL);
19298 SDValue A = N->getOperand(0);
19299 SDValue B = N->getOperand(1);
19300 SDValue Passthru = N->getOperand(2);
19301 if (!Passthru.isUndef())
19302 // TODO:This could be a vmerge instead
19303 return SDValue();
19304 ;
19305 if (ISD::isConstantSplatVectorAllZeros(B.getNode()))
19306 return A;
19307 // Peek through fixed to scalable
19308 if (B.getOpcode() == ISD::INSERT_SUBVECTOR && B.getOperand(0).isUndef() &&
19309 ISD::isConstantSplatVectorAllZeros(B.getOperand(1).getNode()))
19310 return A;
19311 return SDValue();
19312}
19313
19314/// Combine a binary or FMA operation to its equivalent VW or VW_W form.
19315/// The supported combines are:
19316/// add | add_vl | or disjoint | or_vl disjoint -> vwadd(u) | vwadd(u)_w
19317/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
19318/// mul | mul_vl -> vwmul(u) | vwmul_su
19319/// shl | shl_vl -> vwsll
19320/// fadd_vl -> vfwadd | vfwadd_w
19321/// fsub_vl -> vfwsub | vfwsub_w
19322/// fmul_vl -> vfwmul
19323/// vwadd_w(u) -> vwadd(u)
19324/// vwsub_w(u) -> vwsub(u)
19325/// vfwadd_w -> vfwadd
19326/// vfwsub_w -> vfwsub
19327static SDValue combineOp_VLToVWOp_VL(SDNode *N,
19328 TargetLowering::DAGCombinerInfo &DCI,
19329 const RISCVSubtarget &Subtarget) {
19330 SelectionDAG &DAG = DCI.DAG;
19331 if (DCI.isBeforeLegalize())
19332 return SDValue();
19333
19334 if (!NodeExtensionHelper::isSupportedRoot(N, Subtarget))
19335 return SDValue();
19336
19337 SmallVector<SDNode *> Worklist;
19338 SmallPtrSet<SDNode *, 8> Inserted;
19339 SmallPtrSet<SDNode *, 8> ExtensionsToRemove;
19340 Worklist.push_back(N);
19341 Inserted.insert(N);
19342 SmallVector<CombineResult> CombinesToApply;
19343
19344 while (!Worklist.empty()) {
19345 SDNode *Root = Worklist.pop_back_val();
19346
19347 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
19348 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
19349 auto AppendUsersIfNeeded =
19350 [&Worklist, &Subtarget, &Inserted,
19351 &ExtensionsToRemove](const NodeExtensionHelper &Op) {
19352 if (Op.needToPromoteOtherUsers()) {
19353 // Remember that we're supposed to remove this extension.
19354 ExtensionsToRemove.insert(Op.OrigOperand.getNode());
19355 for (SDUse &Use : Op.OrigOperand->uses()) {
19356 SDNode *TheUser = Use.getUser();
19357 if (!NodeExtensionHelper::isSupportedRoot(TheUser, Subtarget))
19358 return false;
19359 // We only support the first 2 operands of FMA.
19360 if (Use.getOperandNo() >= 2)
19361 return false;
19362 if (Inserted.insert(TheUser).second)
19363 Worklist.push_back(TheUser);
19364 }
19365 }
19366 return true;
19367 };
19368
19369 // Control the compile time by limiting the number of node we look at in
19370 // total.
19371 if (Inserted.size() > ExtensionMaxWebSize)
19372 return SDValue();
19373
19374 SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
19375 NodeExtensionHelper::getSupportedFoldings(Root, Subtarget);
19376
19377 assert(!FoldingStrategies.empty() && "Nothing to be folded");
19378 bool Matched = false;
19379 for (int Attempt = 0;
19380 (Attempt != 1 + NodeExtensionHelper::isCommutative(Root)) && !Matched;
19381 ++Attempt) {
19382
19383 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
19384 FoldingStrategies) {
19385 std::optional<CombineResult> Res =
19386 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
19387 if (Res) {
19388 // If this strategy wouldn't remove an extension we're supposed to
19389 // remove, reject it.
19390 if (!Res->LHSExt.has_value() &&
19391 ExtensionsToRemove.contains(LHS.OrigOperand.getNode()))
19392 continue;
19393 if (!Res->RHSExt.has_value() &&
19394 ExtensionsToRemove.contains(RHS.OrigOperand.getNode()))
19395 continue;
19396
19397 Matched = true;
19398 CombinesToApply.push_back(*Res);
19399 // All the inputs that are extended need to be folded, otherwise
19400 // we would be leaving the old input (since it is may still be used),
19401 // and the new one.
19402 if (Res->LHSExt.has_value())
19403 if (!AppendUsersIfNeeded(LHS))
19404 return SDValue();
19405 if (Res->RHSExt.has_value())
19406 if (!AppendUsersIfNeeded(RHS))
19407 return SDValue();
19408 break;
19409 }
19410 }
19411 std::swap(LHS, RHS);
19412 }
19413 // Right now we do an all or nothing approach.
19414 if (!Matched)
19415 return SDValue();
19416 }
19417 // Store the value for the replacement of the input node separately.
19418 SDValue InputRootReplacement;
19419 // We do the RAUW after we materialize all the combines, because some replaced
19420 // nodes may be feeding some of the yet-to-be-replaced nodes. Put differently,
19421 // some of these nodes may appear in the NodeExtensionHelpers of some of the
19422 // yet-to-be-visited CombinesToApply roots.
19423 SmallVector<std::pair<SDValue, SDValue>> ValuesToReplace;
19424 ValuesToReplace.reserve(CombinesToApply.size());
19425 for (CombineResult Res : CombinesToApply) {
19426 SDValue NewValue = Res.materialize(DAG, Subtarget);
19427 if (!InputRootReplacement) {
19428 assert(Res.Root == N &&
19429 "First element is expected to be the current node");
19430 InputRootReplacement = NewValue;
19431 } else {
19432 ValuesToReplace.emplace_back(SDValue(Res.Root, 0), NewValue);
19433 }
19434 }
19435 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
19436 DCI.CombineTo(OldNewValues.first.getNode(), OldNewValues.second);
19437 }
19438 return InputRootReplacement;
19439}
19440
19441// Fold (vwadd(u).wv y, (vmerge cond, x, 0)) -> vwadd(u).wv y, x, y, cond
19442// (vwsub(u).wv y, (vmerge cond, x, 0)) -> vwsub(u).wv y, x, y, cond
19443// y will be the Passthru and cond will be the Mask.
19444static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG) {
19445 unsigned Opc = N->getOpcode();
19446 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19447 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19448
19449 SDValue Y = N->getOperand(0);
19450 SDValue MergeOp = N->getOperand(1);
19451 unsigned MergeOpc = MergeOp.getOpcode();
19452
19453 if (MergeOpc != RISCVISD::VMERGE_VL && MergeOpc != ISD::VSELECT)
19454 return SDValue();
19455
19456 SDValue X = MergeOp->getOperand(1);
19457
19458 if (!MergeOp.hasOneUse())
19459 return SDValue();
19460
19461 // Passthru should be undef
19462 SDValue Passthru = N->getOperand(2);
19463 if (!Passthru.isUndef())
19464 return SDValue();
19465
19466 // Mask should be all ones
19467 SDValue Mask = N->getOperand(3);
19468 if (Mask.getOpcode() != RISCVISD::VMSET_VL)
19469 return SDValue();
19470
19471 // False value of MergeOp should be all zeros
19472 SDValue Z = MergeOp->getOperand(2);
19473
19474 if (Z.getOpcode() == ISD::INSERT_SUBVECTOR &&
19475 (isNullOrNullSplat(Z.getOperand(0)) || Z.getOperand(0).isUndef()))
19476 Z = Z.getOperand(1);
19477
19478 if (!ISD::isConstantSplatVectorAllZeros(Z.getNode()))
19479 return SDValue();
19480
19481 return DAG.getNode(Opc, SDLoc(N), N->getValueType(0),
19482 {Y, X, Y, MergeOp->getOperand(0), N->getOperand(4)},
19483 N->getFlags());
19484}
19485
19486// vwaddu C (vabd A B) -> vwabda(A B C)
19487// vwaddu C (vabdu A B) -> vwabdau(A B C)
19488static SDValue performVWABDACombine(SDNode *N, SelectionDAG &DAG,
19489 const RISCVSubtarget &Subtarget) {
19490 if (!Subtarget.hasStdExtZvabd())
19491 return SDValue();
19492
19493 MVT VT = N->getSimpleValueType(0);
19494 if (VT.getVectorElementType() != MVT::i8 &&
19495 VT.getVectorElementType() != MVT::i16)
19496 return SDValue();
19497
19498 SDValue Op0 = N->getOperand(0);
19499 SDValue Op1 = N->getOperand(1);
19500 SDValue Passthru = N->getOperand(2);
19501 if (!Passthru->isUndef())
19502 return SDValue();
19503
19504 SDValue Mask = N->getOperand(3);
19505 SDValue VL = N->getOperand(4);
19506 auto IsABD = [](SDValue Op) {
19507 if (Op->getOpcode() != RISCVISD::ABDS_VL &&
19508 Op->getOpcode() != RISCVISD::ABDU_VL)
19509 return SDValue();
19510 return Op;
19511 };
19512
19513 SDValue Diff = IsABD(Op0);
19514 Diff = Diff ? Diff : IsABD(Op1);
19515 if (!Diff)
19516 return SDValue();
19517 SDValue Acc = Diff == Op0 ? Op1 : Op0;
19518
19519 SDLoc DL(N);
19520 Acc = DAG.getNode(RISCVISD::VZEXT_VL, DL, VT, Acc, Mask, VL);
19521 SDValue Result = DAG.getNode(
19522 Diff.getOpcode() == RISCVISD::ABDS_VL ? RISCVISD::VWABDA_VL
19523 : RISCVISD::VWABDAU_VL,
19524 DL, VT, Diff.getOperand(0), Diff.getOperand(1), Acc, Mask, VL);
19525 return Result;
19526}
19527
19528// vwaddu_wv C (vabd A B) -> vwabda(A B C)
19529// vwaddu_wv C (zext (vabd A B)) -> vwabda(A (sext B) (sext C))
19530// vwaddu_wv C (vabdu A B) -> vwabdau(A B C)
19531// vwaddu_wv C (zext (vabdu A B)) -> vwabdau(A (zext B) (zext C))
19532static SDValue performVWABDACombineWV(SDNode *N, SelectionDAG &DAG,
19533 const RISCVSubtarget &Subtarget) {
19534 if (!Subtarget.hasStdExtZvabd())
19535 return SDValue();
19536
19537 MVT VT = N->getSimpleValueType(0);
19538 // The result is widened, so we can accept i16/i32 here.
19539 if (VT.getVectorElementType() != MVT::i16 &&
19540 VT.getVectorElementType() != MVT::i32)
19541 return SDValue();
19542
19543 SDValue Op0 = N->getOperand(0);
19544 SDValue Op1 = N->getOperand(1);
19545 SDValue Passthru = N->getOperand(2);
19546 if (!Passthru->isUndef())
19547 return SDValue();
19548
19549 SDValue Mask = N->getOperand(3);
19550 SDValue VL = N->getOperand(4);
19551 unsigned ExtOpc = 0;
19552 MVT ExtVT;
19553 auto GetDiff = [&](SDValue Op) {
19554 unsigned Opc = Op.getOpcode();
19555 if (Opc == RISCVISD::VZEXT_VL) {
19556 SDValue Src = Op->getOperand(0);
19557 unsigned SrcOpc = Src.getOpcode();
19558 switch (SrcOpc) {
19559 default:
19560 return SDValue();
19561 case ISD::ABDS:
19562 case RISCVISD::ABDS_VL:
19563 ExtOpc = RISCVISD::VSEXT_VL;
19564 break;
19565 case ISD::ABDU:
19566 case RISCVISD::ABDU_VL:
19567 ExtOpc = RISCVISD::VZEXT_VL;
19568 break;
19569 }
19570 ExtVT = Op->getSimpleValueType(0);
19571 return Src;
19572 }
19573
19574 if (Opc != ISD::ABDS && Opc != ISD::ABDU && Opc != RISCVISD::ABDS_VL &&
19575 Opc != RISCVISD::ABDU_VL)
19576 return SDValue();
19577 return Op;
19578 };
19579
19580 SDValue Diff = GetDiff(Op0);
19581 if (!Diff) {
19582 std::swap(Op0, Op1);
19583 Diff = GetDiff(Op0);
19584 if (!Diff)
19585 return SDValue();
19586 }
19587 SDValue Acc = Op1;
19588
19589 SDLoc DL(N);
19590 SDValue DiffA = Diff.getOperand(0);
19591 SDValue DiffB = Diff.getOperand(1);
19592 if (ExtOpc) {
19593 DiffA = DAG.getNode(ExtOpc, DL, ExtVT, DiffA, Mask, VL);
19594 DiffB = DAG.getNode(ExtOpc, DL, ExtVT, DiffB, Mask, VL);
19595 }
19596 SDValue Result = DAG.getNode(Diff.getOpcode() == ISD::ABDS ||
19597 Diff.getOpcode() == RISCVISD::ABDS_VL
19598 ? RISCVISD::VWABDA_VL
19599 : RISCVISD::VWABDAU_VL,
19600 DL, VT, DiffA, DiffB, Acc, Mask, VL);
19601 return Result;
19602}
19603
19604static SDValue performVWADDSUBW_VLCombine(SDNode *N,
19605 TargetLowering::DAGCombinerInfo &DCI,
19606 const RISCVSubtarget &Subtarget) {
19607 [[maybe_unused]] unsigned Opc = N->getOpcode();
19608 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19609 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19610
19611 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
19612 return V;
19613
19614 return combineVWADDSUBWSelect(N, DCI.DAG);
19615}
19616
19617// Helper function for performMemPairCombine.
19618// Try to combine the memory loads/stores LSNode1 and LSNode2
19619// into a single memory pair operation.
19620static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1,
19621 LSBaseSDNode *LSNode2, SDValue BasePtr,
19622 uint64_t Imm) {
19623 SmallPtrSet<const SDNode *, 32> Visited;
19624 SmallVector<const SDNode *, 8> Worklist = {LSNode1, LSNode2};
19625
19626 if (SDNode::hasPredecessorHelper(LSNode1, Visited, Worklist) ||
19627 SDNode::hasPredecessorHelper(LSNode2, Visited, Worklist))
19628 return SDValue();
19629
19630 MachineFunction &MF = DAG.getMachineFunction();
19631 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19632
19633 // The new operation has twice the width.
19634 MVT XLenVT = Subtarget.getXLenVT();
19635 EVT MemVT = LSNode1->getMemoryVT();
19636 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
19637 MachineMemOperand *MMO = LSNode1->getMemOperand();
19638 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
19639 MMO, MMO->getPointerInfo(), MemVT == MVT::i32 ? 8 : 16);
19640
19641 if (LSNode1->getOpcode() == ISD::LOAD) {
19642 auto Ext = cast<LoadSDNode>(LSNode1)->getExtensionType();
19643 unsigned Opcode;
19644 if (MemVT == MVT::i32)
19645 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
19646 else
19647 Opcode = RISCVISD::TH_LDD;
19648
19649 SDValue Res = DAG.getMemIntrinsicNode(
19650 Opcode, SDLoc(LSNode1), DAG.getVTList({XLenVT, XLenVT, MVT::Other}),
19651 {LSNode1->getChain(), BasePtr,
19652 DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
19653 NewMemVT, NewMMO);
19654
19655 SDValue Node1 =
19656 DAG.getMergeValues({Res.getValue(0), Res.getValue(2)}, SDLoc(LSNode1));
19657 SDValue Node2 =
19658 DAG.getMergeValues({Res.getValue(1), Res.getValue(2)}, SDLoc(LSNode2));
19659
19660 DAG.ReplaceAllUsesWith(LSNode2, Node2.getNode());
19661 return Node1;
19662 } else {
19663 unsigned Opcode = (MemVT == MVT::i32) ? RISCVISD::TH_SWD : RISCVISD::TH_SDD;
19664
19665 SDValue Res = DAG.getMemIntrinsicNode(
19666 Opcode, SDLoc(LSNode1), DAG.getVTList(MVT::Other),
19667 {LSNode1->getChain(), LSNode1->getOperand(1), LSNode2->getOperand(1),
19668 BasePtr, DAG.getConstant(Imm, SDLoc(LSNode1), XLenVT)},
19669 NewMemVT, NewMMO);
19670
19671 DAG.ReplaceAllUsesWith(LSNode2, Res.getNode());
19672 return Res;
19673 }
19674}
19675
19676// Try to combine two adjacent loads/stores to a single pair instruction from
19677// the XTHeadMemPair vendor extension.
19678static SDValue performMemPairCombine(SDNode *N,
19679 TargetLowering::DAGCombinerInfo &DCI) {
19680 SelectionDAG &DAG = DCI.DAG;
19681 MachineFunction &MF = DAG.getMachineFunction();
19682 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19683
19684 // Target does not support load/store pair.
19685 if (!Subtarget.hasVendorXTHeadMemPair())
19686 return SDValue();
19687
19688 LSBaseSDNode *LSNode1 = cast<LSBaseSDNode>(N);
19689 EVT MemVT = LSNode1->getMemoryVT();
19690 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
19691
19692 // No volatile, indexed or atomic loads/stores.
19693 if (!LSNode1->isSimple() || LSNode1->isIndexed())
19694 return SDValue();
19695
19696 // Function to get a base + constant representation from a memory value.
19697 auto ExtractBaseAndOffset = [](SDValue Ptr) -> std::pair<SDValue, uint64_t> {
19698 if (Ptr->getOpcode() == ISD::ADD)
19699 if (auto *C1 = dyn_cast<ConstantSDNode>(Ptr->getOperand(1)))
19700 return {Ptr->getOperand(0), C1->getZExtValue()};
19701 return {Ptr, 0};
19702 };
19703
19704 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->getOperand(OpNum));
19705
19706 SDValue Chain = N->getOperand(0);
19707 for (SDUse &Use : Chain->uses()) {
19708 if (Use.getUser() != N && Use.getResNo() == 0 &&
19709 Use.getUser()->getOpcode() == N->getOpcode()) {
19710 LSBaseSDNode *LSNode2 = cast<LSBaseSDNode>(Use.getUser());
19711
19712 // No volatile, indexed or atomic loads/stores.
19713 if (!LSNode2->isSimple() || LSNode2->isIndexed())
19714 continue;
19715
19716 // Check if LSNode1 and LSNode2 have the same type and extension.
19717 if (LSNode1->getOpcode() == ISD::LOAD)
19718 if (cast<LoadSDNode>(LSNode2)->getExtensionType() !=
19719 cast<LoadSDNode>(LSNode1)->getExtensionType())
19720 continue;
19721
19722 if (LSNode1->getMemoryVT() != LSNode2->getMemoryVT())
19723 continue;
19724
19725 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->getOperand(OpNum));
19726
19727 // Check if the base pointer is the same for both instruction.
19728 if (Base1 != Base2)
19729 continue;
19730
19731 // Check if the offsets match the XTHeadMemPair encoding constraints.
19732 bool Valid = false;
19733 if (MemVT == MVT::i32) {
19734 // Check for adjacent i32 values and a 2-bit index.
19735 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(Offset1))
19736 Valid = true;
19737 } else if (MemVT == MVT::i64) {
19738 // Check for adjacent i64 values and a 2-bit index.
19739 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(Offset1))
19740 Valid = true;
19741 }
19742
19743 if (!Valid)
19744 continue;
19745
19746 // Try to combine.
19747 if (SDValue Res =
19748 tryMemPairCombine(DAG, LSNode1, LSNode2, Base1, Offset1))
19749 return Res;
19750 }
19751 }
19752
19753 return SDValue();
19754}
19755
19756// Fold
19757// (fp_to_int (froundeven X)) -> fcvt X, rne
19758// (fp_to_int (ftrunc X)) -> fcvt X, rtz
19759// (fp_to_int (ffloor X)) -> fcvt X, rdn
19760// (fp_to_int (fceil X)) -> fcvt X, rup
19761// (fp_to_int (fround X)) -> fcvt X, rmm
19762// (fp_to_int (frint X)) -> fcvt X
19763static SDValue performFP_TO_INTCombine(SDNode *N,
19764 TargetLowering::DAGCombinerInfo &DCI,
19765 const RISCVSubtarget &Subtarget) {
19766 SelectionDAG &DAG = DCI.DAG;
19767 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19768 MVT XLenVT = Subtarget.getXLenVT();
19769
19770 SDValue Src = N->getOperand(0);
19771
19772 // Don't do this for strict-fp Src.
19773 if (Src->isStrictFPOpcode())
19774 return SDValue();
19775
19776 // Ensure the FP type is legal.
19777 if (!TLI.isTypeLegal(Src.getValueType()))
19778 return SDValue();
19779
19780 // Don't do this for f16 with Zfhmin and not Zfh.
19781 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
19782 return SDValue();
19783
19784 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src.getOpcode());
19785 // If the result is invalid, we didn't find a foldable instruction.
19786 if (FRM == RISCVFPRndMode::Invalid)
19787 return SDValue();
19788
19789 SDLoc DL(N);
19790 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
19791 EVT VT = N->getValueType(0);
19792
19793 if (VT.isVector() && TLI.isTypeLegal(VT)) {
19794 MVT SrcVT = Src.getSimpleValueType();
19795 MVT SrcContainerVT = SrcVT;
19796 MVT ContainerVT = VT.getSimpleVT();
19797 SDValue XVal = Src.getOperand(0);
19798
19799 // For widening and narrowing conversions we just combine it into a
19800 // VFCVT_..._VL node, as there are no specific VFWCVT/VFNCVT VL nodes. They
19801 // end up getting lowered to their appropriate pseudo instructions based on
19802 // their operand types
19803 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 ||
19804 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits())
19805 return SDValue();
19806
19807 // Make fixed-length vectors scalable first
19808 if (SrcVT.isFixedLengthVector()) {
19809 SrcContainerVT = getContainerForFixedLengthVector(SrcVT, Subtarget);
19810 XVal = convertToScalableVector(SrcContainerVT, XVal, DAG, Subtarget);
19811 ContainerVT = getContainerForFixedLengthVector(ContainerVT, Subtarget);
19812 }
19813
19814 auto [Mask, VL] =
19815 getDefaultVLOps(SrcVT, SrcContainerVT, DL, DAG, Subtarget);
19816
19817 SDValue FpToInt;
19818 if (FRM == RISCVFPRndMode::RTZ) {
19819 // Use the dedicated trunc static rounding mode if we're truncating so we
19820 // don't need to generate calls to fsrmi/fsrm
19821 unsigned Opc =
19822 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
19823 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask, VL);
19824 } else {
19825 unsigned Opc =
19826 IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
19827 FpToInt = DAG.getNode(Opc, DL, ContainerVT, XVal, Mask,
19828 DAG.getTargetConstant(FRM, DL, XLenVT), VL);
19829 }
19830
19831 // If converted from fixed-length to scalable, convert back
19832 if (VT.isFixedLengthVector())
19833 FpToInt = convertFromScalableVector(VT, FpToInt, DAG, Subtarget);
19834
19835 return FpToInt;
19836 }
19837
19838 // Only handle XLen or i32 types. Other types narrower than XLen will
19839 // eventually be legalized to XLenVT.
19840 if (VT != MVT::i32 && VT != XLenVT)
19841 return SDValue();
19842
19843 unsigned Opc;
19844 if (VT == XLenVT)
19845 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
19846 else
19847 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
19848
19849 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src.getOperand(0),
19850 DAG.getTargetConstant(FRM, DL, XLenVT));
19851 return DAG.getNode(ISD::TRUNCATE, DL, VT, FpToInt);
19852}
19853
19854// Fold
19855// (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
19856// (fp_to_int_sat (ftrunc X)) -> (select X == nan, 0, (fcvt X, rtz))
19857// (fp_to_int_sat (ffloor X)) -> (select X == nan, 0, (fcvt X, rdn))
19858// (fp_to_int_sat (fceil X)) -> (select X == nan, 0, (fcvt X, rup))
19859// (fp_to_int_sat (fround X)) -> (select X == nan, 0, (fcvt X, rmm))
19860// (fp_to_int_sat (frint X)) -> (select X == nan, 0, (fcvt X, dyn))
19861static SDValue performFP_TO_INT_SATCombine(SDNode *N,
19862 TargetLowering::DAGCombinerInfo &DCI,
19863 const RISCVSubtarget &Subtarget) {
19864 SelectionDAG &DAG = DCI.DAG;
19865 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19866 MVT XLenVT = Subtarget.getXLenVT();
19867
19868 // Only handle XLen types. Other types narrower than XLen will eventually be
19869 // legalized to XLenVT.
19870 EVT DstVT = N->getValueType(0);
19871 if (DstVT != XLenVT)
19872 return SDValue();
19873
19874 SDValue Src = N->getOperand(0);
19875
19876 // Don't do this for strict-fp Src.
19877 if (Src->isStrictFPOpcode())
19878 return SDValue();
19879
19880 // Ensure the FP type is also legal.
19881 if (!TLI.isTypeLegal(Src.getValueType()))
19882 return SDValue();
19883
19884 // Don't do this for f16 with Zfhmin and not Zfh.
19885 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
19886 return SDValue();
19887
19888 EVT SatVT = cast<VTSDNode>(N->getOperand(1))->getVT();
19889
19890 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Src.getOpcode());
19891 if (FRM == RISCVFPRndMode::Invalid)
19892 return SDValue();
19893
19894 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
19895
19896 unsigned Opc;
19897 if (SatVT == DstVT)
19898 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
19899 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
19900 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
19901 else
19902 return SDValue();
19903 // FIXME: Support other SatVTs by clamping before or after the conversion.
19904
19905 Src = Src.getOperand(0);
19906
19907 SDLoc DL(N);
19908 SDValue FpToInt = DAG.getNode(Opc, DL, XLenVT, Src,
19909 DAG.getTargetConstant(FRM, DL, XLenVT));
19910
19911 // fcvt.wu.* sign extends bit 31 on RV64. FP_TO_UINT_SAT expects to zero
19912 // extend.
19913 if (Opc == RISCVISD::FCVT_WU_RV64)
19914 FpToInt = DAG.getZeroExtendInReg(FpToInt, DL, MVT::i32);
19915
19916 // RISC-V FP-to-int conversions saturate to the destination register size, but
19917 // don't produce 0 for nan.
19918 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
19919 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
19920}
19921
19922// Combine (bitreverse (bswap X)) to the BREV8 GREVI encoding if the type is
19923// smaller than XLenVT.
19924static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG,
19925 const RISCVSubtarget &Subtarget) {
19926 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension");
19927
19928 SDValue Src = N->getOperand(0);
19929 if (Src.getOpcode() != ISD::BSWAP)
19930 return SDValue();
19931
19932 EVT VT = N->getValueType(0);
19933 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() ||
19934 !llvm::has_single_bit<uint32_t>(VT.getSizeInBits()))
19935 return SDValue();
19936
19937 SDLoc DL(N);
19938 return DAG.getNode(RISCVISD::BREV8, DL, VT, Src.getOperand(0));
19939}
19940
19941/// Matches a reverse shifted right EVL elements, or a vp.reverse.
19942// TODO: Remove vp.reverse
19943static auto m_ReverseEVL = [](auto X, auto EVL) {
19944 using namespace SDPatternMatch;
19945 return m_AnyOf(m_SpliceRight(m_OneUse(m_VectorReverse(X)), m_Poison(), EVL),
19946 m_Node(ISD::EXPERIMENTAL_VP_REVERSE, X, m_Value(), EVL));
19947};
19948
19949// TODO: A vlse.v is not necessarily faster than a vrgather.vv on all uarchs.
19950// Remove once a cost model driven transform is implemented in the loop
19951// vectorizer.
19952static SDValue performReverseEVLCombine(SDNode *N,
19953 TargetLowering::DAGCombinerInfo &DCI,
19954 const RISCVSubtarget &Subtarget) {
19955 SelectionDAG &DAG = DCI.DAG;
19956 // Fold:
19957 // vp.reverse(vp.load(ADDR, REVMASK, EVL), EVL)
19958 // -> vp.strided.load(ADDR, -1, MASK, EVL)
19959 //
19960 // splice.right(reverse(vp.load(ADDR, REVMASK, EVL)), poison, EVL)
19961 // -> vp.strided.load(ADDR, -1, MASK, EVL)
19962 //
19963 // vp.reverse(binop(vp.load(ADDR, REVMASK, EVL), splat), EVL)
19964 // -> binop(vp.strided.load(ADDR, -1, MASK, EVL), splat)
19965 using namespace SDPatternMatch;
19966 SDValue Op, EVL;
19967 if (!sd_match(N, m_ReverseEVL(m_Value(Op), m_Value(EVL))))
19968 return SDValue();
19969
19970 VPLoadSDNode *VPLoad = nullptr;
19971 // Find the single vp_load and check all other leaves are splats.
19972 SmallVector<SDValue> Worklist = {Op};
19973 while (!Worklist.empty()) {
19974 SDValue X = Worklist.pop_back_val();
19975 if (DAG.isSplatValue(X))
19976 continue;
19977 if (!X.hasOneUser())
19978 return SDValue();
19979 if (auto *VPL = dyn_cast<VPLoadSDNode>(X)) {
19980 if (VPLoad && VPLoad != VPL)
19981 return SDValue();
19982 VPLoad = VPL;
19983 } else if (DAG.getTargetLoweringInfo().isBinOp(X.getOpcode()) &&
19984 X->getNumValues() == 1) {
19985 append_range(Worklist, X->op_values());
19986 } else {
19987 return SDValue();
19988 }
19989 }
19990 if (!VPLoad)
19991 return SDValue();
19992
19993 EVT LoadVT = VPLoad->getValueType(0);
19994 // We do not have a strided_load version for masks, and the evl of vp.reverse
19995 // and vp.load should always be the same.
19996 if (!LoadVT.getVectorElementType().isByteSized() ||
19997 EVL != VPLoad->getVectorLength())
19998 return SDValue();
19999
20000 SDValue LoadMask = VPLoad->getMask();
20001 // If Mask is all ones, then load is unmasked and can be reversed.
20002 if (!isOneOrOneSplat(LoadMask)) {
20003 // If the mask is not all ones, we can reverse the load if the mask was also
20004 // reversed by a vp.reverse with the same EVL.
20005 SDValue OrigMask;
20006 if (!sd_match(LoadMask, m_ReverseEVL(m_Value(OrigMask), m_Specific(EVL))))
20007 return SDValue();
20008 LoadMask = OrigMask;
20009 }
20010
20011 // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
20012 SDLoc DL(N);
20013 MVT XLenVT = Subtarget.getXLenVT();
20014 SDValue NumElem = VPLoad->getVectorLength();
20015 uint64_t ElemWidthByte = VPLoad->getValueType(0).getScalarSizeInBits() / 8;
20016
20017 SDValue Temp1 = DAG.getNode(ISD::SUB, DL, XLenVT, NumElem,
20018 DAG.getConstant(1, DL, XLenVT));
20019 SDValue Temp2 = DAG.getNode(ISD::MUL, DL, XLenVT, Temp1,
20020 DAG.getConstant(ElemWidthByte, DL, XLenVT));
20021 SDValue Base = DAG.getNode(ISD::ADD, DL, XLenVT, VPLoad->getBasePtr(), Temp2);
20022 SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
20023
20024 MachineFunction &MF = DAG.getMachineFunction();
20025 MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
20026 MachineMemOperand *MMO = MF.getMachineMemOperand(
20027 PtrInfo, VPLoad->getMemOperand()->getFlags(),
20028 LocationSize::beforeOrAfterPointer(), VPLoad->getAlign());
20029
20030 SDValue Ret = DAG.getStridedLoadVP(
20031 LoadVT, DL, VPLoad->getChain(), Base, Stride, LoadMask,
20032 VPLoad->getVectorLength(), MMO, VPLoad->isExpandingLoad());
20033
20034 DCI.CombineTo(VPLoad, Ret.getValue(0), Ret.getValue(1));
20035
20036 // Remove the top level reverse.
20037 (void)sd_match(N, m_ReverseEVL(m_Value(Op), m_Value()));
20038 return Op;
20039}
20040
20041// Fold (i32 (bitcast (v4i8/v2i16 const_splat))) to a scalar i32 constant
20042// on RV64.
20043static SDValue performP_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
20044 const RISCVSubtarget &Subtarget) {
20045 SDValue N0 = N->getOperand(0);
20046 EVT VT = N->getValueType(0);
20047 EVT SrcVT = N0.getValueType();
20048 if (!Subtarget.is64Bit() || VT != MVT::i32 ||
20049 (SrcVT != MVT::v4i8 && SrcVT != MVT::v2i16))
20050 return SDValue();
20051
20052 APInt SplatVal;
20053 if (!ISD::isConstantSplatVector(N0.getNode(), SplatVal))
20054 return SDValue();
20055 return DAG.getConstant(APInt::getSplat(VT.getSizeInBits(), SplatVal),
20056 SDLoc(N), VT);
20057}
20058
20059static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
20060 const RISCVSubtarget &Subtarget) {
20061 // Fold:
20062 // vp.store(vp.reverse(VAL, EVL), ADDR, REVMASK, EVL)
20063 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
20064 //
20065 // vp.store(splice.right(reverse(VAL), poison, EVL), ADDR, REVMASK, EVL)
20066 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
20067 auto *VPStore = cast<VPStoreSDNode>(N);
20068 SDValue EVL = VPStore->getVectorLength();
20069
20070 using namespace SDPatternMatch;
20071 SDValue Val;
20072 if (!sd_match(VPStore->getValue(),
20073 m_OneUse(m_ReverseEVL(m_Value(Val), m_Specific(EVL)))))
20074 return SDValue();
20075
20076 EVT ReverseVT = VPStore->getValue()->getValueType(0);
20077
20078 // We do not have a strided_store version for masks.
20079 if (!ReverseVT.getVectorElementType().isByteSized())
20080 return SDValue();
20081
20082 SDValue StoreMask = VPStore->getMask();
20083 // If Mask is all ones, then load is unmasked and can be reversed.
20084 if (!isOneOrOneSplat(StoreMask)) {
20085 // If the mask is not all ones, we can reverse the store if the mask was
20086 // also reversed by a vp.reverse with the same EVL.
20087 SDValue OrigMask;
20088 if (!sd_match(StoreMask, m_ReverseEVL(m_Value(OrigMask), m_Specific(EVL))))
20089 return SDValue();
20090 StoreMask = OrigMask;
20091 }
20092
20093 // Base = StoreAddr + (NumElem - 1) * ElemWidthByte
20094 SDLoc DL(N);
20095 MVT XLenVT = Subtarget.getXLenVT();
20096 SDValue NumElem = VPStore->getVectorLength();
20097 uint64_t ElemWidthByte = ReverseVT.getScalarSizeInBits() / 8;
20098
20099 SDValue Temp1 = DAG.getNode(ISD::SUB, DL, XLenVT, NumElem,
20100 DAG.getConstant(1, DL, XLenVT));
20101 SDValue Temp2 = DAG.getNode(ISD::MUL, DL, XLenVT, Temp1,
20102 DAG.getConstant(ElemWidthByte, DL, XLenVT));
20103 SDValue Base =
20104 DAG.getNode(ISD::ADD, DL, XLenVT, VPStore->getBasePtr(), Temp2);
20105 SDValue Stride = DAG.getSignedConstant(-ElemWidthByte, DL, XLenVT);
20106
20107 MachineFunction &MF = DAG.getMachineFunction();
20108 MachinePointerInfo PtrInfo(VPStore->getAddressSpace());
20109 MachineMemOperand *MMO = MF.getMachineMemOperand(
20110 PtrInfo, VPStore->getMemOperand()->getFlags(),
20111 LocationSize::beforeOrAfterPointer(), VPStore->getAlign());
20112
20113 return DAG.getStridedStoreVP(
20114 VPStore->getChain(), DL, Val, Base, VPStore->getOffset(), Stride,
20115 StoreMask, VPStore->getVectorLength(), VPStore->getMemoryVT(), MMO,
20116 VPStore->getAddressingMode(), VPStore->isTruncatingStore(),
20117 VPStore->isCompressingStore());
20118}
20119
20120// Convert from one FMA opcode to another based on whether we are negating the
20121// multiply result and/or the accumulator.
20122// NOTE: Only supports RVV operations with VL.
20123static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc) {
20124 // Negating the multiply result changes ADD<->SUB and toggles 'N'.
20125 if (NegMul) {
20126 // clang-format off
20127 switch (Opcode) {
20128 default: llvm_unreachable("Unexpected opcode");
20129 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
20130 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
20131 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
20132 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
20133 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
20134 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
20135 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
20136 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
20137 }
20138 // clang-format on
20139 }
20140
20141 // Negating the accumulator changes ADD<->SUB.
20142 if (NegAcc) {
20143 // clang-format off
20144 switch (Opcode) {
20145 default: llvm_unreachable("Unexpected opcode");
20146 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
20147 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
20148 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
20149 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
20150 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
20151 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
20152 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
20153 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
20154 }
20155 // clang-format on
20156 }
20157
20158 return Opcode;
20159}
20160
20161static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) {
20162 // Fold FNEG_VL into FMA opcodes.
20163 // The first operand of strict-fp is chain.
20164 bool IsStrict =
20165 DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(N->getOpcode());
20166 unsigned Offset = IsStrict ? 1 : 0;
20167 SDValue A = N->getOperand(0 + Offset);
20168 SDValue B = N->getOperand(1 + Offset);
20169 SDValue C = N->getOperand(2 + Offset);
20170 SDValue Mask = N->getOperand(3 + Offset);
20171 SDValue VL = N->getOperand(4 + Offset);
20172
20173 auto invertIfNegative = [&Mask, &VL](SDValue &V) {
20174 if (V.getOpcode() == RISCVISD::FNEG_VL && V.getOperand(1) == Mask &&
20175 V.getOperand(2) == VL) {
20176 // Return the negated input.
20177 V = V.getOperand(0);
20178 return true;
20179 }
20180
20181 return false;
20182 };
20183
20184 bool NegA = invertIfNegative(A);
20185 bool NegB = invertIfNegative(B);
20186 bool NegC = invertIfNegative(C);
20187
20188 // If no operands are negated, we're done.
20189 if (!NegA && !NegB && !NegC)
20190 return SDValue();
20191
20192 unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC);
20193 if (IsStrict)
20194 return DAG.getNode(NewOpcode, SDLoc(N), N->getVTList(),
20195 {N->getOperand(0), A, B, C, Mask, VL});
20196 return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), A, B, C, Mask,
20197 VL);
20198}
20199
20200static SDValue performVFMADD_VLCombine(SDNode *N,
20201 TargetLowering::DAGCombinerInfo &DCI,
20202 const RISCVSubtarget &Subtarget) {
20203 SelectionDAG &DAG = DCI.DAG;
20204
20205 if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
20206 return V;
20207
20208 // FIXME: Ignore strict opcodes for now.
20209 if (DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(N->getOpcode()))
20210 return SDValue();
20211
20212 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
20213}
20214
20215static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
20216 const RISCVSubtarget &Subtarget) {
20217 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
20218
20219 EVT VT = N->getValueType(0);
20220
20221 if (VT != Subtarget.getXLenVT())
20222 return SDValue();
20223
20224 if (!isa<ConstantSDNode>(N->getOperand(1)))
20225 return SDValue();
20226 uint64_t ShAmt = N->getConstantOperandVal(1);
20227
20228 SDValue N0 = N->getOperand(0);
20229
20230 // Combine (sra (sext_inreg (shl X, C1), iX), C2) ->
20231 // (sra (shl X, C1+(XLen-iX)), C2+(XLen-iX)) so it gets selected as SLLI+SRAI.
20232 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse()) {
20233 unsigned ExtSize =
20234 cast<VTSDNode>(N0.getOperand(1))->getVT().getSizeInBits();
20235 if (ShAmt < ExtSize && N0.getOperand(0).getOpcode() == ISD::SHL &&
20236 N0.getOperand(0).hasOneUse() &&
20237 isa<ConstantSDNode>(N0.getOperand(0).getOperand(1))) {
20238 uint64_t LShAmt = N0.getOperand(0).getConstantOperandVal(1);
20239 if (LShAmt < ExtSize) {
20240 unsigned Size = VT.getSizeInBits();
20241 SDLoc ShlDL(N0.getOperand(0));
20242 SDValue Shl =
20243 DAG.getNode(ISD::SHL, ShlDL, VT, N0.getOperand(0).getOperand(0),
20244 DAG.getConstant(LShAmt + (Size - ExtSize), ShlDL, VT));
20245 SDLoc DL(N);
20246 return DAG.getNode(ISD::SRA, DL, VT, Shl,
20247 DAG.getConstant(ShAmt + (Size - ExtSize), DL, VT));
20248 }
20249 }
20250 }
20251
20252 if (ShAmt > 32 || VT != MVT::i64)
20253 return SDValue();
20254
20255 // Combine (sra (shl X, 32), 32 - C) -> (shl (sext_inreg X, i32), C)
20256 // FIXME: Should this be a generic combine? There's a similar combine on X86.
20257 //
20258 // Also try these folds where an add or sub is in the middle.
20259 // (sra (add (shl X, 32), C1), 32 - C) -> (shl (sext_inreg (add X, C1), C)
20260 // (sra (sub C1, (shl X, 32)), 32 - C) -> (shl (sext_inreg (sub C1, X), C)
20261 SDValue Shl;
20262 ConstantSDNode *AddC = nullptr;
20263
20264 // We might have an ADD or SUB between the SRA and SHL.
20265 bool IsAdd = N0.getOpcode() == ISD::ADD;
20266 if ((IsAdd || N0.getOpcode() == ISD::SUB)) {
20267 // Other operand needs to be a constant we can modify.
20268 AddC = dyn_cast<ConstantSDNode>(N0.getOperand(IsAdd ? 1 : 0));
20269 if (!AddC)
20270 return SDValue();
20271
20272 // AddC needs to have at least 32 trailing zeros.
20273 if (llvm::countr_zero(AddC->getZExtValue()) < 32)
20274 return SDValue();
20275
20276 // All users should be a shift by constant less than or equal to 32. This
20277 // ensures we'll do this optimization for each of them to produce an
20278 // add/sub+sext_inreg they can all share.
20279 for (SDNode *U : N0->users()) {
20280 if (U->getOpcode() != ISD::SRA ||
20281 !isa<ConstantSDNode>(U->getOperand(1)) ||
20282 U->getConstantOperandVal(1) > 32)
20283 return SDValue();
20284 }
20285
20286 Shl = N0.getOperand(IsAdd ? 0 : 1);
20287 } else {
20288 // Not an ADD or SUB.
20289 Shl = N0;
20290 }
20291
20292 // Look for a shift left by 32.
20293 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Shl.getOperand(1)) ||
20294 Shl.getConstantOperandVal(1) != 32)
20295 return SDValue();
20296
20297 // We if we didn't look through an add/sub, then the shl should have one use.
20298 // If we did look through an add/sub, the sext_inreg we create is free so
20299 // we're only creating 2 new instructions. It's enough to only remove the
20300 // original sra+add/sub.
20301 if (!AddC && !Shl.hasOneUse())
20302 return SDValue();
20303
20304 SDLoc DL(N);
20305 SDValue In = Shl.getOperand(0);
20306
20307 // If we looked through an ADD or SUB, we need to rebuild it with the shifted
20308 // constant.
20309 if (AddC) {
20310 SDValue ShiftedAddC =
20311 DAG.getConstant(AddC->getZExtValue() >> 32, DL, MVT::i64);
20312 if (IsAdd)
20313 In = DAG.getNode(ISD::ADD, DL, MVT::i64, In, ShiftedAddC);
20314 else
20315 In = DAG.getNode(ISD::SUB, DL, MVT::i64, ShiftedAddC, In);
20316 }
20317
20318 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, In,
20319 DAG.getValueType(MVT::i32));
20320 if (ShAmt == 32)
20321 return SExt;
20322
20323 return DAG.getNode(
20324 ISD::SHL, DL, MVT::i64, SExt,
20325 DAG.getConstant(32 - ShAmt, DL, MVT::i64));
20326}
20327
20328// Invert (and/or (set cc X, Y), (xor Z, 1)) to (or/and (set !cc X, Y)), Z) if
20329// the result is used as the condition of a br_cc or select_cc we can invert,
20330// inverting the setcc is free, and Z is 0/1. Caller will invert the
20331// br_cc/select_cc.
20332static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
20333 bool IsAnd = Cond.getOpcode() == ISD::AND;
20334 if (!IsAnd && Cond.getOpcode() != ISD::OR)
20335 return SDValue();
20336
20337 if (!Cond.hasOneUse())
20338 return SDValue();
20339
20340 SDValue Setcc = Cond.getOperand(0);
20341 SDValue Xor = Cond.getOperand(1);
20342 // Canonicalize setcc to LHS.
20343 if (Setcc.getOpcode() != ISD::SETCC)
20344 std::swap(Setcc, Xor);
20345 // LHS should be a setcc and RHS should be an xor.
20346 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() ||
20347 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
20348 return SDValue();
20349
20350 // If the condition is an And, SimplifyDemandedBits may have changed
20351 // (xor Z, 1) to (not Z).
20352 SDValue Xor1 = Xor.getOperand(1);
20353 if (!isOneConstant(Xor1) && !(IsAnd && isAllOnesConstant(Xor1)))
20354 return SDValue();
20355
20356 EVT VT = Cond.getValueType();
20357 SDValue Xor0 = Xor.getOperand(0);
20358
20359 // The LHS of the xor needs to be 0/1.
20360 APInt Mask = APInt::getBitsSetFrom(VT.getSizeInBits(), 1);
20361 if (!DAG.MaskedValueIsZero(Xor0, Mask))
20362 return SDValue();
20363
20364 // We can only invert integer setccs.
20365 EVT SetCCOpVT = Setcc.getOperand(0).getValueType();
20366 if (!SetCCOpVT.isScalarInteger())
20367 return SDValue();
20368
20369 ISD::CondCode CCVal = cast<CondCodeSDNode>(Setcc.getOperand(2))->get();
20370 if (ISD::isIntEqualitySetCC(CCVal)) {
20371 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
20372 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(0),
20373 Setcc.getOperand(1), CCVal);
20374 } else if (CCVal == ISD::SETLT && isNullConstant(Setcc.getOperand(0))) {
20375 // Invert (setlt 0, X) by converting to (setlt X, 1).
20376 Setcc = DAG.getSetCC(SDLoc(Setcc), VT, Setcc.getOperand(1),
20377 DAG.getConstant(1, SDLoc(Setcc), VT), CCVal);
20378 } else if (CCVal == ISD::SETLT && isOneConstant(Setcc.getOperand(1))) {
20379 // (setlt X, 1) by converting to (setlt 0, X).
20380 Setcc = DAG.getSetCC(SDLoc(Setcc), VT,
20381 DAG.getConstant(0, SDLoc(Setcc), VT),
20382 Setcc.getOperand(0), CCVal);
20383 } else
20384 return SDValue();
20385
20386 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
20387 return DAG.getNode(Opc, SDLoc(Cond), VT, Setcc, Xor.getOperand(0));
20388}
20389
20390// Perform common combines for BR_CC and SELECT_CC conditions.
20391static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
20392 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
20393 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
20394
20395 // As far as arithmetic right shift always saves the sign,
20396 // shift can be omitted.
20397 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
20398 // setge (sra X, N), 0 -> setge X, 0
20399 if (isNullConstant(RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
20400 LHS.getOpcode() == ISD::SRA) {
20401 LHS = LHS.getOperand(0);
20402 return true;
20403 }
20404
20405 if (!ISD::isIntEqualitySetCC(CCVal))
20406 return false;
20407
20408 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
20409 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
20410 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
20411 LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
20412 // If we're looking for eq 0 instead of ne 0, we need to invert the
20413 // condition.
20414 bool Invert = CCVal == ISD::SETEQ;
20415 CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
20416 if (Invert)
20417 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
20418
20419 RHS = LHS.getOperand(1);
20420 LHS = LHS.getOperand(0);
20421 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG, Subtarget);
20422
20423 CC = DAG.getCondCode(CCVal);
20424 return true;
20425 }
20426
20427 auto isFoldableXorEq = [&DAG](SDValue LHS, SDValue RHS) -> bool {
20428 if (LHS.getOpcode() != ISD::XOR || !isNullConstant(RHS))
20429 return false;
20430
20431 // If XOR cannot be an XORI, allow the fold.
20432 const auto *XorCnst = dyn_cast<ConstantSDNode>(LHS.getOperand(1));
20433 if (!XorCnst || !isInt<12>(XorCnst->getSExtValue()))
20434 return true;
20435
20436 // Fold (X(i1) ^ 1) == 0 -> X != 0
20437 SDValue VarOp = LHS.getOperand(0);
20438 const APInt Mask = APInt::getBitsSetFrom(VarOp.getValueSizeInBits(), 1);
20439 if (XorCnst->getSExtValue() == 1 && DAG.MaskedValueIsZero(VarOp, Mask))
20440 return true;
20441
20442 // If the Xor is only used by select or br_cc, allow the fold.
20443 return all_of(LHS->users(), [](const SDNode *UserNode) {
20444 const unsigned Opcode = UserNode->getOpcode();
20445 return Opcode == RISCVISD::SELECT_CC || Opcode == RISCVISD::BR_CC;
20446 });
20447 };
20448 // Fold ((xor X, Y), 0, eq/ne) -> (X, Y, eq/ne)
20449 if (isFoldableXorEq(LHS, RHS)) {
20450 RHS = LHS.getOperand(1);
20451 LHS = LHS.getOperand(0);
20452 return true;
20453 }
20454 // Fold ((sext (xor X, C)), 0, eq/ne) -> ((sext(X), C, eq/ne)
20455 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG) {
20456 const SDValue LHS0 = LHS.getOperand(0);
20457 if (isFoldableXorEq(LHS0, RHS) && isa<ConstantSDNode>(LHS0.getOperand(1))) {
20458 // SEXT(XOR(X, Y)) -> XOR(SEXT(X), SEXT(Y)))
20459 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, LHS.getValueType(),
20460 LHS0.getOperand(1), LHS.getOperand(1));
20461 LHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, LHS.getValueType(),
20462 LHS0.getOperand(0), LHS.getOperand(1));
20463 return true;
20464 }
20465 }
20466
20467 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, XLen-1-C), 0, ge/lt)
20468 if (isNullConstant(RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
20469 LHS.getOperand(1).getOpcode() == ISD::Constant) {
20470 SDValue LHS0 = LHS.getOperand(0);
20471 if (LHS0.getOpcode() == ISD::AND &&
20472 LHS0.getOperand(1).getOpcode() == ISD::Constant) {
20473 uint64_t Mask = LHS0.getConstantOperandVal(1);
20474 uint64_t ShAmt = LHS.getConstantOperandVal(1);
20475 if (isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
20476 // XAndesPerf supports branch on test bit.
20477 if (Subtarget.hasVendorXAndesPerf()) {
20478 LHS =
20479 DAG.getNode(ISD::AND, DL, LHS.getValueType(), LHS0.getOperand(0),
20480 DAG.getConstant(Mask, DL, LHS.getValueType()));
20481 return true;
20482 }
20483
20484 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
20485 CC = DAG.getCondCode(CCVal);
20486
20487 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
20488 LHS = LHS0.getOperand(0);
20489 if (ShAmt != 0)
20490 LHS =
20491 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0),
20492 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
20493 return true;
20494 }
20495 }
20496 }
20497
20498 // (X, 1, setne) -> // (X, 0, seteq) if we can prove X is 0/1.
20499 // This can occur when legalizing some floating point comparisons.
20500 APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
20501 if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
20502 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
20503 CC = DAG.getCondCode(CCVal);
20504 RHS = DAG.getConstant(0, DL, LHS.getValueType());
20505 return true;
20506 }
20507
20508 if (isNullConstant(RHS)) {
20509 if (SDValue NewCond = tryDemorganOfBooleanCondition(LHS, DAG)) {
20510 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
20511 CC = DAG.getCondCode(CCVal);
20512 LHS = NewCond;
20513 return true;
20514 }
20515 }
20516
20517 return false;
20518}
20519
20520// Fold
20521// (select C, (add Y, X), Y) -> (add Y, (select C, X, 0)).
20522// (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
20523// (select C, (or Y, X), Y) -> (or Y, (select C, X, 0)).
20524// (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
20525// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
20526// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
20527static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
20528 SDValue TrueVal, SDValue FalseVal,
20529 bool Swapped) {
20530 bool Commutative = true;
20531 unsigned Opc = TrueVal.getOpcode();
20532 switch (Opc) {
20533 default:
20534 return SDValue();
20535 case ISD::SHL:
20536 case ISD::SRA:
20537 case ISD::SRL:
20538 case ISD::SUB:
20539 case ISD::ROTL:
20540 case ISD::ROTR:
20541 Commutative = false;
20542 break;
20543 case ISD::ADD:
20544 case ISD::OR:
20545 case ISD::XOR:
20546 case ISD::UMIN:
20547 case ISD::UMAX:
20548 break;
20549 }
20550
20551 if (!TrueVal.hasOneUse())
20552 return SDValue();
20553
20554 unsigned OpToFold;
20555 if (FalseVal == TrueVal.getOperand(0))
20556 OpToFold = 0;
20557 else if (Commutative && FalseVal == TrueVal.getOperand(1))
20558 OpToFold = 1;
20559 else
20560 return SDValue();
20561
20562 EVT VT = N->getValueType(0);
20563 SDLoc DL(N);
20564 SDValue OtherOp = TrueVal.getOperand(1 - OpToFold);
20565 EVT OtherOpVT = OtherOp.getValueType();
20566 SDValue IdentityOperand =
20567 DAG.getIdentityElement(Opc, DL, OtherOpVT, N->getFlags());
20568 if (!Commutative)
20569 IdentityOperand = DAG.getConstant(0, DL, OtherOpVT);
20570 assert(IdentityOperand && "No identity operand!");
20571
20572 if (Swapped)
20573 std::swap(OtherOp, IdentityOperand);
20574 SDValue NewSel =
20575 DAG.getSelect(DL, OtherOpVT, N->getOperand(0), OtherOp, IdentityOperand);
20576 return DAG.getNode(TrueVal.getOpcode(), DL, VT, FalseVal, NewSel);
20577}
20578
20579// This tries to get rid of `select` and `icmp` that are being used to handle
20580// `Targets` that do not support `cttz(0)`/`ctlz(0)`.
20581static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG) {
20582 SDValue Cond = N->getOperand(0);
20583
20584 // This represents either CTTZ or CTLZ instruction.
20585 SDValue CountZeroes;
20586
20587 SDValue ValOnZero;
20588
20589 if (Cond.getOpcode() != ISD::SETCC)
20590 return SDValue();
20591
20592 if (!isNullConstant(Cond->getOperand(1)))
20593 return SDValue();
20594
20595 ISD::CondCode CCVal = cast<CondCodeSDNode>(Cond->getOperand(2))->get();
20596 if (CCVal == ISD::CondCode::SETEQ) {
20597 CountZeroes = N->getOperand(2);
20598 ValOnZero = N->getOperand(1);
20599 } else if (CCVal == ISD::CondCode::SETNE) {
20600 CountZeroes = N->getOperand(1);
20601 ValOnZero = N->getOperand(2);
20602 } else {
20603 return SDValue();
20604 }
20605
20606 if (CountZeroes.getOpcode() == ISD::TRUNCATE ||
20607 CountZeroes.getOpcode() == ISD::ZERO_EXTEND)
20608 CountZeroes = CountZeroes.getOperand(0);
20609
20610 if (CountZeroes.getOpcode() != ISD::CTTZ &&
20611 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_POISON &&
20612 CountZeroes.getOpcode() != ISD::CTLZ &&
20613 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_POISON)
20614 return SDValue();
20615
20616 if (!isNullConstant(ValOnZero))
20617 return SDValue();
20618
20619 SDValue CountZeroesArgument = CountZeroes->getOperand(0);
20620 if (Cond->getOperand(0) != CountZeroesArgument)
20621 return SDValue();
20622
20623 unsigned BitWidth = CountZeroes.getValueSizeInBits();
20624 if (!isPowerOf2_32(BitWidth))
20625 return SDValue();
20626
20627 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_POISON) {
20628 CountZeroes = DAG.getNode(ISD::CTTZ, SDLoc(CountZeroes),
20629 CountZeroes.getValueType(), CountZeroesArgument);
20630 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_POISON) {
20631 CountZeroes = DAG.getNode(ISD::CTLZ, SDLoc(CountZeroes),
20632 CountZeroes.getValueType(), CountZeroesArgument);
20633 }
20634
20635 SDValue BitWidthMinusOne =
20636 DAG.getConstant(BitWidth - 1, SDLoc(N), CountZeroes.getValueType());
20637
20638 auto AndNode = DAG.getNode(ISD::AND, SDLoc(N), CountZeroes.getValueType(),
20639 CountZeroes, BitWidthMinusOne);
20640 return DAG.getZExtOrTrunc(AndNode, SDLoc(N), N->getValueType(0));
20641}
20642
20643static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG,
20644 const RISCVSubtarget &Subtarget) {
20645 SDValue Cond = N->getOperand(0);
20646 SDValue True = N->getOperand(1);
20647 SDValue False = N->getOperand(2);
20648 SDLoc DL(N);
20649 EVT VT = N->getValueType(0);
20650 EVT CondVT = Cond.getValueType();
20651
20652 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
20653 return SDValue();
20654
20655 // Replace (setcc eq (and x, C)) with (setcc ne (and x, C))) to generate
20656 // BEXTI, where C is power of 2.
20657 if (Subtarget.hasBEXTILike() && VT.isScalarInteger() &&
20658 (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())) {
20659 SDValue LHS = Cond.getOperand(0);
20660 SDValue RHS = Cond.getOperand(1);
20661 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
20662 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND &&
20663 isa<ConstantSDNode>(LHS.getOperand(1)) && isNullConstant(RHS)) {
20664 const APInt &MaskVal = LHS.getConstantOperandAPInt(1);
20665 if (MaskVal.isPowerOf2() && !MaskVal.isSignedIntN(12))
20666 return DAG.getSelect(DL, VT,
20667 DAG.getSetCC(DL, CondVT, LHS, RHS, ISD::SETNE),
20668 False, True);
20669 }
20670 }
20671 return SDValue();
20672}
20673
20674static SDValue
20675canonicalizeVSelectTrueToOneUse(SDNode *N, SelectionDAG &DAG,
20676 const RISCVSubtarget &Subtarget) {
20677 SDValue CC = N->getOperand(0);
20678 SDValue TrueVal = N->getOperand(1);
20679 SDValue FalseVal = N->getOperand(2);
20680
20681 if (CC.getOpcode() != ISD::SETCC || !CC.hasOneUse() || TrueVal.hasOneUse() ||
20682 !FalseVal.hasOneUse())
20683 return SDValue();
20684
20685 // Only handles ISD::SETEQ and ISD::SETNE; no extra RVV introduced.
20686 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC.getOperand(2))->get();
20687 if (!isIntEqualitySetCC(CCVal))
20688 return SDValue();
20689
20690 if (DAG.isSplatValue(TrueVal) || DAG.isSplatValue(FalseVal) ||
20691 TrueVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
20692 FalseVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
20693 TrueVal.getOpcode() == RISCVISD::VMV_V_X_VL ||
20694 FalseVal.getOpcode() == RISCVISD::VMV_V_X_VL)
20695 return SDValue();
20696
20697 SDLoc DL(N);
20698 EVT CVT = CC.getValueType();
20699 SDValue InvertedCC = DAG.getSetCC(DL, CVT, CC.getOperand(0), CC.getOperand(1),
20700 ISD::getSetCCInverse(CCVal, CVT));
20701 return DAG.getNode(ISD::VSELECT, DL, N->getValueType(0), InvertedCC, FalseVal,
20702 TrueVal);
20703}
20704
20705static bool matchSelectAddSub(SDValue TrueVal, SDValue FalseVal, bool &SwapCC) {
20706 if (!TrueVal.hasOneUse() || !FalseVal.hasOneUse())
20707 return false;
20708
20709 SwapCC = false;
20710 if (TrueVal.getOpcode() == ISD::SUB && FalseVal.getOpcode() == ISD::ADD) {
20711 std::swap(TrueVal, FalseVal);
20712 SwapCC = true;
20713 }
20714
20715 if (TrueVal.getOpcode() != ISD::ADD || FalseVal.getOpcode() != ISD::SUB)
20716 return false;
20717
20718 SDValue A = FalseVal.getOperand(0);
20719 SDValue B = FalseVal.getOperand(1);
20720 // Add is commutative, so check both orders
20721 return ((TrueVal.getOperand(0) == A && TrueVal.getOperand(1) == B) ||
20722 (TrueVal.getOperand(1) == A && TrueVal.getOperand(0) == B));
20723}
20724
20725static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
20726 const RISCVSubtarget &Subtarget) {
20727 SDLoc DL(N);
20728 EVT VT = N->getValueType(0);
20729 SDValue CC = N->getOperand(0);
20730 SDValue TrueVal = N->getOperand(1);
20731 SDValue FalseVal = N->getOperand(2);
20732
20733 // Convert (vselect CC, true, false) to (vselect InvertCC, false, true) when
20734 // false has one use and true has multiple use.
20735 // It relies on RISCVVectorPeephole.cpp foldVMergeToMask to eliminate
20736 // vmerge.vv
20737 if (SDValue V = canonicalizeVSelectTrueToOneUse(N, DAG, Subtarget))
20738 return V;
20739
20740 // Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).
20741 // This allows us match a vadd.vv fed by a masked vrsub, which reduces
20742 // register pressure over the add followed by masked vsub sequence.
20743 bool SwapCC;
20744 if (!matchSelectAddSub(TrueVal, FalseVal, SwapCC))
20745 return SDValue();
20746
20747 SDValue Sub = SwapCC ? TrueVal : FalseVal;
20748 SDValue A = Sub.getOperand(0);
20749 SDValue B = Sub.getOperand(1);
20750
20751 // Arrange the select such that we can match a masked
20752 // vrsub.vi to perform the conditional negate
20753 SDValue NegB = DAG.getNegative(B, DL, VT);
20754 if (!SwapCC)
20755 CC = DAG.getLogicalNOT(DL, CC, CC->getValueType(0));
20756 SDValue NewB = DAG.getNode(ISD::VSELECT, DL, VT, CC, NegB, B);
20757 return DAG.getNode(ISD::ADD, DL, VT, A, NewB);
20758}
20759
20760// Fold (iN (select (src >u ((1 << N) - 1)), sext(src >s -1), trunc(src))) to
20761// USATI. This pattern saturates a signed value to an unsigned N-bit range
20762// [0, 2^N-1]:
20763// - If src < 0: result = 0
20764// (via the inner comparison src > -1 = false, sext to 0)
20765// - If src > ((1 << C) - 1): result = all 1s
20766// (via sext(true) = -1 = 0xFF...)
20767// - Otherwise: result = src (via trunc(src))
20768// The outer comparison is unsigned, so negative values appear as large
20769// unsigned values and trigger the saturation to MaxVal path, where the
20770// inner signed comparison then produces 0.
20771// TODO: Support (select (src <=u ((1 << C) - 1)), trunc(src), sext(src >s -1)).
20772static SDValue foldSelectToUSATI(SDNode *N, SelectionDAG &DAG,
20773 const RISCVSubtarget &Subtarget) {
20774 if (!Subtarget.hasStdExtP())
20775 return SDValue();
20776
20777 EVT VT = N->getValueType(0);
20778 MVT XLenVT = Subtarget.getXLenVT();
20779
20780 // Only support scalar integer types smaller than XLenVT
20781 if (!VT.isScalarInteger() || VT.bitsGE(XLenVT))
20782 return SDValue();
20783
20784 unsigned SatWidth = VT.getSizeInBits();
20785 uint64_t MaxVal = (1ULL << SatWidth) - 1;
20786
20787 using namespace SDPatternMatch;
20788
20789 SDValue Src, InnerSetCC, FalseSrc;
20790 if (!sd_match(N, m_Select(m_SetCC(m_Value(Src), m_SpecificInt(MaxVal),
20791 m_SpecificCondCode(ISD::SETUGT)),
20792 m_SExt(m_Value(InnerSetCC)),
20793 m_Trunc(m_Value(FalseSrc)))))
20794 return SDValue();
20795
20796 // Src can't be larger than XLenVT.
20797 if (Src.getValueType().bitsGT(XLenVT))
20798 return SDValue();
20799
20800 // Check inner setcc: src > -1 (signed comparison)
20801 if (!sd_match(InnerSetCC,
20802 m_SpecificVT(MVT::i1, m_SetCC(m_Specific(Src), m_AllOnes(),
20803 m_SpecificCondCode(ISD::SETGT)))))
20804 return SDValue();
20805
20806 // It's possible that the input to the setccs is also a truncate, in that
20807 // case the input to the truncate on the select's false operand may be the
20808 // same as the input to this setcc truncate. We need to look through the
20809 // setcc truncate to make sure CmpSrc and FalseSrc come from the same value.
20810 SDValue CmpSrc = Src;
20811 if (CmpSrc != FalseSrc && CmpSrc.getOpcode() == ISD::TRUNCATE)
20812 CmpSrc = CmpSrc.getOperand(0);
20813
20814 if (CmpSrc != FalseSrc)
20815 return SDValue();
20816
20817 // We found a USATI pattern.
20818 SDLoc DL(N);
20819 Src = DAG.getNode(ISD::SIGN_EXTEND, DL, XLenVT, Src);
20820 SDValue USATI = DAG.getNode(RISCVISD::USATI, DL, XLenVT, Src,
20821 DAG.getTargetConstant(SatWidth, DL, XLenVT));
20822 return DAG.getNode(ISD::TRUNCATE, DL, VT, USATI);
20823}
20824
20825static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
20826 const RISCVSubtarget &Subtarget) {
20827 if (SDValue Folded = foldSelectOfCTTZOrCTLZ(N, DAG))
20828 return Folded;
20829
20830 if (SDValue V = foldSelectToUSATI(N, DAG, Subtarget))
20831 return V;
20832
20833 if (SDValue V = useInversedSetcc(N, DAG, Subtarget))
20834 return V;
20835
20836 if (Subtarget.hasConditionalMoveFusion())
20837 return SDValue();
20838
20839 SDValue TrueVal = N->getOperand(1);
20840 SDValue FalseVal = N->getOperand(2);
20841 if (SDValue V = tryFoldSelectIntoOp(N, DAG, TrueVal, FalseVal, /*Swapped*/false))
20842 return V;
20843 return tryFoldSelectIntoOp(N, DAG, FalseVal, TrueVal, /*Swapped*/true);
20844}
20845
20846/// If we have a build_vector where each lane is binop X, C, where C
20847/// is a constant (but not necessarily the same constant on all lanes),
20848/// form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
20849/// We assume that materializing a constant build vector will be no more
20850/// expensive that performing O(n) binops.
20851static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
20852 const RISCVSubtarget &Subtarget,
20853 const RISCVTargetLowering &TLI) {
20854 SDLoc DL(N);
20855 EVT VT = N->getValueType(0);
20856
20857 assert(!VT.isScalableVector() && "unexpected build vector");
20858
20859 if (VT.getVectorNumElements() == 1)
20860 return SDValue();
20861
20862 const unsigned Opcode = N->op_begin()->getNode()->getOpcode();
20863 if (!TLI.isBinOp(Opcode))
20864 return SDValue();
20865
20866 if (!TLI.isOperationLegalOrCustom(Opcode, VT) || !TLI.isTypeLegal(VT))
20867 return SDValue();
20868
20869 // This BUILD_VECTOR involves an implicit truncation, and sinking
20870 // truncates through binops is non-trivial.
20871 if (N->op_begin()->getValueType() != VT.getVectorElementType())
20872 return SDValue();
20873
20874 SmallVector<SDValue> LHSOps;
20875 SmallVector<SDValue> RHSOps;
20876 for (SDValue Op : N->ops()) {
20877 if (Op.isUndef()) {
20878 // We can't form a divide or remainder from undef.
20879 if (!DAG.isSafeToSpeculativelyExecute(Opcode))
20880 return SDValue();
20881
20882 LHSOps.push_back(Op);
20883 RHSOps.push_back(Op);
20884 continue;
20885 }
20886
20887 // TODO: We can handle operations which have an neutral rhs value
20888 // (e.g. x + 0, a * 1 or a << 0), but we then have to keep track
20889 // of profit in a more explicit manner.
20890 if (Op.getOpcode() != Opcode || !Op.hasOneUse())
20891 return SDValue();
20892
20893 LHSOps.push_back(Op.getOperand(0));
20894 if (!isa<ConstantSDNode>(Op.getOperand(1)) &&
20895 !isa<ConstantFPSDNode>(Op.getOperand(1)))
20896 return SDValue();
20897 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
20898 // have different LHS and RHS types.
20899 if (Op.getOperand(0).getValueType() != Op.getOperand(1).getValueType())
20900 return SDValue();
20901
20902 RHSOps.push_back(Op.getOperand(1));
20903 }
20904
20905 return DAG.getNode(Opcode, DL, VT, DAG.getBuildVector(VT, DL, LHSOps),
20906 DAG.getBuildVector(VT, DL, RHSOps));
20907}
20908
20909static MVT getQDOTXResultType(MVT OpVT) {
20910 ElementCount OpEC = OpVT.getVectorElementCount();
20911 assert(OpEC.isKnownMultipleOf(4) && OpVT.getVectorElementType() == MVT::i8);
20912 return MVT::getVectorVT(MVT::i32, OpEC.divideCoefficientBy(4));
20913}
20914
20915/// Given fixed length vectors A and B with equal element types, but possibly
20916/// different number of elements, return A + B where either A or B is zero
20917/// padded to the larger number of elements.
20918static SDValue getZeroPaddedAdd(const SDLoc &DL, SDValue A, SDValue B,
20919 SelectionDAG &DAG) {
20920 // NOTE: Manually doing the extract/add/insert scheme produces
20921 // significantly better codegen than the naive pad with zeros
20922 // and add scheme.
20923 EVT AVT = A.getValueType();
20924 EVT BVT = B.getValueType();
20925 assert(AVT.getVectorElementType() == BVT.getVectorElementType());
20926 if (AVT.getVectorMinNumElements() > BVT.getVectorMinNumElements()) {
20927 std::swap(A, B);
20928 std::swap(AVT, BVT);
20929 }
20930
20931 SDValue BPart = DAG.getExtractSubvector(DL, AVT, B, 0);
20932 SDValue Res = DAG.getNode(ISD::ADD, DL, AVT, A, BPart);
20933 return DAG.getInsertSubvector(DL, B, Res, 0);
20934}
20935
20936static SDValue foldReduceOperandViaVDOT4A(SDValue InVec, const SDLoc &DL,
20937 SelectionDAG &DAG,
20938 const RISCVSubtarget &Subtarget,
20939 const RISCVTargetLowering &TLI) {
20940 using namespace SDPatternMatch;
20941 // Note: We intentionally do not check the legality of the reduction type.
20942 // We want to handle the m4/m8 *src* types, and thus need to let illegal
20943 // intermediate types flow through here.
20944 if (InVec.getValueType().getVectorElementType() != MVT::i32 ||
20945 !InVec.getValueType().getVectorElementCount().isKnownMultipleOf(4))
20946 return SDValue();
20947
20948 // Recurse through adds/disjoint ors (since generic dag canonicalizes to that
20949 // form).
20950 SDValue A, B;
20951 if (sd_match(InVec, m_AddLike(m_Value(A), m_Value(B)))) {
20952 SDValue AOpt = foldReduceOperandViaVDOT4A(A, DL, DAG, Subtarget, TLI);
20953 SDValue BOpt = foldReduceOperandViaVDOT4A(B, DL, DAG, Subtarget, TLI);
20954 if (AOpt || BOpt) {
20955 if (AOpt)
20956 A = AOpt;
20957 if (BOpt)
20958 B = BOpt;
20959 // From here, we're doing A + B with mixed types, implicitly zero
20960 // padded to the wider type. Note that we *don't* need the result
20961 // type to be the original VT, and in fact prefer narrower ones
20962 // if possible.
20963 return getZeroPaddedAdd(DL, A, B, DAG);
20964 }
20965 }
20966
20967 // zext a <--> partial_reduce_umla 0, a, 1
20968 // sext a <--> partial_reduce_smla 0, a, 1
20969 if (InVec.getOpcode() == ISD::ZERO_EXTEND ||
20970 InVec.getOpcode() == ISD::SIGN_EXTEND) {
20971 SDValue A = InVec.getOperand(0);
20972 EVT OpVT = A.getValueType();
20973 if (OpVT.getVectorElementType() != MVT::i8 || !TLI.isTypeLegal(OpVT))
20974 return SDValue();
20975
20976 MVT ResVT = getQDOTXResultType(A.getSimpleValueType());
20977 SDValue B = DAG.getConstant(0x1, DL, OpVT);
20978 bool IsSigned = InVec.getOpcode() == ISD::SIGN_EXTEND;
20979 unsigned Opc =
20980 IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
20981