LLVM 24.0.0git
RISCVLegalizerInfo.cpp
Go to the documentation of this file.
1//===-- RISCVLegalizerInfo.cpp ----------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the Machinelegalizer class for RISC-V.
10/// \todo This should be generated by TableGen.
11//===----------------------------------------------------------------------===//
12
13#include "RISCVLegalizerInfo.h"
16#include "RISCVSubtarget.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsRISCV.h"
32#include "llvm/IR/Type.h"
33
34using namespace llvm;
35using namespace LegalityPredicates;
36using namespace LegalizeMutations;
37using namespace MIPatternMatch;
38
40typeIsLegalIntOrFPVec(unsigned TypeIdx,
41 std::initializer_list<LLT> IntOrFPVecTys,
42 const RISCVSubtarget &ST) {
43 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
44 return ST.hasVInstructions() &&
45 (Query.Types[TypeIdx].getScalarSizeInBits() != 64 ||
46 ST.hasVInstructionsI64()) &&
47 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
48 ST.getELen() == 64);
49 };
50
51 return all(typeInSet(TypeIdx, IntOrFPVecTys), P);
52}
53
55typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list<LLT> BoolVecTys,
56 const RISCVSubtarget &ST) {
57 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
58 return ST.hasVInstructions() &&
59 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
60 ST.getELen() == 64);
61 };
62 return all(typeInSet(TypeIdx, BoolVecTys), P);
63}
64
65static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx,
66 std::initializer_list<LLT> PtrVecTys,
67 const RISCVSubtarget &ST) {
68 LegalityPredicate P = [=, &ST](const LegalityQuery &Query) {
69 return ST.hasVInstructions() &&
70 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 1 ||
71 ST.getELen() == 64) &&
72 (Query.Types[TypeIdx].getElementCount().getKnownMinValue() != 16 ||
73 Query.Types[TypeIdx].getScalarSizeInBits() == 32);
74 };
75 return all(typeInSet(TypeIdx, PtrVecTys), P);
76}
77
79 : STI(ST), XLen(STI.getXLen()), sXLen(LLT::scalar(XLen)),
80 iXLen(LLT::integer(XLen)) {
81 const LLT sDoubleXLen = LLT::scalar(2 * XLen);
82 const LLT p0 = LLT::pointer(0, XLen);
83 const LLT s1 = LLT::scalar(1);
84 const LLT s8 = LLT::scalar(8);
85 const LLT s16 = LLT::scalar(16);
86 const LLT f16 = LLT::float16();
87 const LLT f32 = LLT::float32();
88 const LLT f64 = LLT::float64();
89 const LLT f128 = LLT::float128();
90 const LLT s32 = LLT::scalar(32);
91 const LLT s64 = LLT::scalar(64);
92 const LLT s128 = LLT::scalar(128);
93
94 const LLT nxv1s1 = LLT::scalable_vector(1, s1);
95 const LLT nxv2s1 = LLT::scalable_vector(2, s1);
96 const LLT nxv4s1 = LLT::scalable_vector(4, s1);
97 const LLT nxv8s1 = LLT::scalable_vector(8, s1);
98 const LLT nxv16s1 = LLT::scalable_vector(16, s1);
99 const LLT nxv32s1 = LLT::scalable_vector(32, s1);
100 const LLT nxv64s1 = LLT::scalable_vector(64, s1);
101
102 const LLT nxv1s8 = LLT::scalable_vector(1, s8);
103 const LLT nxv2s8 = LLT::scalable_vector(2, s8);
104 const LLT nxv4s8 = LLT::scalable_vector(4, s8);
105 const LLT nxv8s8 = LLT::scalable_vector(8, s8);
106 const LLT nxv16s8 = LLT::scalable_vector(16, s8);
107 const LLT nxv32s8 = LLT::scalable_vector(32, s8);
108 const LLT nxv64s8 = LLT::scalable_vector(64, s8);
109
110 const LLT nxv1s16 = LLT::scalable_vector(1, s16);
111 const LLT nxv2s16 = LLT::scalable_vector(2, s16);
112 const LLT nxv4s16 = LLT::scalable_vector(4, s16);
113 const LLT nxv8s16 = LLT::scalable_vector(8, s16);
114 const LLT nxv16s16 = LLT::scalable_vector(16, s16);
115 const LLT nxv32s16 = LLT::scalable_vector(32, s16);
116
117 const LLT nxv1s32 = LLT::scalable_vector(1, s32);
118 const LLT nxv2s32 = LLT::scalable_vector(2, s32);
119 const LLT nxv4s32 = LLT::scalable_vector(4, s32);
120 const LLT nxv8s32 = LLT::scalable_vector(8, s32);
121 const LLT nxv16s32 = LLT::scalable_vector(16, s32);
122
123 const LLT nxv1s64 = LLT::scalable_vector(1, s64);
124 const LLT nxv2s64 = LLT::scalable_vector(2, s64);
125 const LLT nxv4s64 = LLT::scalable_vector(4, s64);
126 const LLT nxv8s64 = LLT::scalable_vector(8, s64);
127
128 const LLT nxv1p0 = LLT::scalable_vector(1, p0);
129 const LLT nxv2p0 = LLT::scalable_vector(2, p0);
130 const LLT nxv4p0 = LLT::scalable_vector(4, p0);
131 const LLT nxv8p0 = LLT::scalable_vector(8, p0);
132 const LLT nxv16p0 = LLT::scalable_vector(16, p0);
133
134 using namespace TargetOpcode;
135
136 auto BoolVecTys = {nxv1s1, nxv2s1, nxv4s1, nxv8s1, nxv16s1, nxv32s1, nxv64s1};
137
138 auto IntOrFPVecTys = {nxv1s8, nxv2s8, nxv4s8, nxv8s8, nxv16s8, nxv32s8,
139 nxv64s8, nxv1s16, nxv2s16, nxv4s16, nxv8s16, nxv16s16,
140 nxv32s16, nxv1s32, nxv2s32, nxv4s32, nxv8s32, nxv16s32,
141 nxv1s64, nxv2s64, nxv4s64, nxv8s64};
142
143 auto PtrVecTys = {nxv1p0, nxv2p0, nxv4p0, nxv8p0, nxv16p0};
144
145 getActionDefinitionsBuilder({G_ADD, G_SUB})
146 .legalFor({sXLen})
147 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
148 .customFor(ST.is64Bit(), {s32})
150 .clampScalar(0, sXLen, sXLen);
151
152 getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
153 .legalFor({sXLen})
154 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
156 .clampScalar(0, sXLen, sXLen);
157
159 {G_UADDE, G_UADDO, G_USUBE, G_USUBO, G_READ_REGISTER, G_WRITE_REGISTER})
160 .lower();
161
162 getActionDefinitionsBuilder({G_SADDE, G_SADDO, G_SSUBE, G_SSUBO})
163 .minScalar(0, sXLen)
164 .lower();
165
166 // TODO: Use Vector Single-Width Saturating Instructions for vector types.
168 {G_UADDSAT, G_SADDSAT, G_USUBSAT, G_SSUBSAT, G_SSHLSAT, G_USHLSAT})
169 .lower();
170
171 getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
172 .legalFor({{sXLen, sXLen}})
173 .customFor(ST.is64Bit(), {{s32, s32}})
174 .widenScalarToNextPow2(0)
175 .clampScalar(1, sXLen, sXLen)
176 .clampScalar(0, sXLen, sXLen);
177
178 getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
179 .legalFor({{s32, s16}})
180 .legalFor(ST.is64Bit(), {{s64, s16}, {s64, s32}})
181 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
182 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)))
183 .customIf(typeIsLegalBoolVec(1, BoolVecTys, ST))
184 .maxScalar(0, sXLen);
185
186 getActionDefinitionsBuilder(G_TRUNC).alwaysLegal();
187
188 {
189 LegalityPredicate ValidSextInRegWidth = all(sizeIs(0, 64), immIs(0, 32));
190
191 if (STI.hasStdExtZbb())
192 ValidSextInRegWidth =
193 LegalityPredicates::any(ValidSextInRegWidth, immInSet(0, {8, 16}));
194
195 getActionDefinitionsBuilder(G_SEXT_INREG)
196 .legalIf(all(typeIs(0, sXLen), ValidSextInRegWidth))
197 .clampScalar(0, sXLen, sXLen)
198 .lower();
199 }
200
201 // Merge/Unmerge
202 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
203 auto &MergeUnmergeActions = getActionDefinitionsBuilder(Op);
204 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
205 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
206 if (XLen == 32 && (ST.hasStdExtD() || ST.hasStdExtZilsd())) {
207 MergeUnmergeActions.legalIf(
208 all(typeIs(BigTyIdx, s64), typeIs(LitTyIdx, s32)));
209 }
210 MergeUnmergeActions.widenScalarToNextPow2(LitTyIdx, XLen)
211 .widenScalarToNextPow2(BigTyIdx, XLen)
212 .clampScalar(LitTyIdx, sXLen, sXLen)
213 .clampScalar(BigTyIdx, sXLen, sXLen);
214 }
215
216 getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
217
218 getActionDefinitionsBuilder({G_ROTR, G_ROTL})
219 .legalFor(ST.hasStdExtZbb() || ST.hasStdExtZbkb(), {{sXLen, sXLen}})
220 .customFor(ST.is64Bit() && (ST.hasStdExtZbb() || ST.hasStdExtZbkb()),
221 {{s32, s32}})
222 .lower();
223
224 getActionDefinitionsBuilder(G_BITREVERSE)
225 .customFor(ST.hasStdExtZbkb(), {s8})
226 .lowerIf(scalarNarrowerThan(0, 8))
227 .widenScalarToNextPow2(0)
228 .maxScalar(0, sXLen)
229 .lower();
230
231 getActionDefinitionsBuilder(G_BITCAST)
232 .legalIf(all(
234 typeIsLegalBoolVec(0, BoolVecTys, ST)),
236 typeIsLegalBoolVec(1, BoolVecTys, ST))))
237 // Scalar int <-> fp bitcasts that fit in a GPR are register moves.
238 .legalIf(all(isScalar(0), isScalar(1), scalarNarrowerThan(0, XLen + 1)))
239 // Wider ones (i128/f128, or f64/i64 on RV32) are split into XLen pieces.
240 .narrowScalarIf(all(isScalar(0), isScalar(1), scalarWiderThan(0, XLen)),
242
243 auto &BSWAPActions = getActionDefinitionsBuilder(G_BSWAP);
244 if (ST.hasStdExtZbb() || ST.hasStdExtZbkb())
245 BSWAPActions.legalFor({sXLen}).clampScalar(0, sXLen, sXLen);
246 else
247 BSWAPActions.maxScalar(0, sXLen).lower();
248
249 getActionDefinitionsBuilder(G_CLMUL)
250 .legalFor(ST.hasStdExtZbkc(), {sXLen})
251 .unsupported();
252
253 getActionDefinitionsBuilder(G_CLMULH)
254 .legalFor(ST.hasStdExtZbkc(), {sXLen})
255 .customFor(ST.is64Bit() && ST.hasStdExtZbkc(), {s32})
256 .unsupported();
257
258 // CLMULR is Zbc-only; Zbkc is a subset that has CLMUL/CLMULH but not CLMULR.
259 getActionDefinitionsBuilder(G_CLMULR)
260 .legalFor(ST.hasStdExtZbc(), {sXLen})
261 .customFor(ST.is64Bit() && ST.hasStdExtZbc(), {s32})
262 .unsupported();
263
264 auto &CTLZPoisonActions = getActionDefinitionsBuilder(G_CTLZ_ZERO_POISON);
265 auto &CTTZPoisonActions = getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON);
266 if (ST.hasStdExtZbb()) {
267 getActionDefinitionsBuilder({G_CTLZ, G_CTTZ})
268 .legalFor({{sXLen, sXLen}})
269 .customFor({{s32, s32}})
270 .clampScalar(0, s32, sXLen)
271 .widenScalarToNextPow2(0)
272 .scalarSameSizeAs(1, 0);
273 } else {
274 auto &CTLZActions = getActionDefinitionsBuilder(G_CTLZ);
275 if (ST.hasVendorXTHeadBb())
276 CTLZActions.legalFor({{sXLen, sXLen}})
277 .clampScalar(0, sXLen, sXLen)
278 .scalarSameSizeAs(1, 0);
279 else
280 CTLZActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower();
281
282 auto &CTTZActions = getActionDefinitionsBuilder(G_CTTZ);
283 if (ST.hasVendorXTHeadBb()) {
284 CTTZActions.clampScalar(0, sXLen, sXLen).scalarSameSizeAs(1, 0).lower();
285 CTLZPoisonActions.clampScalar(0, sXLen, sXLen).scalarSameSizeAs(1, 0);
286 CTTZPoisonActions.clampScalar(0, sXLen, sXLen).scalarSameSizeAs(1, 0);
287 } else {
288 CTTZActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower();
289 CTLZPoisonActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0);
290 CTTZPoisonActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0);
291 }
292 }
293 CTLZPoisonActions.lower();
294 CTTZPoisonActions.lower();
295
296 auto &CountSignActions = getActionDefinitionsBuilder(G_CTLS);
297 if (ST.hasStdExtP()) {
298 CountSignActions.legalFor({{sXLen, sXLen}})
299 .customFor({{s32, s32}})
300 .clampScalar(0, s32, sXLen)
301 .widenScalarToNextPow2(0)
302 .scalarSameSizeAs(1, 0);
303 } else {
304 CountSignActions.maxScalar(0, sXLen).scalarSameSizeAs(1, 0).lower();
305 }
306
307 auto &CTPOPActions = getActionDefinitionsBuilder(G_CTPOP);
308 if (ST.hasStdExtZbb()) {
309 CTPOPActions.legalFor({{sXLen, sXLen}})
310 .clampScalar(0, sXLen, sXLen)
311 .scalarSameSizeAs(1, 0);
312 } else {
313 CTPOPActions.widenScalarToNextPow2(0, /*Min*/ 8)
314 .clampScalar(0, s8, sXLen)
315 .scalarSameSizeAs(1, 0)
316 .lower();
317 }
318
319 getActionDefinitionsBuilder(G_CONSTANT)
320 .legalFor({p0})
321 .legalFor(!ST.is64Bit(), {s32})
322 .customFor(ST.is64Bit(), {s64})
323 .widenScalarToNextPow2(0)
324 .clampScalar(0, sXLen, sXLen);
325
326 // TODO: transform illegal vector types into legal vector type
327 getActionDefinitionsBuilder(G_FREEZE)
328 .legalFor({s16, s32, p0})
329 .legalFor(ST.is64Bit(), {s64})
330 .legalIf(typeIsLegalBoolVec(0, BoolVecTys, ST))
331 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
332 .widenScalarToNextPow2(0)
333 .clampScalar(0, s16, sXLen);
334
335 // TODO: transform illegal vector types into legal vector type
336 // TODO: Merge with G_FREEZE?
337 getActionDefinitionsBuilder(
338 {G_IMPLICIT_DEF, G_CONSTANT_FOLD_BARRIER})
339 .legalFor({s32, sXLen, p0})
340 .legalIf(typeIsLegalBoolVec(0, BoolVecTys, ST))
341 .legalIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST))
342 .widenScalarToNextPow2(0)
343 .clampScalar(0, s32, sXLen);
344
345 getActionDefinitionsBuilder(G_ICMP)
346 .legalFor({{sXLen, sXLen}, {sXLen, p0}})
347 .legalIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST),
348 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)))
349 .widenScalarOrEltToNextPow2OrMinSize(1, 8)
350 .clampScalar(1, sXLen, sXLen)
351 .clampScalar(0, sXLen, sXLen);
352
353 getActionDefinitionsBuilder(G_SELECT)
354 .legalFor({{s32, sXLen}, {p0, sXLen}})
355 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
356 typeIsLegalBoolVec(1, BoolVecTys, ST)))
357 .legalFor(XLen == 64 || ST.hasStdExtD(), {{s64, sXLen}})
358 .widenScalarToNextPow2(0)
359 .clampScalar(0, s32, (XLen == 64 || ST.hasStdExtD()) ? s64 : s32)
360 .clampScalar(1, sXLen, sXLen);
361
362 auto &LoadActions = getActionDefinitionsBuilder(G_LOAD);
363 auto &StoreActions = getActionDefinitionsBuilder(G_STORE);
364 auto &ExtLoadActions = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD});
365
366 // Return the alignment needed for scalar memory ops. If unaligned scalar mem
367 // is supported, we only require byte alignment. Otherwise, we need the memory
368 // op to be natively aligned.
369 auto getScalarMemAlign = [&ST](unsigned Size) {
370 return ST.enableUnalignedScalarMem() ? 8 : Size;
371 };
372
373 LoadActions.legalForTypesWithMemDesc(
374 {{s16, p0, s8, getScalarMemAlign(8)},
375 {s32, p0, s8, getScalarMemAlign(8)},
376 {s16, p0, s16, getScalarMemAlign(16)},
377 {s32, p0, s16, getScalarMemAlign(16)},
378 {s32, p0, s32, getScalarMemAlign(32)},
379 {p0, p0, sXLen, getScalarMemAlign(XLen)}});
380 StoreActions.legalForTypesWithMemDesc(
381 {{s16, p0, s8, getScalarMemAlign(8)},
382 {s32, p0, s8, getScalarMemAlign(8)},
383 {s16, p0, s16, getScalarMemAlign(16)},
384 {s32, p0, s16, getScalarMemAlign(16)},
385 {s32, p0, s32, getScalarMemAlign(32)},
386 {p0, p0, sXLen, getScalarMemAlign(XLen)}});
387 ExtLoadActions.legalForTypesWithMemDesc(
388 {{sXLen, p0, s8, getScalarMemAlign(8)},
389 {sXLen, p0, s16, getScalarMemAlign(16)}});
390 if (XLen == 64) {
391 LoadActions.legalForTypesWithMemDesc(
392 {{s64, p0, s8, getScalarMemAlign(8)},
393 {s64, p0, s16, getScalarMemAlign(16)},
394 {s64, p0, s32, getScalarMemAlign(32)},
395 {s64, p0, s64, getScalarMemAlign(64)}});
396 StoreActions.legalForTypesWithMemDesc(
397 {{s64, p0, s8, getScalarMemAlign(8)},
398 {s64, p0, s16, getScalarMemAlign(16)},
399 {s64, p0, s32, getScalarMemAlign(32)},
400 {s64, p0, s64, getScalarMemAlign(64)}});
401 ExtLoadActions.legalForTypesWithMemDesc(
402 {{s64, p0, s32, getScalarMemAlign(32)}});
403 } else if (ST.hasStdExtD() || ST.hasStdExtZilsd()) {
404 // RV32: s64 is legal for f64 (D/Zdinx) or i64 (Zilsd); LLT cannot tell the
405 // two apart, so one rule covers both.
406 // FIXME: We require natural alignment, so under -mzilsd-word-align an
407 // align-4 i64 access is split into two s32 ones. riscv-prera-zilsd-opt
408 // merges them back into LD_RV32/SD_RV32, but it is skipped at -O0.
409 LoadActions.legalForTypesWithMemDesc(
410 {{s64, p0, s64, getScalarMemAlign(64)}});
411 StoreActions.legalForTypesWithMemDesc(
412 {{s64, p0, s64, getScalarMemAlign(64)}});
413 }
414
415 // Vector loads/stores.
416 if (ST.hasVInstructions()) {
417 LoadActions.legalForTypesWithMemDesc({{nxv2s8, p0, nxv2s8, 8},
418 {nxv4s8, p0, nxv4s8, 8},
419 {nxv8s8, p0, nxv8s8, 8},
420 {nxv16s8, p0, nxv16s8, 8},
421 {nxv32s8, p0, nxv32s8, 8},
422 {nxv64s8, p0, nxv64s8, 8},
423 {nxv2s16, p0, nxv2s16, 16},
424 {nxv4s16, p0, nxv4s16, 16},
425 {nxv8s16, p0, nxv8s16, 16},
426 {nxv16s16, p0, nxv16s16, 16},
427 {nxv32s16, p0, nxv32s16, 16},
428 {nxv2s32, p0, nxv2s32, 32},
429 {nxv4s32, p0, nxv4s32, 32},
430 {nxv8s32, p0, nxv8s32, 32},
431 {nxv16s32, p0, nxv16s32, 32}});
432 StoreActions.legalForTypesWithMemDesc({{nxv2s8, p0, nxv2s8, 8},
433 {nxv4s8, p0, nxv4s8, 8},
434 {nxv8s8, p0, nxv8s8, 8},
435 {nxv16s8, p0, nxv16s8, 8},
436 {nxv32s8, p0, nxv32s8, 8},
437 {nxv64s8, p0, nxv64s8, 8},
438 {nxv2s16, p0, nxv2s16, 16},
439 {nxv4s16, p0, nxv4s16, 16},
440 {nxv8s16, p0, nxv8s16, 16},
441 {nxv16s16, p0, nxv16s16, 16},
442 {nxv32s16, p0, nxv32s16, 16},
443 {nxv2s32, p0, nxv2s32, 32},
444 {nxv4s32, p0, nxv4s32, 32},
445 {nxv8s32, p0, nxv8s32, 32},
446 {nxv16s32, p0, nxv16s32, 32}});
447
448 if (ST.getELen() == 64) {
449 LoadActions.legalForTypesWithMemDesc({{nxv1s8, p0, nxv1s8, 8},
450 {nxv1s16, p0, nxv1s16, 16},
451 {nxv1s32, p0, nxv1s32, 32}});
452 StoreActions.legalForTypesWithMemDesc({{nxv1s8, p0, nxv1s8, 8},
453 {nxv1s16, p0, nxv1s16, 16},
454 {nxv1s32, p0, nxv1s32, 32}});
455 }
456
457 if (ST.hasVInstructionsI64()) {
458 LoadActions.legalForTypesWithMemDesc({{nxv1s64, p0, nxv1s64, 64},
459 {nxv2s64, p0, nxv2s64, 64},
460 {nxv4s64, p0, nxv4s64, 64},
461 {nxv8s64, p0, nxv8s64, 64}});
462 StoreActions.legalForTypesWithMemDesc({{nxv1s64, p0, nxv1s64, 64},
463 {nxv2s64, p0, nxv2s64, 64},
464 {nxv4s64, p0, nxv4s64, 64},
465 {nxv8s64, p0, nxv8s64, 64}});
466 }
467
468 // we will take the custom lowering logic if we have scalable vector types
469 // with non-standard alignments
470 LoadActions.customIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST));
471 StoreActions.customIf(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST));
472
473 // Pointers require that XLen sized elements are legal.
474 if (XLen <= ST.getELen()) {
475 LoadActions.customIf(typeIsLegalPtrVec(0, PtrVecTys, ST));
476 StoreActions.customIf(typeIsLegalPtrVec(0, PtrVecTys, ST));
477 }
478 }
479
480 LoadActions.widenScalarToNextPow2(0, /* MinSize = */ 8)
481 .lowerIfMemSizeNotByteSizePow2()
482 .clampScalar(0, s16, sXLen)
483 .lower();
484 StoreActions
485 .clampScalar(0, s16, sXLen)
486 .lowerIfMemSizeNotByteSizePow2()
487 .lower();
488
489 ExtLoadActions.widenScalarToNextPow2(0).clampScalar(0, sXLen, sXLen).lower();
490
491 getActionDefinitionsBuilder({G_PTR_ADD, G_PTRMASK}).legalFor({{p0, sXLen}});
492
493 getActionDefinitionsBuilder(G_PTRTOINT)
494 .legalFor({{sXLen, p0}})
495 .clampScalar(0, sXLen, sXLen);
496
497 getActionDefinitionsBuilder(G_INTTOPTR)
498 .legalFor({{p0, sXLen}})
499 .clampScalar(1, sXLen, sXLen);
500
501 getActionDefinitionsBuilder(G_BR).alwaysLegal();
502
503 getActionDefinitionsBuilder(G_BRCOND).legalFor({sXLen}).minScalar(0, sXLen);
504
505 getActionDefinitionsBuilder(G_BRJT).customFor({{p0, sXLen}});
506
507 getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});
508
509 getActionDefinitionsBuilder(G_PHI)
510 .legalFor({p0, s32, sXLen})
511 .widenScalarToNextPow2(0)
512 .clampScalar(0, s32, sXLen);
513
514 getActionDefinitionsBuilder({G_GLOBAL_VALUE, G_JUMP_TABLE, G_CONSTANT_POOL})
515 .legalFor({p0});
516
517 if (ST.hasStdExtZmmul()) {
518 getActionDefinitionsBuilder(G_MUL)
519 .legalFor({sXLen})
520 .widenScalarToNextPow2(0)
521 .clampScalar(0, sXLen, sXLen);
522
523 // clang-format off
524 getActionDefinitionsBuilder({G_SMULH, G_UMULH})
525 .legalFor({sXLen})
526 .lower();
527 // clang-format on
528
529 getActionDefinitionsBuilder({G_SMULO, G_UMULO}).minScalar(0, sXLen).lower();
530 } else {
531 getActionDefinitionsBuilder(G_MUL)
532 .libcallFor({sXLen, sDoubleXLen})
533 .widenScalarToNextPow2(0)
534 .clampScalar(0, sXLen, sDoubleXLen);
535
536 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).lowerFor({sXLen});
537
538 getActionDefinitionsBuilder({G_SMULO, G_UMULO})
539 .minScalar(0, sXLen)
540 // Widen sXLen to sDoubleXLen so we can use a single libcall to get
541 // the low bits for the mul result and high bits to do the overflow
542 // check.
543 .widenScalarIf(typeIs(0, sXLen),
544 LegalizeMutations::changeTo(0, sDoubleXLen))
545 .lower();
546 }
547
548 if (ST.hasStdExtM()) {
549 getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_UREM})
550 .legalFor({sXLen})
551 .customFor({s32})
552 .libcallFor({sDoubleXLen})
553 .clampScalar(0, s32, sDoubleXLen)
554 .widenScalarToNextPow2(0);
555 getActionDefinitionsBuilder(G_SREM)
556 .legalFor({sXLen})
557 .libcallFor({sDoubleXLen})
558 .clampScalar(0, sXLen, sDoubleXLen)
559 .widenScalarToNextPow2(0);
560 } else {
561 getActionDefinitionsBuilder({G_UDIV, G_SDIV, G_UREM, G_SREM})
562 .libcallFor({sXLen, sDoubleXLen})
563 .clampScalar(0, sXLen, sDoubleXLen)
564 .widenScalarToNextPow2(0);
565 }
566
567 // TODO: Use libcall for sDoubleXLen.
568 getActionDefinitionsBuilder({G_SDIVREM, G_UDIVREM}).lower();
569
570 getActionDefinitionsBuilder(G_ABS)
571 .customFor(ST.hasStdExtZbb(), {sXLen})
572 .minScalar(ST.hasStdExtZbb(), 0, sXLen)
573 .lower();
574
575 getActionDefinitionsBuilder({G_ABDS, G_ABDU})
576 .minScalar(ST.hasStdExtZbb(), 0, sXLen)
577 .lower();
578
579 getActionDefinitionsBuilder({G_UMAX, G_UMIN, G_SMAX, G_SMIN})
580 .legalFor(ST.hasStdExtZbb(), {sXLen})
581 .minScalar(ST.hasStdExtZbb(), 0, sXLen)
582 .lower();
583
584 getActionDefinitionsBuilder({G_SCMP, G_UCMP}).lower();
585
586 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
587
588 getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
589
590 getActionDefinitionsBuilder({G_MEMCPY_INLINE, G_MEMSET_INLINE}).lower();
591
592 getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
593 .lower();
594
595 // On RV64 the 64-bit counter CSRs (cycle/time) are read directly. On RV32
596 // they are custom-legally lowered to a re-read-the-high-half loop (see
597 // legalizeReadCounter).
598 getActionDefinitionsBuilder({G_READCYCLECOUNTER, G_READSTEADYCOUNTER})
599 .legalFor(ST.is64Bit(), {s64})
600 .customFor(!ST.is64Bit(), {s64});
601
602 // FP Operations
603
604 // FIXME: Support f128 for rv32 when libcall handling is able to use sret.
605 getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FMA, G_FSQRT,
606 G_FMAXNUM, G_FMINNUM, G_FMAXIMUMNUM,
607 G_FMINIMUMNUM})
608 .legalFor(ST.hasStdExtF(), {f32})
609 .legalFor(ST.hasStdExtD(), {f64})
610 .legalFor(ST.hasStdExtZfh(), {f16})
611 .libcallFor({f32, f64})
612 .libcallFor(ST.is64Bit(), {f128});
613
614 getActionDefinitionsBuilder({G_FNEG, G_FABS})
615 .legalFor(ST.hasStdExtF(), {f32})
616 .legalFor(ST.hasStdExtD(), {f64})
617 .legalFor(ST.hasStdExtZfh(), {f16})
618 .lowerFor({f32, f64, f128});
619
620 getActionDefinitionsBuilder(G_FREM)
621 .libcallFor({f32, f64})
622 .libcallFor(ST.is64Bit(), {f128})
623 .minScalar(0, f32)
624 .scalarize(0);
625
626 getActionDefinitionsBuilder(G_FCOPYSIGN)
627 .legalFor(ST.hasStdExtF(), {{f32, f32}})
628 .legalFor(ST.hasStdExtD(), {{f64, f64}, {f32, f64}, {f64, f32}})
629 .legalFor(ST.hasStdExtZfh(), {{f16, f16}, {f16, f32}, {f32, f16}})
630 .legalFor(ST.hasStdExtZfh() && ST.hasStdExtD(), {{f16, f64}, {f64, f16}})
631 .lower();
632
633 getActionDefinitionsBuilder(G_FPTRUNC)
634 .legalFor(ST.hasStdExtD(), {{f32, f64}})
635 .legalFor(ST.hasStdExtZfhmin(), {{f16, f32}})
636 .legalFor(ST.hasStdExtZfhmin() && ST.hasStdExtD(), {{f16, f64}})
637 .libcallFor({{f16, f32}, {f16, f64}, {f32, f64}})
638 .libcallFor(ST.is64Bit(), {{f32, f128}, {f64, f128}});
639 getActionDefinitionsBuilder(G_FPEXT)
640 .legalFor(ST.hasStdExtD(), {{f64, f32}})
641 .legalFor(ST.hasStdExtZfhmin(), {{f32, f16}})
642 .legalFor(ST.hasStdExtZfhmin() && ST.hasStdExtD(), {{f64, f16}})
643 .libcallFor(!ST.hasStdExtZfhmin(), {{f32, f16}})
644 .libcallFor(!(ST.hasStdExtZfhmin() && ST.hasStdExtD()), {{f64, f16}})
645 .libcallFor({{f64, f32}})
646 .libcallFor(ST.is64Bit(), {{f128, f32}, {f128, f64}});
647
648 getActionDefinitionsBuilder(G_FCMP)
649 .legalFor(ST.hasStdExtF(), {{sXLen, f32}})
650 .legalFor(ST.hasStdExtD(), {{sXLen, f64}})
651 .legalFor(ST.hasStdExtZfh(), {{sXLen, f16}})
652 .clampScalar(0, sXLen, sXLen)
653 .minScalar(1, f32)
654 .libcallFor({{sXLen, f32}, {sXLen, f64}})
655 .libcallFor(ST.is64Bit(), {{sXLen, f128}});
656
657 // TODO: Support vector version of G_IS_FPCLASS.
658 getActionDefinitionsBuilder(G_IS_FPCLASS)
659 .customFor(ST.hasStdExtF(), {{s1, f32}})
660 .customFor(ST.hasStdExtD(), {{s1, f64}})
661 .customFor(ST.hasStdExtZfh(), {{s1, f16}})
662 .lower();
663
664 getActionDefinitionsBuilder(G_FCONSTANT)
665 .legalFor(ST.hasStdExtF(), {f32})
666 .legalFor(ST.hasStdExtD(), {f64})
667 .legalFor(ST.hasStdExtZfhmin(), {f16})
668 .lowerFor({f32, f64, f128, LLT::bfloat16()});
669
670 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
671 .legalFor(ST.hasStdExtF(), {{sXLen, f32}})
672 .legalFor(ST.hasStdExtD(), {{sXLen, f64}})
673 .legalFor(ST.hasStdExtZfh(), {{sXLen, f16}})
674 .customFor(ST.is64Bit() && ST.hasStdExtF(), {{s32, f32}})
675 .customFor(ST.is64Bit() && ST.hasStdExtD(), {{s32, f64}})
676 .customFor(ST.is64Bit() && ST.hasStdExtZfh(), {{s32, f16}})
677 .widenScalarToNextPow2(0)
678 .minScalar(0, s32)
679 // The magnitude of a half is at most 65504, so with Zfh use fcvt.w[u].h
680 // and extend the i32 result. Otherwise promote the half source to float
681 // (via fcvt.s.h with Zfhmin, __extendhfsf2 without) and use the float
682 // conversion. On RV32, exclude i64 results here so that they get
683 // narrowed to i32 first.
684 .widenScalarIf(
685 [=, &ST](const LegalityQuery &Query) {
686 return Query.Types[1] == f16 && !ST.hasStdExtZfh() &&
687 (ST.is64Bit() || Query.Types[0].isScalar(32));
688 },
689 changeTo(1, f32))
690 .libcallFor(!ST.hasStdExtZfhmin(), {{s64, f16}})
691 .narrowScalarFor({{s64, f16}}, changeTo(0, s32))
692 .libcallFor({{s32, f32}, {s64, f32}, {s32, f64}, {s64, f64}})
693 .libcallFor(ST.is64Bit(), {{s32, f128}, {s64, f128}}) // FIXME RV32.
694 .libcallFor(ST.is64Bit(), {{s128, f32}, {s128, f64}, {s128, f128}});
695
696 // Unlike G_FPTOSI/G_FPTOUI these need no libcalls for out-of-range values:
697 // the RISC-V conversions already saturate. There is no legalFor() because
698 // they do not produce 0 for NaN, which legalizeCustom fixes up with a
699 // compare and a select. The remaining cases fall back to the generic
700 // lowering, which clamps before converting and only handles scalars.
701 getActionDefinitionsBuilder({G_FPTOSI_SAT, G_FPTOUI_SAT})
702 .customFor(ST.hasStdExtF(), {{sXLen, f32}})
703 .customFor(ST.hasStdExtD(), {{sXLen, f64}})
704 .customFor(ST.hasStdExtZfh(), {{sXLen, f16}})
705 .customFor(ST.is64Bit() && ST.hasStdExtF(), {{s32, f32}})
706 .customFor(ST.is64Bit() && ST.hasStdExtD(), {{s32, f64}})
707 .customFor(ST.is64Bit() && ST.hasStdExtZfh(), {{s32, f16}})
708 // f16 extends exactly to f32; converting via f32 avoids needing rules
709 // for every {sXX, f16} combination.
710 .widenScalarIf(typeIs(1, f16), changeTo(1, f32))
711 .lowerIf(all(isScalar(0), isScalar(1)));
712
713 getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
714 .legalFor(ST.hasStdExtF(), {{sXLen, f32}})
715 .legalFor(ST.hasStdExtD(), {{sXLen, f64}})
716 .legalFor(ST.hasStdExtZfh(), {{sXLen, f16}})
717 .customFor(ST.is64Bit() && ST.hasStdExtF(), {{s32, f32}})
718 .customFor(ST.is64Bit() && ST.hasStdExtD(), {{s32, f64}})
719 .customFor(ST.is64Bit() && ST.hasStdExtZfh(), {{s32, f16}})
720 .minScalar(1, f32)
721 .libcallFor({{s32, f32},
722 {s64, f32},
723 {s32, f64},
724 {s64, f64},
725 {s32, f128},
726 {s64, f128}});
727
728 getActionDefinitionsBuilder({G_INTRINSIC_LRINT, G_INTRINSIC_LLRINT})
729 .legalFor(ST.hasStdExtF(), {{sXLen, f32}})
730 .legalFor(ST.hasStdExtD(), {{sXLen, f64}})
731 .legalFor(ST.hasStdExtZfh(), {{sXLen, f16}})
732 .minScalar(0, sXLen)
733 .minScalar(1, f32)
734 .libcallFor({{s32, f32},
735 {s64, f32},
736 {s32, f64},
737 {s64, f64},
738 {s32, f128},
739 {s64, f128}});
740
741 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
742 .legalFor(ST.hasStdExtF(), {{f32, sXLen}})
743 .legalFor(ST.hasStdExtD(), {{f64, sXLen}})
744 .legalFor(ST.hasStdExtZfh(), {{f16, sXLen}})
745 .widenScalarToNextPow2(1)
746 // Promote to XLen if the operation is legal.
747 .widenScalarIf(
748 [=, &ST](const LegalityQuery &Query) {
749 return Query.Types[0].isScalar() && Query.Types[1].isScalar() &&
750 (Query.Types[1].getSizeInBits() < ST.getXLen()) &&
751 ((ST.hasStdExtF() && Query.Types[0].getSizeInBits() == 32) ||
752 (ST.hasStdExtD() && Query.Types[0].getSizeInBits() == 64) ||
753 (ST.hasStdExtZfh() &&
754 Query.Types[0].getSizeInBits() == 16));
755 },
757 // Otherwise only promote to s32 since we have si libcalls.
758 .minScalar(1, s32)
759 .libcallFor({{f32, s32}, {f64, s32}, {f32, s64}, {f64, s64}})
760 .libcallFor(ST.is64Bit(), {{f128, s32}, {f128, s64}}) // FIXME RV32.
761 .libcallFor(ST.is64Bit(), {{f32, s128}, {f64, s128}, {f128, s128}});
762
763 // FIXME: We can do custom inline expansion like SelectionDAG.
764 getActionDefinitionsBuilder({G_FCEIL, G_FFLOOR, G_FRINT, G_FNEARBYINT,
765 G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
766 G_INTRINSIC_ROUNDEVEN})
767 .legalFor(ST.hasStdExtZfa(), {f32})
768 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {f64})
769 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {f16})
770 .libcallFor({f32, f64})
771 .libcallFor(ST.is64Bit(), {f128});
772
773 getActionDefinitionsBuilder({G_FMAXIMUM, G_FMINIMUM})
774 .legalFor(ST.hasStdExtZfa(), {f32})
775 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtD(), {f64})
776 .legalFor(ST.hasStdExtZfa() && ST.hasStdExtZfh(), {f16});
777
778 getActionDefinitionsBuilder({G_FCOS, G_FSIN, G_FTAN, G_FPOW, G_FLOG, G_FLOG2,
779 G_FLOG10, G_FEXP, G_FEXP2, G_FEXP10, G_FACOS,
780 G_FASIN, G_FATAN, G_FATAN2, G_FCOSH, G_FSINH,
781 G_FTANH, G_FMODF, G_FSINCOS})
782 .libcallFor({f32, f64})
783 .libcallFor(ST.is64Bit(), {f128});
784 getActionDefinitionsBuilder({G_FPOWI, G_FLDEXP})
785 .libcallFor({{f32, s32}, {f64, s32}})
786 .libcallFor(ST.is64Bit(), {f128, s32});
787
788 getActionDefinitionsBuilder(G_FCANONICALIZE)
789 .legalFor(ST.hasStdExtF(), {f32})
790 .legalFor(ST.hasStdExtD(), {f64})
791 .legalFor(ST.hasStdExtZfh(), {f16});
792
793 getActionDefinitionsBuilder(G_VASTART).customFor({p0});
794
795 // va_list must be a pointer, but most sized types are pretty easy to handle
796 // as the destination.
797 getActionDefinitionsBuilder(G_VAARG)
798 // TODO: Implement narrowScalar and widenScalar for G_VAARG for types
799 // other than sXLen.
800 .clampScalar(0, sXLen, sXLen)
801 .lowerForCartesianProduct({sXLen, p0}, {p0});
802
803 getActionDefinitionsBuilder(G_VSCALE)
804 .clampScalar(0, sXLen, sXLen)
805 .customFor({sXLen});
806
807 auto &SplatActions =
808 getActionDefinitionsBuilder(G_SPLAT_VECTOR)
809 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
810 typeIs(1, sXLen)))
811 .customIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST), typeIs(1, s1)));
812 // Handle case of s64 element vectors on RV32. If the subtarget does not have
813 // f64, then try to lower it to G_SPLAT_VECTOR_SPLIT_64_VL. If the subtarget
814 // does have f64, then we don't know whether the type is an f64 or an i64,
815 // so mark the G_SPLAT_VECTOR as legal and decide later what to do with it,
816 // depending on how the instructions it consumes are legalized. They are not
817 // legalized yet since legalization is in reverse postorder, so we cannot
818 // make the decision at this moment.
819 if (XLen == 32) {
820 if (ST.hasVInstructionsF64() && ST.hasStdExtD())
821 SplatActions.legalIf(all(
822 typeInSet(0, {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), typeIs(1, s64)));
823 else if (ST.hasVInstructionsI64())
824 SplatActions.customIf(all(
825 typeInSet(0, {nxv1s64, nxv2s64, nxv4s64, nxv8s64}), typeIs(1, s64)));
826 }
827
828 SplatActions.clampScalar(1, sXLen, sXLen);
829
830 LegalityPredicate ExtractSubvecBitcastPred = [=](const LegalityQuery &Query) {
831 LLT DstTy = Query.Types[0];
832 LLT SrcTy = Query.Types[1];
833 return DstTy.getElementType() == LLT::scalar(1) &&
834 DstTy.getElementCount().getKnownMinValue() >= 8 &&
835 SrcTy.getElementCount().getKnownMinValue() >= 8;
836 };
837 getActionDefinitionsBuilder(G_EXTRACT_SUBVECTOR)
838 // We don't have the ability to slide mask vectors down indexed by their
839 // i1 elements; the smallest we can do is i8. Often we are able to bitcast
840 // to equivalent i8 vectors.
841 .bitcastIf(
842 all(typeIsLegalBoolVec(0, BoolVecTys, ST),
843 typeIsLegalBoolVec(1, BoolVecTys, ST), ExtractSubvecBitcastPred),
844 [=](const LegalityQuery &Query) {
845 LLT CastTy = LLT::vector(
846 Query.Types[0].getElementCount().divideCoefficientBy(8), 8);
847 return std::pair(0, CastTy);
848 })
849 .customIf(LegalityPredicates::any(
850 all(typeIsLegalBoolVec(0, BoolVecTys, ST),
851 typeIsLegalBoolVec(1, BoolVecTys, ST)),
852 all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
853 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST))));
854
855 getActionDefinitionsBuilder(G_INSERT_SUBVECTOR)
856 .customIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST),
857 typeIsLegalBoolVec(1, BoolVecTys, ST)))
858 .customIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
859 typeIsLegalIntOrFPVec(1, IntOrFPVecTys, ST)));
860
861 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
862 .lowerIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(2, p0)));
863
864 getActionDefinitionsBuilder({G_ATOMIC_CMPXCHG, G_ATOMICRMW_ADD,
865 G_ATOMICRMW_XCHG, G_ATOMICRMW_AND,
866 G_ATOMICRMW_OR, G_ATOMICRMW_XOR})
867 .legalFor(ST.hasStdExtA(), {{sXLen, p0}})
868 .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
869 .clampScalar(0, sXLen, sXLen);
870
871 getActionDefinitionsBuilder(G_ATOMICRMW_SUB)
872 .libcallFor(!ST.hasStdExtA(), {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}})
873 .clampScalar(0, sXLen, sXLen)
874 .lower();
875
876 getActionDefinitionsBuilder(
877 {G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, G_ATOMICRMW_UMIN})
878 .legalFor(ST.hasStdExtA(), {{sXLen, p0}})
879 .clampScalar(0, sXLen, sXLen)
880 .unsupported();
881
882 getActionDefinitionsBuilder(G_PREFETCH).legalIf(typeIs(0, p0));
883
884 LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
885 LLT VecTy = Query.Types[0];
886 LLT EltTy = Query.Types[1];
887 return VecTy.getElementType() == EltTy;
888 };
889
890 getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
891 .legalIf(all(typeIsLegalIntOrFPVec(0, IntOrFPVecTys, ST),
892 InsertVectorEltPred, typeIs(2, sXLen)))
893 .legalIf(all(typeIsLegalBoolVec(0, BoolVecTys, ST), InsertVectorEltPred,
894 typeIs(2, sXLen)));
895
896 getActionDefinitionsBuilder({G_INTRINSIC, G_INTRINSIC_W_SIDE_EFFECTS})
897 .alwaysLegal();
898
899 getActionDefinitionsBuilder(G_FENCE).alwaysLegal();
900
901 getActionDefinitionsBuilder({G_TRAP, G_DEBUGTRAP, G_UBSANTRAP}).alwaysLegal();
902
903 verify(*ST.getInstrInfo());
904}
905
907 MachineInstr &MI) const {
908 Intrinsic::ID IntrinsicID = cast<GIntrinsic>(MI).getIntrinsicID();
909
911 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID)) {
912 if (II->hasScalarOperand() && !II->IsFPIntrinsic) {
913 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
914 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
915
916 auto OldScalar = MI.getOperand(II->ScalarOperand + 2).getReg();
917 // Legalize integer vx form intrinsic.
918 if (MRI.getType(OldScalar).isScalar()) {
919 if (MRI.getType(OldScalar).getSizeInBits() < iXLen.getSizeInBits()) {
920 Helper.Observer.changingInstr(MI);
921 Helper.widenScalarSrc(MI, iXLen, II->ScalarOperand + 2,
922 TargetOpcode::G_ANYEXT);
923 Helper.Observer.changedInstr(MI);
924 } else if (MRI.getType(OldScalar).getSizeInBits() >
925 iXLen.getSizeInBits()) {
926 // TODO: i64 in riscv32.
927 return false;
928 }
929 }
930 }
931 return true;
932 }
933
934 switch (IntrinsicID) {
935 default:
936 return false;
937 case Intrinsic::riscv_orc_b: {
938 if (!STI.hasStdExtZbb())
939 return false;
940
941 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
942 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
943 LLT Ty = MRI.getType(MI.getOperand(0).getReg());
944 if (Ty == iXLen)
945 return true;
946 if (!STI.is64Bit() || Ty != LLT::scalar(32))
947 return false;
948
949 // Each output byte depends only on the corresponding input byte.
950 auto Src = MIRBuilder.buildAnyExt(iXLen, MI.getOperand(2));
951 auto Result = MIRBuilder.buildIntrinsic(IntrinsicID, {DstOp(iXLen)})
952 .addUse(Src.getReg(0));
953 MIRBuilder.buildTrunc(MI.getOperand(0), Result);
954 MI.eraseFromParent();
955 return true;
956 }
957 case Intrinsic::riscv_clmulh:
958 Helper.MIRBuilder.buildInstr(TargetOpcode::G_CLMULH, {MI.getOperand(0)},
959 {MI.getOperand(2), MI.getOperand(3)});
960 MI.eraseFromParent();
961 return true;
962 case Intrinsic::riscv_clmulr:
963 Helper.MIRBuilder.buildInstr(TargetOpcode::G_CLMULR, {MI.getOperand(0)},
964 {MI.getOperand(2), MI.getOperand(3)});
965 MI.eraseFromParent();
966 return true;
967 case Intrinsic::vacopy: {
968 // vacopy arguments must be legal because of the intrinsic signature.
969 // No need to check here.
970
971 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
972 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
973 MachineFunction &MF = *MI.getMF();
974 const DataLayout &DL = MIRBuilder.getDataLayout();
975 LLVMContext &Ctx = MF.getFunction().getContext();
976
977 Register DstLst = MI.getOperand(1).getReg();
978 LLT PtrTy = MRI.getType(DstLst);
979
980 // Load the source va_list
981 Align Alignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx));
983 MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, Alignment);
984 auto Tmp = MIRBuilder.buildLoad(PtrTy, MI.getOperand(2), *LoadMMO);
985
986 // Store the result in the destination va_list
989 MIRBuilder.buildStore(Tmp, DstLst, *StoreMMO);
990
991 MI.eraseFromParent();
992 return true;
993 }
994 case Intrinsic::riscv_vsetvli:
995 case Intrinsic::riscv_vsetvlimax:
996 case Intrinsic::riscv_masked_atomicrmw_add:
997 case Intrinsic::riscv_masked_atomicrmw_sub:
998 case Intrinsic::riscv_masked_atomicrmw_xchg:
999 case Intrinsic::riscv_masked_atomicrmw_max:
1000 case Intrinsic::riscv_masked_atomicrmw_min:
1001 case Intrinsic::riscv_masked_atomicrmw_umax:
1002 case Intrinsic::riscv_masked_atomicrmw_umin:
1003 case Intrinsic::riscv_masked_cmpxchg:
1004 return true;
1005 }
1006}
1007
1008bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI,
1009 MachineIRBuilder &MIRBuilder) const {
1010 // Stores the address of the VarArgsFrameIndex slot into the memory location
1011 assert(MI.getOpcode() == TargetOpcode::G_VASTART);
1012 MachineFunction *MF = MI.getParent()->getParent();
1014 int FI = FuncInfo->getVarArgsFrameIndex();
1015 LLT AddrTy = MIRBuilder.getMRI()->getType(MI.getOperand(0).getReg());
1016 auto FINAddr = MIRBuilder.buildFrameIndex(AddrTy, FI);
1017 assert(MI.hasOneMemOperand());
1018 MIRBuilder.buildStore(FINAddr, MI.getOperand(0).getReg(),
1019 *MI.memoperands()[0]);
1020 MI.eraseFromParent();
1021 return true;
1022}
1023
1024bool RISCVLegalizerInfo::legalizeReadCounter(
1025 MachineInstr &MI, MachineIRBuilder &MIRBuilder,
1026 GISelChangeObserver &Observer) const {
1027 assert((MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER ||
1028 MI.getOpcode() == TargetOpcode::G_READSTEADYCOUNTER) &&
1029 "Unexpected opcode");
1030 assert(!STI.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
1031 "has custom type legalization on riscv32");
1032
1033 // On RV32 a 64-bit counter CSR must be read as two 32-bit halves. Because
1034 // the count may wrap between the two reads, re-read the high half and loop
1035 // until the two high reads agree.
1036 int64_t LoCounter, HiCounter;
1037 if (MI.getOpcode() == TargetOpcode::G_READCYCLECOUNTER) {
1038 LoCounter = RISCVSysReg::cycle;
1039 HiCounter = RISCVSysReg::cycleh;
1040 } else {
1041 LoCounter = RISCVSysReg::time;
1042 HiCounter = RISCVSysReg::timeh;
1043 }
1044
1045 MachineBasicBlock *BB = MI.getParent();
1046 MachineFunction &MF = *BB->getParent();
1047 const BasicBlock *LLVMBB = BB->getBasicBlock();
1048 DebugLoc DL = MI.getDebugLoc();
1049 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1050
1051 // Split BB into an entry that falls through into a loop block, and a done
1052 // block that receives the remainder of BB and its original successors.
1053 MachineFunction::iterator It = std::next(BB->getIterator());
1054 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVMBB);
1055 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVMBB);
1056 MF.insert(It, LoopMBB);
1057 MF.insert(It, DoneMBB);
1058
1059 // Splice the instructions after the readcyclecounter into DoneMBB, notifying
1060 // the observer about each moved instruction so CSEInfo stays consistent.
1061 for (MachineBasicBlock::iterator I = std::next(MI.getIterator()),
1062 E = BB->end();
1063 I != E; ++I)
1064 Observer.changingInstr(*I);
1065 DoneMBB->splice(DoneMBB->begin(), BB,
1066 std::next(MachineBasicBlock::iterator(MI)), BB->end());
1067 for (MachineInstr &MovedMI : DoneMBB->instrs())
1068 Observer.changedInstr(MovedMI);
1070 BB->addSuccessor(LoopMBB);
1071
1072 LLT i32 = LLT::integer(32);
1073 // Generic vregs carry the s32 type for G_MERGE_VALUES below, but are also
1074 // constrained to GPR so the target CSRRS/BNE instructions satisfy the
1075 // verifier's register-class constraints.
1076 auto CreateGPR = [&]() {
1078 MRI.setRegClass(R, &RISCV::GPRRegClass);
1079 return R;
1080 };
1081 Register LoReg = CreateGPR();
1082 Register HiReg = CreateGPR();
1083 Register ReadAgainReg = CreateGPR();
1084
1085 // read:
1086 // csrrs HiReg, counterh # high word
1087 // csrrs LoReg, counter # low word
1088 // csrrs ReadAgainReg, counterh
1089 // bne HiReg, ReadAgainReg, read
1090 // Emit the target instructions directly with BuildMI.
1091 const RISCVInstrInfo *TII = STI.getInstrInfo();
1092 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
1093 .addImm(HiCounter)
1094 .addReg(RISCV::X0);
1095 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
1096 .addImm(LoCounter)
1097 .addReg(RISCV::X0);
1098 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
1099 .addImm(HiCounter)
1100 .addReg(RISCV::X0);
1101
1102 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
1103 .addReg(HiReg)
1104 .addReg(ReadAgainReg)
1105 .addMBB(LoopMBB);
1106
1107 LoopMBB->addSuccessor(LoopMBB);
1108 LoopMBB->addSuccessor(DoneMBB);
1109
1110 // Re-pair the two halves into the 64-bit result.
1111 Register DstReg = MI.getOperand(0).getReg();
1112 Observer.erasingInstr(MI);
1113 MI.eraseFromParent();
1114
1115 MIRBuilder.setInsertPt(*DoneMBB, DoneMBB->begin());
1116 MIRBuilder.setDebugLoc(DL);
1117 MIRBuilder.buildMergeValues(DstReg, {LoReg, HiReg});
1118 return true;
1119}
1120
1121bool RISCVLegalizerInfo::legalizeBRJT(MachineInstr &MI,
1122 MachineIRBuilder &MIRBuilder) const {
1123 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1124 auto &MF = *MI.getParent()->getParent();
1125 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
1126 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
1127
1128 Register PtrReg = MI.getOperand(0).getReg();
1129 LLT PtrTy = MRI.getType(PtrReg);
1130 Register IndexReg = MI.getOperand(2).getReg();
1131 LLT IndexTy = MRI.getType(IndexReg);
1132
1133 if (!isPowerOf2_32(EntrySize))
1134 return false;
1135
1136 auto ShiftAmt = MIRBuilder.buildConstant(IndexTy, Log2_32(EntrySize));
1137 IndexReg = MIRBuilder.buildShl(IndexTy, IndexReg, ShiftAmt).getReg(0);
1138
1139 auto Addr = MIRBuilder.buildPtrAdd(PtrTy, PtrReg, IndexReg);
1140
1141 MachineMemOperand *MMO = MF.getMachineMemOperand(
1143 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout())));
1144
1145 Register TargetReg;
1146 switch (MJTI->getEntryKind()) {
1147 default:
1148 return false;
1150 // For PIC, the sequence is:
1151 // BRIND(load(Jumptable + index) + RelocBase)
1152 // RelocBase can be JumpTable, GOT or some sort of global base.
1153 unsigned LoadOpc =
1154 STI.is64Bit() ? TargetOpcode::G_SEXTLOAD : TargetOpcode::G_LOAD;
1155 auto Load = MIRBuilder.buildLoadInstr(LoadOpc, IndexTy, Addr, *MMO);
1156 TargetReg = MIRBuilder.buildPtrAdd(PtrTy, PtrReg, Load).getReg(0);
1157 break;
1158 }
1160 auto Load = MIRBuilder.buildLoadInstr(TargetOpcode::G_SEXTLOAD, IndexTy,
1161 Addr, *MMO);
1162 TargetReg = MIRBuilder.buildIntToPtr(PtrTy, Load).getReg(0);
1163 break;
1164 }
1166 TargetReg = MIRBuilder.buildLoad(PtrTy, Addr, *MMO).getReg(0);
1167 break;
1168 }
1169
1170 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch()) {
1171 // When cf-protection-branch is enabled, we need to use a software
1172 // guarded branch for jump table branch.
1173 MIRBuilder.buildInstr(RISCV::G_SW_GUARDED_BRIND, {}, {TargetReg});
1174 } else {
1175 MIRBuilder.buildBrIndirect(TargetReg);
1176 }
1177
1178 MI.eraseFromParent();
1179 return true;
1180}
1181
1182bool RISCVLegalizerInfo::shouldBeInConstantPool(const APInt &APImm,
1183 bool ShouldOptForSize) const {
1184 assert(APImm.getBitWidth() == 32 || APImm.getBitWidth() == 64);
1185 int64_t Imm = APImm.getSExtValue();
1186 // All simm32 constants should be handled by isel.
1187 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
1188 // this check redundant, but small immediates are common so this check
1189 // should have better compile time.
1190 if (isInt<32>(Imm))
1191 return false;
1192
1193 // We only need to cost the immediate, if constant pool lowering is enabled.
1194 if (!STI.useConstantPoolForLargeInts())
1195 return false;
1196
1198 if (Seq.size() <= STI.getMaxBuildIntsCost())
1199 return false;
1200
1201 // Optimizations below are disabled for opt size. If we're optimizing for
1202 // size, use a constant pool.
1203 if (ShouldOptForSize)
1204 return true;
1205 //
1206 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
1207 // that if it will avoid a constant pool.
1208 // It will require an extra temporary register though.
1209 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
1210 // low and high 32 bits are the same and bit 31 and 63 are set.
1211 unsigned ShiftAmt, AddOpc;
1212 RISCVMatInt::InstSeq SeqLo =
1213 RISCVMatInt::generateTwoRegInstSeq(Imm, STI, ShiftAmt, AddOpc);
1214 return !(!SeqLo.empty() && (SeqLo.size() + 2) <= STI.getMaxBuildIntsCost());
1215}
1216
1217bool RISCVLegalizerInfo::legalizeVScale(MachineInstr &MI,
1218 MachineIRBuilder &MIB) const {
1219 Register Dst = MI.getOperand(0).getReg();
1220
1221 // We define our scalable vector types for lmul=1 to use a 64 bit known
1222 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
1223 // vscale as VLENB / 8.
1224 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
1225 if (STI.getRealMinVLen() < RISCV::RVVBitsPerBlock)
1226 // Support for VLEN==32 is incomplete.
1227 return false;
1228
1229 // We assume VLENB is a multiple of 8. We manually choose the best shift
1230 // here because SimplifyDemandedBits isn't always able to simplify it.
1231 uint64_t Val = MI.getOperand(1).getCImm()->getZExtValue();
1232 if (isPowerOf2_64(Val)) {
1233 uint64_t Log2 = Log2_64(Val);
1234 if (Log2 < 3) {
1235 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {iXLen}, {});
1236 MIB.buildLShr(Dst, VLENB, MIB.buildConstant(iXLen, 3 - Log2),
1238 } else if (Log2 > 3) {
1239 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {iXLen}, {});
1240 MIB.buildShl(Dst, VLENB, MIB.buildConstant(iXLen, Log2 - 3));
1241 } else {
1242 MIB.buildInstr(RISCV::G_READ_VLENB, {Dst}, {});
1243 }
1244 } else if ((Val % 8) == 0) {
1245 // If the multiplier is a multiple of 8, scale it down to avoid needing
1246 // to shift the VLENB value.
1247 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {iXLen}, {});
1248 MIB.buildMul(Dst, VLENB, MIB.buildConstant(iXLen, Val / 8));
1249 } else {
1250 auto VLENB = MIB.buildInstr(RISCV::G_READ_VLENB, {iXLen}, {});
1251 auto VScale = MIB.buildLShr(iXLen, VLENB, MIB.buildConstant(iXLen, 3),
1253 MIB.buildMul(Dst, VScale, MIB.buildConstant(iXLen, Val));
1254 }
1255 MI.eraseFromParent();
1256 return true;
1257}
1258
1259// Custom-lower extensions from mask vectors by using a vselect either with 1
1260// for zero/any-extension or -1 for sign-extension:
1261// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
1262// Note that any-extension is lowered identically to zero-extension.
1263bool RISCVLegalizerInfo::legalizeExt(MachineInstr &MI,
1264 MachineIRBuilder &MIB) const {
1265
1266 unsigned Opc = MI.getOpcode();
1267 assert(Opc == TargetOpcode::G_ZEXT || Opc == TargetOpcode::G_SEXT ||
1268 Opc == TargetOpcode::G_ANYEXT);
1269
1270 MachineRegisterInfo &MRI = *MIB.getMRI();
1271 Register Dst = MI.getOperand(0).getReg();
1272 Register Src = MI.getOperand(1).getReg();
1273
1274 LLT DstTy = MRI.getType(Dst);
1275 int64_t ExtTrueVal = Opc == TargetOpcode::G_SEXT ? -1 : 1;
1276 LLT DstEltTy = DstTy.getElementType();
1277 auto SplatZero = MIB.buildSplatVector(DstTy, MIB.buildConstant(DstEltTy, 0));
1278 auto SplatTrue =
1279 MIB.buildSplatVector(DstTy, MIB.buildConstant(DstEltTy, ExtTrueVal));
1280 MIB.buildSelect(Dst, Src, SplatTrue, SplatZero);
1281
1282 MI.eraseFromParent();
1283 return true;
1284}
1285
1286bool RISCVLegalizerInfo::legalizeLoadStore(MachineInstr &MI,
1287 LegalizerHelper &Helper,
1288 MachineIRBuilder &MIB) const {
1290 "Machine instructions must be Load/Store.");
1291 MachineRegisterInfo &MRI = *MIB.getMRI();
1292 MachineFunction *MF = MI.getMF();
1293 const DataLayout &DL = MIB.getDataLayout();
1294 LLVMContext &Ctx = MF->getFunction().getContext();
1295
1296 Register DstReg = MI.getOperand(0).getReg();
1297 LLT DataTy = MRI.getType(DstReg);
1298 if (!DataTy.isVector())
1299 return false;
1300
1301 if (!MI.hasOneMemOperand())
1302 return false;
1303
1304 MachineMemOperand *MMO = *MI.memoperands_begin();
1305
1306 const auto *TLI = STI.getTargetLowering();
1307 EVT VT = EVT::getEVT(getTypeForLLT(DataTy, Ctx));
1308
1309 if (TLI->allowsMemoryAccessForAlignment(Ctx, DL, VT, *MMO))
1310 return true;
1311
1312 unsigned EltSizeBits = DataTy.getScalarSizeInBits();
1313 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
1314 "Unexpected unaligned RVV load type");
1315
1316 // Calculate the new vector type with i8 elements
1317 unsigned NumElements =
1318 DataTy.getElementCount().getKnownMinValue() * (EltSizeBits / 8);
1319 LLT NewDataTy = LLT::scalable_vector(NumElements, 8);
1320
1321 Helper.bitcast(MI, 0, NewDataTy);
1322
1323 return true;
1324}
1325
1326/// Return the type of the mask type suitable for masking the provided
1327/// vector type. This is simply an i1 element type vector of the same
1328/// (possibly scalable) length.
1329static LLT getMaskTypeFor(LLT VecTy) {
1330 assert(VecTy.isVector());
1331 ElementCount EC = VecTy.getElementCount();
1332 return LLT::vector(EC, LLT::integer(1));
1333}
1334
1335/// Creates an all ones mask suitable for masking a vector of type VecTy with
1336/// vector length VL.
1338 MachineIRBuilder &MIB,
1340 LLT MaskTy = getMaskTypeFor(VecTy);
1341 return MIB.buildInstr(RISCV::G_VMSET_VL, {MaskTy}, {VL});
1342}
1343
1344/// Gets the two common "VL" operands: an all-ones mask and the vector length.
1345/// VecTy is a scalable vector type.
1346static std::pair<MachineInstrBuilder, MachineInstrBuilder>
1348 assert(VecTy.isScalableVector() && "Expecting scalable container type");
1349 const RISCVSubtarget &STI = MIB.getMF().getSubtarget<RISCVSubtarget>();
1350 LLT XLenTy(STI.getXLenVT());
1351 auto VL = MIB.buildConstant(XLenTy, -1);
1352 auto Mask = buildAllOnesMask(VecTy, VL, MIB, MRI);
1353 return {Mask, VL};
1354}
1355
1357buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo,
1358 Register Hi, const SrcOp &VL, MachineIRBuilder &MIB,
1360 // TODO: If the Hi bits of the splat are undefined, then it's fine to just
1361 // splat Lo even if it might be sign extended. I don't think we have
1362 // introduced a case where we're build a s64 where the upper bits are undef
1363 // yet.
1364
1365 // Fall back to a stack store and stride x0 vector load.
1366 // TODO: need to lower G_SPLAT_VECTOR_SPLIT_I64. This is done in
1367 // preprocessDAG in SDAG.
1368 return MIB.buildInstr(RISCV::G_SPLAT_VECTOR_SPLIT_I64_VL, {Dst},
1369 {Passthru, Lo, Hi, VL});
1370}
1371
1373buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru,
1374 const SrcOp &Scalar, const SrcOp &VL,
1376 assert(Scalar.getLLTTy(MRI) == LLT::scalar(64) && "Unexpected VecTy!");
1377 auto Unmerge = MIB.buildUnmerge(LLT::integer(32), Scalar);
1378 return buildSplatPartsS64WithVL(Dst, Passthru, Unmerge.getReg(0),
1379 Unmerge.getReg(1), VL, MIB, MRI);
1380}
1381
1382// Lower splats of s1 types to G_ICMP. For each mask vector type, we have a
1383// legal equivalently-sized i8 type, so we can use that as a go-between.
1384// Splats of s1 types that have constant value can be legalized as VMSET_VL or
1385// VMCLR_VL.
1386bool RISCVLegalizerInfo::legalizeSplatVector(MachineInstr &MI,
1387 MachineIRBuilder &MIB) const {
1388 assert(MI.getOpcode() == TargetOpcode::G_SPLAT_VECTOR);
1389
1390 MachineRegisterInfo &MRI = *MIB.getMRI();
1391
1392 Register Dst = MI.getOperand(0).getReg();
1393 Register SplatVal = MI.getOperand(1).getReg();
1394
1395 LLT VecTy = MRI.getType(Dst);
1396 LLT XLenTy(STI.getXLenVT());
1397
1398 // Handle case of s64 element vectors on rv32
1399 if (XLenTy.getSizeInBits() == 32 &&
1400 VecTy.getElementType().getSizeInBits() == 64) {
1401 auto [_, VL] = buildDefaultVLOps(MRI.getType(Dst), MIB, MRI);
1402 buildSplatSplitS64WithVL(Dst, MIB.buildUndef(VecTy), SplatVal, VL, MIB,
1403 MRI);
1404 MI.eraseFromParent();
1405 return true;
1406 }
1407
1408 // All-zeros or all-ones splats are handled specially.
1409 MachineInstr &SplatValMI = *MRI.getVRegDef(SplatVal);
1410 if (isAllOnesOrAllOnesSplat(SplatValMI, MRI)) {
1411 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1412 MIB.buildInstr(RISCV::G_VMSET_VL, {Dst}, {VL});
1413 MI.eraseFromParent();
1414 return true;
1415 }
1416 if (isNullOrNullSplat(SplatValMI, MRI)) {
1417 auto VL = buildDefaultVLOps(VecTy, MIB, MRI).second;
1418 MIB.buildInstr(RISCV::G_VMCLR_VL, {Dst}, {VL});
1419 MI.eraseFromParent();
1420 return true;
1421 }
1422
1423 // Handle non-constant mask splat (i.e. not sure if it's all zeros or all
1424 // ones) by promoting it to an s8 splat.
1425 LLT InterEltTy = LLT::integer(8);
1426 LLT InterTy = VecTy.changeElementType(InterEltTy);
1427 auto ZExtSplatVal = MIB.buildZExt(InterEltTy, SplatVal);
1428 auto And =
1429 MIB.buildAnd(InterEltTy, ZExtSplatVal, MIB.buildConstant(InterEltTy, 1));
1430 auto LHS = MIB.buildSplatVector(InterTy, And);
1431 auto ZeroSplat =
1432 MIB.buildSplatVector(InterTy, MIB.buildConstant(InterEltTy, 0));
1433 MIB.buildICmp(CmpInst::Predicate::ICMP_NE, Dst, LHS, ZeroSplat);
1434 MI.eraseFromParent();
1435 return true;
1436}
1437
1438static LLT getLMUL1Ty(LLT VecTy) {
1439 assert(VecTy.getElementType().getSizeInBits() <= 64 &&
1440 "Unexpected vector LLT");
1442 VecTy.getElementType().getSizeInBits(),
1443 VecTy.getElementType());
1444}
1445
1446bool RISCVLegalizerInfo::legalizeExtractSubvector(MachineInstr &MI,
1447 MachineIRBuilder &MIB) const {
1448 GExtractSubvector &ES = cast<GExtractSubvector>(MI);
1449
1450 MachineRegisterInfo &MRI = *MIB.getMRI();
1451
1452 Register Dst = ES.getReg(0);
1453 Register Src = ES.getSrcVec();
1454 uint64_t Idx = ES.getIndexImm();
1455
1456 // With an index of 0 this is a cast-like subvector, which can be performed
1457 // with subregister operations.
1458 if (Idx == 0)
1459 return true;
1460
1461 LLT LitTy = MRI.getType(Dst);
1462 LLT BigTy = MRI.getType(Src);
1463
1464 if (LitTy.getElementType() == LLT::scalar(1)) {
1465 // We can't slide this mask vector up indexed by its i1 elements.
1466 // This poses a problem when we wish to insert a scalable vector which
1467 // can't be re-expressed as a larger type. Just choose the slow path and
1468 // extend to a larger type, then truncate back down.
1469 LLT ExtBigTy = BigTy.changeElementType(LLT::integer(8));
1470 LLT ExtLitTy = LitTy.changeElementType(LLT::integer(8));
1471 auto BigZExt = MIB.buildZExt(ExtBigTy, Src);
1472 auto ExtractZExt = MIB.buildExtractSubvector(ExtLitTy, BigZExt, Idx);
1473 auto SplatZero = MIB.buildSplatVector(
1474 ExtLitTy, MIB.buildConstant(ExtLitTy.getElementType(), 0));
1475 MIB.buildICmp(CmpInst::Predicate::ICMP_NE, Dst, ExtractZExt, SplatZero);
1476 MI.eraseFromParent();
1477 return true;
1478 }
1479
1480 // extract_subvector scales the index by vscale if the subvector is scalable,
1481 // and decomposeSubvectorInsertExtractToSubRegs takes this into account.
1482 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1483 MVT LitTyMVT = getMVTForLLT(LitTy);
1484 auto Decompose =
1486 getMVTForLLT(BigTy), LitTyMVT, Idx, TRI);
1487 unsigned RemIdx = Decompose.second;
1488
1489 // If the Idx has been completely eliminated then this is a subvector extract
1490 // which naturally aligns to a vector register. These can easily be handled
1491 // using subregister manipulation.
1492 if (RemIdx == 0)
1493 return true;
1494
1495 // Else LitTy is M1 or smaller and may need to be slid down: if LitTy
1496 // was > M1 then the index would need to be a multiple of VLMAX, and so would
1497 // divide exactly.
1498 assert(
1501
1502 // If the vector type is an LMUL-group type, extract a subvector equal to the
1503 // nearest full vector register type.
1504 LLT InterLitTy = BigTy;
1505 Register Vec = Src;
1507 getLMUL1Ty(BigTy).getSizeInBits())) {
1508 // If BigTy has an LMUL > 1, then LitTy should have a smaller LMUL, and
1509 // we should have successfully decomposed the extract into a subregister.
1510 assert(Decompose.first != RISCV::NoSubRegister);
1511 InterLitTy = getLMUL1Ty(BigTy);
1512 // SDAG builds a TargetExtractSubreg. We cannot create a a Copy with SubReg
1513 // specified on the source Register (the equivalent) since generic virtual
1514 // register does not allow subregister index.
1515 Vec = MIB.buildExtractSubvector(InterLitTy, Src, Idx - RemIdx).getReg(0);
1516 }
1517
1518 // Slide this vector register down by the desired number of elements in order
1519 // to place the desired subvector starting at element 0.
1520 const LLT XLenTy(STI.getXLenVT());
1521 auto SlidedownAmt = MIB.buildVScale(XLenTy, RemIdx);
1522 auto [Mask, VL] = buildDefaultVLOps(InterLitTy, MIB, MRI);
1524 auto Slidedown = MIB.buildInstr(
1525 RISCV::G_VSLIDEDOWN_VL, {InterLitTy},
1526 {MIB.buildUndef(InterLitTy), Vec, SlidedownAmt, Mask, VL, Policy});
1527
1528 // Now the vector is in the right position, extract our final subvector. This
1529 // should resolve to a COPY.
1530 MIB.buildExtractSubvector(Dst, Slidedown, 0);
1531
1532 MI.eraseFromParent();
1533 return true;
1534}
1535
1536bool RISCVLegalizerInfo::legalizeInsertSubvector(MachineInstr &MI,
1537 LegalizerHelper &Helper,
1538 MachineIRBuilder &MIB) const {
1539 GInsertSubvector &IS = cast<GInsertSubvector>(MI);
1540
1541 MachineRegisterInfo &MRI = *MIB.getMRI();
1542
1543 Register Dst = IS.getReg(0);
1544 Register BigVec = IS.getBigVec();
1545 Register LitVec = IS.getSubVec();
1546 uint64_t Idx = IS.getIndexImm();
1547
1548 LLT BigTy = MRI.getType(BigVec);
1549 LLT LitTy = MRI.getType(LitVec);
1550
1551 if (Idx == 0 && mi_match(BigVec, MRI, m_GImplicitDef()))
1552 return true;
1553
1554 // We don't have the ability to slide mask vectors up indexed by their i1
1555 // elements; the smallest we can do is i8. Often we are able to bitcast to
1556 // equivalent i8 vectors. Otherwise, we can must zeroextend to equivalent i8
1557 // vectors and truncate down after the insert.
1558 if (LitTy.getElementType() == LLT::scalar(1)) {
1559 auto BigTyMinElts = BigTy.getElementCount().getKnownMinValue();
1560 auto LitTyMinElts = LitTy.getElementCount().getKnownMinValue();
1561 if (BigTyMinElts >= 8 && LitTyMinElts >= 8)
1562 return Helper.bitcast(
1563 IS, 0,
1565
1566 // We can't slide this mask vector up indexed by its i1 elements.
1567 // This poses a problem when we wish to insert a scalable vector which
1568 // can't be re-expressed as a larger type. Just choose the slow path and
1569 // extend to a larger type, then truncate back down.
1570 LLT ExtBigTy = BigTy.changeElementType(LLT::integer(8));
1571 return Helper.widenScalar(IS, 0, ExtBigTy);
1572 }
1573
1574 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
1575 unsigned SubRegIdx, RemIdx;
1576 std::tie(SubRegIdx, RemIdx) =
1578 getMVTForLLT(BigTy), getMVTForLLT(LitTy), Idx, TRI);
1579
1580 TypeSize VecRegSize = TypeSize::getScalable(RISCV::RVVBitsPerBlock);
1582 STI.expandVScale(LitTy.getSizeInBits()).getKnownMinValue()));
1583 bool ExactlyVecRegSized =
1584 STI.expandVScale(LitTy.getSizeInBits())
1585 .isKnownMultipleOf(STI.expandVScale(VecRegSize));
1586
1587 // If the Idx has been completely eliminated and this subvector's size is a
1588 // vector register or a multiple thereof, or the surrounding elements are
1589 // undef, then this is a subvector insert which naturally aligns to a vector
1590 // register. These can easily be handled using subregister manipulation.
1591 if (RemIdx == 0 && ExactlyVecRegSized)
1592 return true;
1593
1594 // If the subvector is smaller than a vector register, then the insertion
1595 // must preserve the undisturbed elements of the register. We do this by
1596 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
1597 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
1598 // subvector within the vector register, and an INSERT_SUBVECTOR of that
1599 // LMUL=1 type back into the larger vector (resolving to another subregister
1600 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
1601 // to avoid allocating a large register group to hold our subvector.
1602
1603 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
1604 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
1605 // (in our case undisturbed). This means we can set up a subvector insertion
1606 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
1607 // size of the subvector.
1608 const LLT XLenTy(STI.getXLenVT());
1609 LLT InterLitTy = BigTy;
1610 Register AlignedExtract = BigVec;
1611 unsigned AlignedIdx = Idx - RemIdx;
1613 getLMUL1Ty(BigTy).getSizeInBits())) {
1614 InterLitTy = getLMUL1Ty(BigTy);
1615 // Extract a subvector equal to the nearest full vector register type. This
1616 // should resolve to a G_EXTRACT on a subreg.
1617 AlignedExtract =
1618 MIB.buildExtractSubvector(InterLitTy, BigVec, AlignedIdx).getReg(0);
1619 }
1620
1621 auto Insert = MIB.buildInsertSubvector(InterLitTy, MIB.buildUndef(InterLitTy),
1622 LitVec, 0);
1623
1624 auto [Mask, _] = buildDefaultVLOps(InterLitTy, MIB, MRI);
1625 auto VL = MIB.buildVScale(XLenTy, LitTy.getElementCount().getKnownMinValue());
1626
1627 // If we're inserting into the lowest elements, use a tail undisturbed
1628 // vmv.v.v.
1629 MachineInstrBuilder Inserted;
1630 bool NeedInsertSubvec =
1631 TypeSize::isKnownGT(BigTy.getSizeInBits(), InterLitTy.getSizeInBits());
1632 Register InsertedDst =
1633 NeedInsertSubvec ? MRI.createGenericVirtualRegister(InterLitTy) : Dst;
1634 if (RemIdx == 0) {
1635 Inserted = MIB.buildInstr(RISCV::G_VMV_V_V_VL, {InsertedDst},
1636 {AlignedExtract, Insert, VL});
1637 } else {
1638 auto SlideupAmt = MIB.buildVScale(XLenTy, RemIdx);
1639 // Construct the vector length corresponding to RemIdx + length(LitTy).
1640 VL = MIB.buildAdd(XLenTy, SlideupAmt, VL);
1641 // Use tail agnostic policy if we're inserting over InterLitTy's tail.
1642 ElementCount EndIndex =
1645 if (STI.expandVScale(EndIndex) ==
1646 STI.expandVScale(InterLitTy.getElementCount()))
1648
1649 Inserted =
1650 MIB.buildInstr(RISCV::G_VSLIDEUP_VL, {InsertedDst},
1651 {AlignedExtract, Insert, SlideupAmt, Mask, VL, Policy});
1652 }
1653
1654 // If required, insert this subvector back into the correct vector register.
1655 // This should resolve to an INSERT_SUBREG instruction.
1656 if (NeedInsertSubvec)
1657 MIB.buildInsertSubvector(Dst, BigVec, Inserted, AlignedIdx);
1658
1659 MI.eraseFromParent();
1660 return true;
1661}
1662
1663bool RISCVLegalizerInfo::legalizeBitreverse(MachineInstr &MI,
1664 MachineIRBuilder &MIB) const {
1665 assert(MI.getOpcode() == TargetOpcode::G_BITREVERSE && "Unexpected opcode");
1666
1667 if (!STI.hasStdExtZbkb())
1668 return false;
1669
1670 MachineRegisterInfo &MRI = *MIB.getMRI();
1671
1672 Register Dst = MI.getOperand(0).getReg();
1673 Register Src = MI.getOperand(1).getReg();
1674
1675 if (!MRI.getType(Dst).isScalar(8))
1676 return false;
1677
1678 auto WideSrc = MIB.buildAnyExt(iXLen, Src);
1679 auto Brev = MIB.buildInstr(RISCV::G_BREV8, {iXLen}, {WideSrc.getReg(0)});
1680 MIB.buildTrunc(Dst, Brev.getReg(0));
1681
1682 MI.eraseFromParent();
1683 return true;
1684}
1685
1686static unsigned getRISCVWOpcode(unsigned Opcode) {
1687 switch (Opcode) {
1688 default:
1689 llvm_unreachable("Unexpected opcode");
1690 case TargetOpcode::G_ASHR:
1691 return RISCV::G_SRAW;
1692 case TargetOpcode::G_LSHR:
1693 return RISCV::G_SRLW;
1694 case TargetOpcode::G_SHL:
1695 return RISCV::G_SLLW;
1696 case TargetOpcode::G_SDIV:
1697 return RISCV::G_DIVW;
1698 case TargetOpcode::G_UDIV:
1699 return RISCV::G_DIVUW;
1700 case TargetOpcode::G_UREM:
1701 return RISCV::G_REMUW;
1702 case TargetOpcode::G_ROTL:
1703 return RISCV::G_ROLW;
1704 case TargetOpcode::G_ROTR:
1705 return RISCV::G_RORW;
1706 case TargetOpcode::G_CTLZ:
1707 return RISCV::G_CLZW;
1708 case TargetOpcode::G_CTTZ:
1709 return RISCV::G_CTZW;
1710 case TargetOpcode::G_CTLS:
1711 return RISCV::G_CLSW;
1712 case TargetOpcode::G_FPTOSI:
1713 return RISCV::G_FCVT_W_RV64;
1714 case TargetOpcode::G_FPTOUI:
1715 return RISCV::G_FCVT_WU_RV64;
1716 }
1717}
1718
1721 LostDebugLocObserver &LocObserver) const {
1722 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1723 MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1724 MachineFunction &MF = *MI.getParent()->getParent();
1725 switch (MI.getOpcode()) {
1726 default:
1727 // No idea what to do.
1728 return false;
1729 case TargetOpcode::G_ABS:
1730 return Helper.lowerAbsToMaxNeg(MI);
1731 case TargetOpcode::G_CLMULH:
1732 case TargetOpcode::G_CLMULR: {
1733 assert(STI.is64Bit() &&
1734 MRI.getType(MI.getOperand(0).getReg()) == LLT::scalar(32) &&
1735 "Unexpected custom legalization");
1736 // Shift both inputs by 32 so the full product has 64 trailing zeros.
1737 // Perform CLMULH or CLMULR on the shifted inputs, then extract the upper
1738 // 32 bits of the result.
1739 auto Shift = MIRBuilder.buildConstant(iXLen, 32);
1740 auto LHS = MIRBuilder.buildAnyExt(iXLen, MI.getOperand(1));
1741 auto RHS = MIRBuilder.buildAnyExt(iXLen, MI.getOperand(2));
1742 auto ShiftedLHS = MIRBuilder.buildShl(iXLen, LHS, Shift);
1743 auto ShiftedRHS = MIRBuilder.buildShl(iXLen, RHS, Shift);
1744 auto Product = MIRBuilder.buildInstr(MI.getOpcode(), {iXLen},
1745 {ShiftedLHS, ShiftedRHS});
1746 auto High = MIRBuilder.buildLShr(iXLen, Product, Shift);
1747 MIRBuilder.buildTrunc(MI.getOperand(0), High);
1748 MI.eraseFromParent();
1749 return true;
1750 }
1751 case TargetOpcode::G_CONSTANT: {
1752 const Function &F = MF.getFunction();
1753 // TODO: if PSI and BFI are present, add " ||
1754 // llvm::shouldOptForSize(*CurMBB, PSI, BFI)".
1755 bool ShouldOptForSize = F.hasOptSize();
1756 const ConstantInt *ConstVal = MI.getOperand(1).getCImm();
1757 if (!shouldBeInConstantPool(ConstVal->getValue(), ShouldOptForSize))
1758 return true;
1759 return Helper.lowerConstant(MI);
1760 }
1761 case TargetOpcode::G_SUB:
1762 case TargetOpcode::G_ADD: {
1763 Helper.Observer.changingInstr(MI);
1764 // The G_ANYEXTs below leave the new high bits unconstrained, so no-wrap
1765 // claims proved at the narrow width no longer hold.
1767 Helper.widenScalarSrc(MI, iXLen, 1, TargetOpcode::G_ANYEXT);
1768 Helper.widenScalarSrc(MI, iXLen, 2, TargetOpcode::G_ANYEXT);
1769
1770 Register DstALU = MRI.createGenericVirtualRegister(iXLen);
1771
1772 MachineOperand &MO = MI.getOperand(0);
1773 MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt());
1774 auto DstSext = MIRBuilder.buildSExtInReg(iXLen, DstALU, 32);
1775
1776 MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {MO}, {DstSext});
1777 MO.setReg(DstALU);
1778
1779 Helper.Observer.changedInstr(MI);
1780 return true;
1781 }
1782 case TargetOpcode::G_ASHR:
1783 case TargetOpcode::G_LSHR:
1784 case TargetOpcode::G_SHL: {
1785 if (getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI)) {
1786 // We don't need a custom node for shift by constant. Just widen the
1787 // source and the shift amount.
1788 unsigned ExtOpc = TargetOpcode::G_ANYEXT;
1789 if (MI.getOpcode() == TargetOpcode::G_ASHR)
1790 ExtOpc = TargetOpcode::G_SEXT;
1791 else if (MI.getOpcode() == TargetOpcode::G_LSHR)
1792 ExtOpc = TargetOpcode::G_ZEXT;
1793
1794 Helper.Observer.changingInstr(MI);
1795 // The G_ANYEXT leave the new high bits unconstrained, so no-wrap
1796 // claims proved at the narrow width no longer hold.
1797 if (ExtOpc == TargetOpcode::G_ANYEXT)
1799 Helper.widenScalarSrc(MI, iXLen, 1, ExtOpc);
1800 Helper.widenScalarSrc(MI, iXLen, 2, TargetOpcode::G_ZEXT);
1801 Helper.widenScalarDst(MI, iXLen);
1802 Helper.Observer.changedInstr(MI);
1803 return true;
1804 }
1805
1806 Helper.Observer.changingInstr(MI);
1807 Helper.widenScalarSrc(MI, iXLen, 1, TargetOpcode::G_ANYEXT);
1808 Helper.widenScalarSrc(MI, iXLen, 2, TargetOpcode::G_ANYEXT);
1809 Helper.widenScalarDst(MI, iXLen);
1810 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));
1811 Helper.Observer.changedInstr(MI);
1812 return true;
1813 }
1814 case TargetOpcode::G_SDIV:
1815 case TargetOpcode::G_UDIV:
1816 case TargetOpcode::G_UREM:
1817 case TargetOpcode::G_ROTL:
1818 case TargetOpcode::G_ROTR: {
1819 Helper.Observer.changingInstr(MI);
1820 Helper.widenScalarSrc(MI, iXLen, 1, TargetOpcode::G_ANYEXT);
1821 Helper.widenScalarSrc(MI, iXLen, 2, TargetOpcode::G_ANYEXT);
1822 Helper.widenScalarDst(MI, iXLen);
1823 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));
1824 Helper.Observer.changedInstr(MI);
1825 return true;
1826 }
1827 case TargetOpcode::G_CTLZ:
1828 case TargetOpcode::G_CTTZ:
1829 case TargetOpcode::G_CTLS: {
1830 Helper.Observer.changingInstr(MI);
1831 Helper.widenScalarSrc(MI, iXLen, 1, TargetOpcode::G_ANYEXT);
1832 Helper.widenScalarDst(MI, iXLen);
1833 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));
1834 Helper.Observer.changedInstr(MI);
1835 return true;
1836 }
1837 case TargetOpcode::G_FPTOSI:
1838 case TargetOpcode::G_FPTOUI: {
1839 Helper.Observer.changingInstr(MI);
1840 Helper.widenScalarDst(MI, iXLen);
1841 MI.setDesc(MIRBuilder.getTII().get(getRISCVWOpcode(MI.getOpcode())));
1843 Helper.Observer.changedInstr(MI);
1844 return true;
1845 }
1846 case TargetOpcode::G_FPTOSI_SAT:
1847 case TargetOpcode::G_FPTOUI_SAT: {
1848 // Mirror lowerFP_TO_INT_SAT (RISCVISelLowering.cpp): the conversion
1849 // saturates out-of-range values, but we need to select 0 for NaN as
1850 // required by the intrinsics.
1851 Register Dst = MI.getOperand(0).getReg();
1852 Register Src = MI.getOperand(1).getReg();
1853 LLT DstTy = MRI.getType(Dst);
1854 unsigned DstBits = DstTy.getScalarSizeInBits();
1855 bool IsSigned = MI.getOpcode() == TargetOpcode::G_FPTOSI_SAT;
1856
1857 unsigned FCvtOp;
1858 if (DstBits == XLen) {
1859 FCvtOp = IsSigned ? RISCV::G_FCVT_X : RISCV::G_FCVT_XU;
1860 } else {
1861 assert(STI.is64Bit() && "Non-XLen results are only custom on RV64");
1862 FCvtOp = IsSigned ? RISCV::G_FCVT_W_RV64 : RISCV::G_FCVT_WU_RV64;
1863 }
1864
1865 auto FCvt = MIRBuilder.buildInstr(FCvtOp, {iXLen}, {Src})
1866 .addImm(RISCVFPRndMode::RTZ);
1867
1868 // RISC-V G_FCMP results are XLen wide, and so are G_SELECT conditions.
1869 auto IsNaN = MIRBuilder.buildFCmp(CmpInst::FCMP_UNO, iXLen, Src, Src);
1870
1871 Register FCvtReg = FCvt.getReg(0);
1872 if (DstBits != XLen)
1873 FCvtReg = MIRBuilder.buildTrunc(DstTy, FCvtReg).getReg(0);
1874
1875 MIRBuilder.buildSelect(Dst, IsNaN, MIRBuilder.buildConstant(DstTy, 0),
1876 FCvtReg);
1877
1878 MI.eraseFromParent();
1879 return true;
1880 }
1881 case TargetOpcode::G_LROUND: {
1882 // The (i32 any_lround) Pat is IsRV32-only; on RV64 lower to
1883 // riscv_fcvt_w_rv64 with FRM_RMM.
1884 Helper.Observer.changingInstr(MI);
1885 Helper.widenScalarDst(MI, iXLen);
1886 MI.setDesc(MIRBuilder.getTII().get(RISCV::G_FCVT_W_RV64));
1888 Helper.Observer.changedInstr(MI);
1889 return true;
1890 }
1891 case TargetOpcode::G_READCYCLECOUNTER:
1892 case TargetOpcode::G_READSTEADYCOUNTER:
1893 return legalizeReadCounter(MI, MIRBuilder, Helper.Observer);
1894 case TargetOpcode::G_IS_FPCLASS: {
1895 Register GISFPCLASS = MI.getOperand(0).getReg();
1896 Register Src = MI.getOperand(1).getReg();
1897 const MachineOperand &ImmOp = MI.getOperand(2);
1898 MachineIRBuilder MIB(MI);
1899
1900 // Turn LLVM IR's floating point classes to that in RISC-V,
1901 // by simply rotating the 10-bit immediate right by two bits.
1902 APInt GFpClassImm(10, static_cast<uint64_t>(ImmOp.getImm()));
1903 auto FClassMask = MIB.buildConstant(iXLen, GFpClassImm.rotr(2).zext(XLen));
1904 auto ConstZero = MIB.buildConstant(iXLen, 0);
1905
1906 auto GFClass = MIB.buildInstr(RISCV::G_FCLASS, {iXLen}, {Src});
1907 auto And = MIB.buildAnd(iXLen, GFClass, FClassMask);
1908 MIB.buildICmp(CmpInst::ICMP_NE, GISFPCLASS, And, ConstZero);
1909
1910 MI.eraseFromParent();
1911 return true;
1912 }
1913 case TargetOpcode::G_BRJT:
1914 return legalizeBRJT(MI, MIRBuilder);
1915 case TargetOpcode::G_VASTART:
1916 return legalizeVAStart(MI, MIRBuilder);
1917 case TargetOpcode::G_VSCALE:
1918 return legalizeVScale(MI, MIRBuilder);
1919 case TargetOpcode::G_ZEXT:
1920 case TargetOpcode::G_SEXT:
1921 case TargetOpcode::G_ANYEXT:
1922 return legalizeExt(MI, MIRBuilder);
1923 case TargetOpcode::G_SPLAT_VECTOR:
1924 return legalizeSplatVector(MI, MIRBuilder);
1925 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1926 return legalizeExtractSubvector(MI, MIRBuilder);
1927 case TargetOpcode::G_INSERT_SUBVECTOR:
1928 return legalizeInsertSubvector(MI, Helper, MIRBuilder);
1929 case TargetOpcode::G_BITREVERSE:
1930 return legalizeBitreverse(MI, MIRBuilder);
1931 case TargetOpcode::G_LOAD:
1932 case TargetOpcode::G_STORE:
1933 return legalizeLoadStore(MI, Helper, MIRBuilder);
1934 }
1935
1936 llvm_unreachable("expected switch to return");
1937}
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
bpf unwind lower
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
#define _
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
uint64_t High
uint64_t IntrinsicInst * II
#define P(N)
ppc ctr loops verify
MachineRegisterInfo & MRI
static LLT getLMUL1Ty(LLT VecTy)
static MachineInstrBuilder buildAllOnesMask(LLT VecTy, const SrcOp &VL, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static std::pair< MachineInstrBuilder, MachineInstrBuilder > buildDefaultVLOps(LLT VecTy, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
Gets the two common "VL" operands: an all-ones mask and the vector length.
static LegalityPredicate typeIsLegalBoolVec(unsigned TypeIdx, std::initializer_list< LLT > BoolVecTys, const RISCVSubtarget &ST)
static MachineInstrBuilder buildSplatSplitS64WithVL(const DstOp &Dst, const SrcOp &Passthru, const SrcOp &Scalar, const SrcOp &VL, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
static LegalityPredicate typeIsLegalIntOrFPVec(unsigned TypeIdx, std::initializer_list< LLT > IntOrFPVecTys, const RISCVSubtarget &ST)
static MachineInstrBuilder buildSplatPartsS64WithVL(const DstOp &Dst, const SrcOp &Passthru, Register Lo, Register Hi, const SrcOp &VL, MachineIRBuilder &MIB, MachineRegisterInfo &MRI)
static LLT getMaskTypeFor(LLT VecTy)
Return the type of the mask type suitable for masking the provided vector type.
static LegalityPredicate typeIsLegalPtrVec(unsigned TypeIdx, std::initializer_list< LLT > PtrVecTys, const RISCVSubtarget &ST)
static unsigned getRISCVWOpcode(unsigned Opcode)
This file declares the targeting of the Machinelegalizer class for RISC-V.
MachineInstr *& Lo
Value * LHS
Class for arbitrary precision integers.
Definition APInt.h:78
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
LLVM_ABI APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
Definition APInt.cpp:1199
int64_t getSExtValue() const
Get sign extended value.
Definition APInt.h:1582
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
This is the shared class of boolean and integer constants.
Definition Constants.h:87
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition Constants.h:159
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
static constexpr ElementCount getScalable(ScalarTy MinVal)
Definition TypeSize.h:309
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
virtual void erasingInstr(MachineInstr &MI)=0
An instruction is about to be erased.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static constexpr LLT float64()
Get a 64-bit IEEE double value.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalable_vector(unsigned MinNumElements, unsigned ScalarSizeInBits)
Get a low-level scalable vector of some number of elements and element width.
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
static constexpr LLT float128()
Get a 128-bit IEEE quad value.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr ElementCount getElementCount() const
static constexpr LLT float16()
Get a 16-bit IEEE half value.
static LLT integer(unsigned SizeInBits)
static constexpr LLT bfloat16()
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
static constexpr LLT float32()
Get a 32-bit IEEE float value.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & alwaysLegal()
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LLVM_ABI void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode)
Legalize a single operand OpIdx of the machine instruction MI as a Use by extending the operand's typ...
LLVM_ABI LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI)
LLVM_ABI LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
LLVM_ABI LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LLVM_ABI LegalizeResult lowerConstant(MachineInstr &MI)
LLVM_ABI void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
Definition MCInstrInfo.h:89
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildInsertSubvector(const DstOp &Res, const SrcOp &Src0, const SrcOp &Src1, unsigned Index)
Build and insert Res = G_INSERT_SUBVECTOR Src0, Src1, Idx.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ZEXT Op.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts)
Build and insert Res = G_VSCALE MinElts.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildMergeValues(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_TRUNC Op.
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildExtractSubvector(const DstOp &Res, const SrcOp &Src, unsigned Index)
Build and insert Res = G_EXTRACT_SUBVECTOR Src, Idx0.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildBrIndirect(Register Tgt)
Build and insert G_BRINDIRECT Tgt.
MachineInstrBuilder buildSplatVector(const DstOp &Res, const SrcOp &Val)
Build and insert Res = G_SPLAT_VECTOR Val.
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
LLVM_ABI unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
LLVM_ABI unsigned getEntryAlignment(const DataLayout &TD) const
getEntryAlignment - Return the alignment of each entry in the jump table.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
LLVM_ABI void setReg(Register Reg)
Change the register this operand corresponds to.
static MachineOperand CreateImm(int64_t Val)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI LLVM_READONLY MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
LLVM_ABI void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
RISCVLegalizerInfo(const RISCVSubtarget &ST)
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
static RISCVVType::VLMUL getLMUL(MVT VT)
Wrapper class representing virtual and physical registers.
Definition Register.h:20
Register getReg() const
bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
This function returns true if the memory access is aligned or if the target allows this specific unal...
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition TypeSize.h:166
static constexpr bool isKnownGT(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition TypeSize.h:224
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition TypeSize.h:253
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
@ BasicBlock
Various leaf nodes.
Definition ISDOpcodes.h:83
LLVM_ABI LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LLVM_ABI LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LLVM_ABI LegalityPredicate immIs(unsigned ImmIdx, int64_t Imm)
True iff the immediate at the given index has the specified value.
Predicate any(Predicate P0, Predicate P1)
True iff P0 or P1 are true.
LLVM_ABI LegalityPredicate sizeIs(unsigned TypeIdx, unsigned Size)
True if the total bitwidth of the specified type index is Size bits.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LLVM_ABI LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
LLVM_ABI LegalityPredicate scalarWiderThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's wider than the given size.
LLVM_ABI LegalityPredicate immInSet(unsigned ImmIdx, std::initializer_list< int64_t > ImmsInit)
True iff the immediate at the given index has one of the specified values.
LLVM_ABI LegalityPredicate scalarNarrowerThan(unsigned TypeIdx, unsigned Size)
True iff the specified type index is a scalar that's narrower than the given size.
LLVM_ABI LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
ImplicitDefMatch m_GImplicitDef()
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
InstSeq generateTwoRegInstSeq(int64_t Val, const MCSubtargetInfo &STI, unsigned &ShiftAmt, unsigned &AddOpc)
SmallVector< Inst, 8 > InstSeq
Definition RISCVMatInt.h:43
LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)
static constexpr unsigned RVVBitsPerBlock
Invariant opcodes: All instruction sets have these as their low opcodes.
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Get the type back from LLT.
Definition Utils.cpp:2140
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition Utils.cpp:1717
@ Load
The value being inserted comes from a load (InsertElement only).
LLVM_ABI MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition Utils.cpp:1699
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
std::function< bool(const LegalityQuery &)> LegalityPredicate
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition Alignment.h:202
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
static LLVM_ABI EVT getEVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< LLT > Types
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.