LLVM 24.0.0git
RISCVInstructionSelector.cpp
Go to the documentation of this file.
1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46
47 void setupMF(MachineFunction &MF, GISelValueTracking *VT,
48 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
49 BlockFrequencyInfo *BFI) override {
50 InstructionSelector::setupMF(MF, VT, CoverageInfo, PSI, BFI);
51 MRI = &MF.getRegInfo();
52 }
53
54 static const char *getName() { return DEBUG_TYPE; }
55
56private:
57 static constexpr unsigned MaxRecursionDepth = 6;
58
59 bool hasAllNBitUsers(const MachineInstr &MI, unsigned Bits,
60 const unsigned Depth = 0) const;
61 bool hasAllHUsers(const MachineInstr &MI) const {
62 return hasAllNBitUsers(MI, 16);
63 }
64 bool hasAllWUsers(const MachineInstr &MI) const {
65 return hasAllNBitUsers(MI, 32);
66 }
67
68 bool isRegInGprb(Register Reg) const;
69 bool isRegInFprb(Register Reg) const;
70 bool isWorthFoldingAdd(Register AddResult) const;
71
72 // tblgen-erated 'select' implementation, used as the initial selector for
73 // the patterns that don't require complex C++.
74 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
75
76 // A lowering phase that runs before any selection attempts.
77 // Returns true if the instruction was modified.
78 void preISelLower(MachineInstr &MI);
79
80 bool replacePtrWithInt(MachineInstr &MI, unsigned OpIdx);
81
82 // Custom selection methods
83 bool selectCopy(MachineInstr &MI) const;
84 bool selectImplicitDef(MachineInstr &MI) const;
85 bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
86 // Emit a constant-materialization instruction sequence.
87 bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
88 MachineInstr &MI) const;
89 bool selectAddr(MachineInstr &MI, bool IsLocal = true,
90 bool IsExternWeak = false) const;
91 bool selectSelect(MachineInstr &MI) const;
92 bool selectFPCompare(MachineInstr &MI) const;
93 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
94 MachineInstr &MI) const;
97 void addVectorLoadStoreOperands(MachineInstr &I,
99 unsigned &CurOp, bool IsMasked,
100 bool IsStridedOrIndexed,
101 LLT *IndexVT = nullptr) const;
102 bool selectIntrinsicWithSideEffects(MachineInstr &I) const;
103 bool selectIntrinsic(MachineInstr &I) const;
104 bool selectExtractSubvector(MachineInstr &MI) const;
105 bool selectInsertSubVector(MachineInstr &I) const;
106 ComplexRendererFns selectShiftMask(MachineOperand &Root,
107 unsigned ShiftWidth) const;
108 ComplexRendererFns selectShiftMaskXLen(MachineOperand &Root) const {
109 return selectShiftMask(Root, STI.getXLen());
110 }
111 ComplexRendererFns selectShiftMask32(MachineOperand &Root) const {
112 return selectShiftMask(Root, 32);
113 }
114 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
115 ComplexRendererFns selectBrindRegImm(MachineOperand &Root) const;
116 ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
117
118 // Plan for materializing a constant address as (Hi materialization, Lo12
119 // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
120 // a multiple of 32.
121 struct ConstAddrPlan {
122 enum { X0, LUI, InstSeq } Kind = X0;
123 int64_t Hi20 = 0;
125 int64_t Lo12 = 0;
126 };
127 ComplexRendererFns computeConstAddr(int64_t CVal, bool IsPrefetch,
128 Register OrigBase) const;
129 // Materialize the high part of Plan into a register. If OrigBase is valid,
130 // ADD it to the materialized high part (for G_PTR_ADD + large constant).
131 Register materializeConstBase(MachineInstrBuilder &MIB,
132 const ConstAddrPlan &Plan,
133 Register OrigBase) const;
134
135 ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
136 template <unsigned Bits>
137 ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
138 return selectSExtBits(Root, Bits);
139 }
140
141 ComplexRendererFns selectZExtBits(MachineOperand &Root, unsigned Bits) const;
142 template <unsigned Bits>
143 ComplexRendererFns selectZExtBits(MachineOperand &Root) const {
144 return selectZExtBits(Root, Bits);
145 }
146
147 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
148 template <unsigned ShAmt>
149 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
150 return selectSHXADDOp(Root, ShAmt);
151 }
152
153 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
154 unsigned ShAmt) const;
155 template <unsigned ShAmt>
156 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
157 return selectSHXADD_UWOp(Root, ShAmt);
158 }
159
160 ComplexRendererFns renderVLOp(MachineOperand &Root) const;
161 ComplexRendererFns renderAddiPair(Register BaseReg, int64_t AddiImm,
162 int64_t OffsetImm) const;
163 // Custom renderers for tablegen
164 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
165 int OpIdx) const;
166 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
167 int OpIdx) const;
168 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
169 int OpIdx) const;
170 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
171 int OpIdx) const;
172
173 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
174 int OpIdx) const;
175 void renderXLenSubTrailingOnes(MachineInstrBuilder &MIB,
176 const MachineInstr &MI, int OpIdx) const;
177
178 void renderAddiPairImmLarge(MachineInstrBuilder &MIB, const MachineInstr &MI,
179 int OpIdx) const;
180 void renderAddiPairImmSmall(MachineInstrBuilder &MIB, const MachineInstr &MI,
181 int OpIdx) const;
182
183 const RISCVSubtarget &STI;
184 const RISCVInstrInfo &TII;
185 const RISCVRegisterInfo &TRI;
186 const RISCVRegisterBankInfo &RBI;
187 const RISCVTargetMachine &TM;
188
189 MachineRegisterInfo *MRI = nullptr;
190
191 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
192 // uses "STI." in the code generated by TableGen. We need to unify the name of
193 // Subtarget variable.
194 const RISCVSubtarget *Subtarget = &STI;
195
196#define GET_GLOBALISEL_PREDICATES_DECL
197#include "RISCVGenGlobalISel.inc"
198#undef GET_GLOBALISEL_PREDICATES_DECL
199
200#define GET_GLOBALISEL_TEMPORARIES_DECL
201#include "RISCVGenGlobalISel.inc"
202#undef GET_GLOBALISEL_TEMPORARIES_DECL
203};
204
205} // end anonymous namespace
206
207#define GET_GLOBALISEL_IMPL
208#include "RISCVGenGlobalISel.inc"
209#undef GET_GLOBALISEL_IMPL
210
211RISCVInstructionSelector::RISCVInstructionSelector(
212 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
213 const RISCVRegisterBankInfo &RBI)
214 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
215 TM(TM),
216
218#include "RISCVGenGlobalISel.inc"
221#include "RISCVGenGlobalISel.inc"
223{
224}
225
226// Mimics optimizations in ISel and RISCVOptWInst Pass
227bool RISCVInstructionSelector::hasAllNBitUsers(const MachineInstr &MI,
228 unsigned Bits,
229 const unsigned Depth) const {
230
231 assert((MI.getOpcode() == TargetOpcode::G_ADD ||
232 MI.getOpcode() == TargetOpcode::G_SUB ||
233 MI.getOpcode() == TargetOpcode::G_MUL ||
234 MI.getOpcode() == TargetOpcode::G_SHL ||
235 MI.getOpcode() == TargetOpcode::G_LSHR ||
236 MI.getOpcode() == TargetOpcode::G_AND ||
237 MI.getOpcode() == TargetOpcode::G_OR ||
238 MI.getOpcode() == TargetOpcode::G_XOR ||
239 MI.getOpcode() == TargetOpcode::G_SEXT_INREG || Depth != 0) &&
240 "Unexpected opcode");
241
242 if (Depth >= RISCVInstructionSelector::MaxRecursionDepth)
243 return false;
244
245 auto DestReg = MI.getOperand(0).getReg();
246 for (auto &UserOp : MRI->use_nodbg_operands(DestReg)) {
247 assert(UserOp.getParent() && "UserOp must have a parent");
248 const MachineInstr &UserMI = *UserOp.getParent();
249 unsigned OpIdx = UserOp.getOperandNo();
250
251 switch (UserMI.getOpcode()) {
252 default:
253 return false;
254 case RISCV::ADDW:
255 case RISCV::ADDIW:
256 case RISCV::SUBW:
257 case RISCV::FCVT_D_W:
258 case RISCV::FCVT_S_W:
259 if (Bits >= 32)
260 break;
261 return false;
262 case RISCV::SLL:
263 case RISCV::SRA:
264 case RISCV::SRL:
265 // Shift amount operands only use log2(Xlen) bits.
266 if (OpIdx == 2 && Bits >= Log2_32(Subtarget->getXLen()))
267 break;
268 return false;
269 case RISCV::SLLI:
270 // SLLI only uses the lower (XLen - ShAmt) bits.
271 if (Bits >= Subtarget->getXLen() - UserMI.getOperand(2).getImm())
272 break;
273 return false;
274 case RISCV::ANDI:
275 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(
276 (uint64_t)UserMI.getOperand(2).getImm()))
277 break;
278 goto RecCheck;
279 case RISCV::AND:
280 case RISCV::OR:
281 case RISCV::XOR:
282 RecCheck:
283 if (hasAllNBitUsers(UserMI, Bits, Depth + 1))
284 break;
285 return false;
286 case RISCV::SRLI: {
287 unsigned ShAmt = UserMI.getOperand(2).getImm();
288 // If we are shifting right by less than Bits, and users don't demand any
289 // bits that were shifted into [Bits-1:0], then we can consider this as an
290 // N-Bit user.
291 if (Bits > ShAmt && hasAllNBitUsers(UserMI, Bits - ShAmt, Depth + 1))
292 break;
293 return false;
294 }
295 }
296 }
297
298 return true;
299}
300
301InstructionSelector::ComplexRendererFns
302RISCVInstructionSelector::selectShiftMask(MachineOperand &Root,
303 unsigned ShiftWidth) const {
304 if (!Root.isReg())
305 return std::nullopt;
306
307 using namespace llvm::MIPatternMatch;
308
309 Register ShAmtReg = Root.getReg();
310 // Peek through zext.
311 Register ZExtSrcReg;
312 if (mi_match(ShAmtReg, *MRI, m_GZExt(m_Reg(ZExtSrcReg))))
313 ShAmtReg = ZExtSrcReg;
314
315 APInt AndMask;
316 Register AndSrcReg;
317 // Try to combine the following pattern (applicable to other shift
318 // instructions as well as 32-bit ones):
319 //
320 // %4:gprb(s64) = G_AND %3, %2
321 // %5:gprb(s64) = G_LSHR %1, %4(s64)
322 //
323 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
324 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
325 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
326 // then it can be eliminated. Given register rs1 or rs2 holding a constant
327 // (the and mask), there are two cases G_AND can be erased:
328 //
329 // 1. the lowest log2(XLEN) bits of the and mask are all set
330 // 2. the bits of the register being masked are already unset (zero set)
331 if (mi_match(ShAmtReg, *MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
332 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
333 if (ShMask.isSubsetOf(AndMask)) {
334 ShAmtReg = AndSrcReg;
335 } else {
336 // SimplifyDemandedBits may have optimized the mask so try restoring any
337 // bits that are known zero.
338 KnownBits Known = VT->getKnownBits(AndSrcReg);
339 if (ShMask.isSubsetOf(AndMask | Known.Zero))
340 ShAmtReg = AndSrcReg;
341 }
342 }
343
344 APInt Imm;
346 if (mi_match(ShAmtReg, *MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
347 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
348 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
349 // to avoid the ADD.
350 ShAmtReg = Reg;
351 } else if (mi_match(ShAmtReg, *MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
352 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
353 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
354 // to generate a NEG instead of a SUB of a constant.
355 ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
356 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
357 return {{[=](MachineInstrBuilder &MIB) {
358 MachineIRBuilder(*MIB.getInstr())
359 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
360 MIB.addReg(ShAmtReg);
361 }}};
362 }
363 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
364 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
365 // to generate a NOT instead of a SUB of a constant.
366 ShAmtReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
367 return {{[=](MachineInstrBuilder &MIB) {
368 MachineIRBuilder(*MIB.getInstr())
369 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
370 .addImm(-1);
371 MIB.addReg(ShAmtReg);
372 }}};
373 }
374 }
375
376 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
377}
378
379InstructionSelector::ComplexRendererFns
380RISCVInstructionSelector::selectSExtBits(MachineOperand &Root,
381 unsigned Bits) const {
382 if (!Root.isReg())
383 return std::nullopt;
384 Register RootReg = Root.getReg();
385
386 Register SrcReg;
387 if (mi_match(RootReg, *MRI,
388 m_GSExtInReg(m_Reg(SrcReg), m_SpecificImm(Bits)))) {
389 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(SrcReg); }}};
390 }
391
392 unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
393 if ((Size - VT->computeNumSignBits(RootReg)) < Bits)
394 return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
395
396 return std::nullopt;
397}
398
399InstructionSelector::ComplexRendererFns
400RISCVInstructionSelector::selectZExtBits(MachineOperand &Root,
401 unsigned Bits) const {
402 if (!Root.isReg())
403 return std::nullopt;
404 Register RootReg = Root.getReg();
405
406 Register RegX;
408 if (mi_match(RootReg, *MRI, m_GAnd(m_Reg(RegX), m_SpecificICst(Mask)))) {
409 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
410 }
411
412 if (mi_match(RootReg, *MRI, m_GZExt(m_Reg(RegX))) &&
413 MRI->getType(RegX).getScalarSizeInBits() == Bits)
414 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RegX); }}};
415
416 unsigned Size = MRI->getType(RootReg).getScalarSizeInBits();
417 if (VT->maskedValueIsZero(RootReg, APInt::getBitsSetFrom(Size, Bits)))
418 return {{[=](MachineInstrBuilder &MIB) { MIB.add(Root); }}};
419
420 return std::nullopt;
421}
422
423InstructionSelector::ComplexRendererFns
424RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
425 unsigned ShAmt) const {
426 using namespace llvm::MIPatternMatch;
427
428 if (!Root.isReg())
429 return std::nullopt;
430 Register RootReg = Root.getReg();
431
432 const unsigned XLen = STI.getXLen();
433 APInt Mask;
434 int64_t C2;
435 Register RegY;
436 std::optional<bool> LeftShift;
437 // (and (shl y, c2), mask)
438 if (mi_match(RootReg, *MRI,
439 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
440 LeftShift = true;
441 // (and (lshr y, c2), mask)
442 else if (mi_match(RootReg, *MRI,
443 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
444 LeftShift = false;
445
446 if (LeftShift.has_value()) {
447 assert(Mask.getBitWidth() == XLen && "Unexpected mask size!");
448 assert(C2 > 0 && C2 < XLen && "Unexpected shift amount!");
449 if (*LeftShift)
451 else
452 Mask &= maskTrailingOnes<uint64_t>(XLen - C2);
453
454 if (Mask.isShiftedMask()) {
455 unsigned Leading = XLen - Mask.getActiveBits();
456 unsigned Trailing = Mask.countr_zero();
457 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
458 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
459 if (*LeftShift && Leading == 0 && C2 < Trailing && Trailing == ShAmt) {
460 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
461 return {{[=](MachineInstrBuilder &MIB) {
462 MachineIRBuilder(*MIB.getInstr())
463 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
464 .addImm(Trailing - C2);
465 MIB.addReg(DstReg);
466 }}};
467 }
468
469 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
470 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
471 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
472 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
473 return {{[=](MachineInstrBuilder &MIB) {
474 MachineIRBuilder(*MIB.getInstr())
475 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
476 .addImm(Leading + Trailing);
477 MIB.addReg(DstReg);
478 }}};
479 }
480 }
481 }
482
483 LeftShift.reset();
484
485 // (shl (and y, mask), c2)
486 if (mi_match(RootReg, *MRI,
487 m_GShl(m_OneUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))), m_ICst(C2))))
488 LeftShift = true;
489 // (lshr (and y, mask), c2)
490 else if (mi_match(RootReg, *MRI,
491 m_GLShr(m_OneUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
492 m_ICst(C2))))
493 LeftShift = false;
494
495 if (LeftShift.has_value() && Mask.isShiftedMask()) {
496 unsigned Leading = XLen - Mask.getActiveBits();
497 unsigned Trailing = Mask.countr_zero();
498
499 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
500 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
501 bool Cond =
502 *LeftShift && Leading == 32 && Trailing > 0 && (Trailing + C2) == ShAmt;
503 if (!Cond)
504 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
505 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
506 Cond = !*LeftShift && Leading == 32 && C2 < Trailing &&
507 (Trailing - C2) == ShAmt;
508
509 if (Cond) {
510 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
511 return {{[=](MachineInstrBuilder &MIB) {
512 MachineIRBuilder(*MIB.getInstr())
513 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
514 .addImm(Trailing);
515 MIB.addReg(DstReg);
516 }}};
517 }
518 }
519
520 return std::nullopt;
521}
522
523InstructionSelector::ComplexRendererFns
524RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
525 unsigned ShAmt) const {
526 using namespace llvm::MIPatternMatch;
527
528 if (!Root.isReg())
529 return std::nullopt;
530 Register RootReg = Root.getReg();
531
532 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
533 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
534 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
535 APInt Mask;
536 int64_t C2;
537 Register RegX;
538 if (mi_match(RootReg, *MRI,
540 m_ICst(Mask))))) {
541 assert(C2 > 0 && C2 < Mask.getBitWidth() && "Unexpected shift amount!");
543
544 if (Mask.isShiftedMask()) {
545 unsigned Leading = Mask.countl_zero();
546 unsigned Trailing = Mask.countr_zero();
547 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
548 Register DstReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
549 return {{[=](MachineInstrBuilder &MIB) {
550 MachineIRBuilder(*MIB.getInstr())
551 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
552 .addImm(C2 - ShAmt);
553 MIB.addReg(DstReg);
554 }}};
555 }
556 }
557 }
558
559 return std::nullopt;
560}
561
562InstructionSelector::ComplexRendererFns
563RISCVInstructionSelector::renderVLOp(MachineOperand &Root) const {
564 assert(Root.isReg() && "Expected operand to be a Register");
565 std::optional<ValueAndVReg> C;
566 if (mi_match(Root.getReg(), *MRI, m_GCst(C))) {
567 if (C->Value.isAllOnes())
568 // If the operand is a G_CONSTANT with value of all ones it is larger than
569 // VLMAX. We convert it to an immediate with value VLMaxSentinel. This is
570 // recognized specially by the vsetvli insertion pass.
571 return {{[=](MachineInstrBuilder &MIB) {
572 MIB.addImm(RISCV::VLMaxSentinel);
573 }}};
574
575 if (isUInt<5>(C->Value.getZExtValue())) {
576 uint64_t ZExtC = C->Value.getZExtValue();
577 return {{[=](MachineInstrBuilder &MIB) { MIB.addImm(ZExtC); }}};
578 }
579 }
580 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); }}};
581}
582
583InstructionSelector::ComplexRendererFns
584RISCVInstructionSelector::renderAddiPair(Register BaseReg, int64_t AddiImm,
585 int64_t OffsetImm) const {
586 return {{[=](MachineInstrBuilder &MIB) {
587 Register Tmp = MRI->createVirtualRegister(&RISCV::GPRRegClass);
588 MachineInstr *Addi =
589 BuildMI(*MIB->getParent(), *MIB.getInstr(), MIB->getDebugLoc(),
590 TII.get(RISCV::ADDI), Tmp)
591 .addReg(BaseReg)
592 .addImm(AddiImm);
594 MIB.addReg(Tmp);
595 },
596 [=](MachineInstrBuilder &MIB) { MIB.addImm(OffsetImm); }}};
597}
598
599InstructionSelector::ComplexRendererFns
600RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
601 if (!Root.isReg())
602 return std::nullopt;
603
604 Register RootReg = Root.getReg();
605
606 // Frame index.
607 int FI;
608 if (mi_match(RootReg, *MRI, m_GFrameIndex(FI))) {
609 return {{
610 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); },
611 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
612 }};
613 }
614
615 // base + constant offset (G_PTR_ADD).
617 int64_t RHSC;
618 if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC)))) {
619 if (isInt<12>(RHSC)) {
620 int BaseFI;
621 if (mi_match(BaseReg, *MRI, m_GFrameIndex(BaseFI)))
622 return {{
623 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(BaseFI); },
624 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
625 }};
626
627 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
628 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
629 }
630
631 // Large constant offset. Fold a -2048/2047 adjustment so the whole
632 // constant can be split across an ADDI and the load/store offset.
633 if (RHSC >= -4096 && RHSC <= 4094) {
634 int64_t Adj = RHSC < 0 ? -2048 : 2047;
635 return renderAddiPair(BaseReg, Adj, RHSC - Adj);
636 }
637
638 if (isWorthFoldingAdd(RootReg))
639 if (auto Fns = computeConstAddr(RHSC, /*IsPrefetch=*/false, BaseReg))
640 return Fns;
641 }
642
643 // Bare constant address. IRTranslator lowers inttoptr(C) to
644 // G_INTTOPTR(G_CONSTANT); look through it to reach the constant.
645 int64_t CVal;
646 if (mi_match(RootReg, *MRI, m_GIntToPtr(m_ICst(CVal))) ||
647 mi_match(RootReg, *MRI, m_ICst(CVal))) {
648 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/false, Register()))
649 return Fns;
650 }
651
652 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
653 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
654}
655
656InstructionSelector::ComplexRendererFns
657RISCVInstructionSelector::selectBrindRegImm(MachineOperand &Root) const {
658 if (!Root.isReg())
659 return std::nullopt;
660
661 Register RootReg = Root.getReg();
662
663 // base + constant offset (G_PTR_ADD). Doesn't match a FrameIndex, unlike
664 // selectAddrRegImm, since the callers of this (e.g. indirect branches)
665 // can't take a FrameIndex or global address operand.
667 int64_t RHSC;
668 if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC))) &&
669 isInt<12>(RHSC)) {
670 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
671 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
672 }
673
674 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
675 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
676}
677
678InstructionSelector::ComplexRendererFns
679RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
680 if (!Root.isReg())
681 return std::nullopt;
682
683 Register RootReg = Root.getReg();
684
685 // Frame index.
686 int FI;
687 if (mi_match(RootReg, *MRI, m_GFrameIndex(FI))) {
688 return {{
689 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(FI); },
690 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
691 }};
692 }
693
694 // base + constant offset (G_PTR_ADD).
696 int64_t RHSC;
697 if (mi_match(RootReg, *MRI, m_GPtrAdd(m_Reg(BaseReg), m_ICst(RHSC)))) {
698 if (isInt<12>(RHSC)) {
699 // Not a multiple of 32: can't encode, use the address as-is.
700 if ((RHSC & 0b11111) != 0) {
701 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
702 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
703 }
704 // Fold the offset.
705 int BaseFI;
706 if (mi_match(BaseReg, *MRI, m_GFrameIndex(BaseFI)))
707 return {{
708 [=](MachineInstrBuilder &MIB) { MIB.addFrameIndex(BaseFI); },
709 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
710 }};
711 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(BaseReg); },
712 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
713 }
714
715 // Large constant: fold a -2048/2016 adjustment to save an instruction.
716 if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
717 int64_t Adj = RHSC < 0 ? -2048 : 2016;
718 return renderAddiPair(BaseReg, RHSC - Adj, Adj);
719 }
720
721 // Otherwise split the constant into Hi (materialized + added to the base)
722 // and Lo12 (folded offset).
723 if (auto Fns = computeConstAddr(RHSC, /*IsPrefetch=*/true, BaseReg))
724 return Fns;
725 }
726
727 // Bare constant address. IRTranslator emits inttoptr(C) as
728 // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
729 int64_t CVal;
730 if (mi_match(RootReg, *MRI, m_GIntToPtr(m_ICst(CVal))) ||
731 mi_match(RootReg, *MRI, m_ICst(CVal))) {
732 if (auto Fns = computeConstAddr(CVal, /*IsPrefetch=*/true, Register()))
733 return Fns;
734 }
735
736 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(RootReg); },
737 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
738}
739
740/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
741/// CC Must be an ICMP Predicate.
742static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
743 switch (CC) {
744 default:
745 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
746 case CmpInst::Predicate::ICMP_EQ:
747 return RISCVCC::COND_EQ;
748 case CmpInst::Predicate::ICMP_NE:
749 return RISCVCC::COND_NE;
750 case CmpInst::Predicate::ICMP_ULT:
751 return RISCVCC::COND_LTU;
752 case CmpInst::Predicate::ICMP_SLT:
753 return RISCVCC::COND_LT;
754 case CmpInst::Predicate::ICMP_UGE:
755 return RISCVCC::COND_GEU;
756 case CmpInst::Predicate::ICMP_SGE:
757 return RISCVCC::COND_GE;
758 }
759}
760
764 // Try to fold an ICmp. If that fails, use a NE compare with X0.
766 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
767 LHS = CondReg;
768 RHS = RISCV::X0;
770 return;
771 }
772
773 // We found an ICmp, do some canonicalization.
774
775 // Adjust comparisons to use comparison with 0 if possible.
777 switch (Pred) {
779 // Convert X > -1 to X >= 0
780 if (*Constant == -1) {
782 RHS = RISCV::X0;
783 return;
784 }
785 break;
787 // Convert X < 1 to 0 >= X
788 if (*Constant == 1) {
790 RHS = LHS;
791 LHS = RISCV::X0;
792 return;
793 }
794 break;
795 default:
796 break;
797 }
798 }
799
800 switch (Pred) {
801 default:
802 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
809 // These CCs are supported directly by RISC-V branches.
810 break;
815 // These CCs are not supported directly by RISC-V branches, but changing the
816 // direction of the CC and swapping LHS and RHS are.
818 std::swap(LHS, RHS);
819 break;
820 }
821
822 CC = getRISCVCCFromICmp(Pred);
823}
824
825/// Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation
826/// \p GenericOpc, appropriate for the GPR register bank and of memory access
827/// size \p OpSize.
828static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize) {
829 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
830 switch (OpSize) {
831 default:
832 llvm_unreachable("Unexpected memory size");
833 case 8:
834 return IsStore ? RISCV::SB_RL : RISCV::LB_AQ;
835 case 16:
836 return IsStore ? RISCV::SH_RL : RISCV::LH_AQ;
837 case 32:
838 return IsStore ? RISCV::SW_RL : RISCV::LW_AQ;
839 case 64:
840 return IsStore ? RISCV::SD_RL : RISCV::LD_AQ;
841 }
842}
843
844/// Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation
845/// \p GenericOpc on \p STI, appropriate for the GPR register bank and of
846/// memory access size \p OpSize. \returns \p GenericOpc if the combination
847/// is unsupported.
848static unsigned selectRegImmLoadStoreOp(const RISCVSubtarget &STI,
849 unsigned GenericOpc, unsigned OpSize) {
850 const bool IsStore = GenericOpc == TargetOpcode::G_STORE;
851 switch (OpSize) {
852 case 8:
853 // Prefer unsigned due to no c.lb in Zcb.
854 return IsStore ? RISCV::SB : RISCV::LBU;
855 case 16:
856 return IsStore ? RISCV::SH : RISCV::LH;
857 case 32:
858 return IsStore ? RISCV::SW : RISCV::LW;
859 case 64:
860 // LD/SD are RV64-only. On RV32 a 64-bit access goes through an even/odd
861 // GPR pair, which Zilsd provides via the LD_RV32/SD_RV32 pseudos.
862 if (!STI.is64Bit() && STI.hasStdExtZilsd())
863 return IsStore ? RISCV::SD_RV32 : RISCV::LD_RV32;
864 return IsStore ? RISCV::SD : RISCV::LD;
865 }
866
867 return GenericOpc;
868}
869
870void RISCVInstructionSelector::addVectorLoadStoreOperands(
871 MachineInstr &I, SmallVectorImpl<Register> &SrcOps, unsigned &CurOp,
872 bool IsMasked, bool IsStridedOrIndexed, LLT *IndexVT) const {
873 // Base Pointer
874 auto PtrReg = I.getOperand(CurOp++).getReg();
875 SrcOps.push_back(PtrReg);
876
877 // Stride or Index
878 if (IsStridedOrIndexed) {
879 auto StrideReg = I.getOperand(CurOp++).getReg();
880 SrcOps.push_back(StrideReg);
881 if (IndexVT)
882 *IndexVT = MRI->getType(StrideReg);
883 }
884
885 // Mask
886 if (IsMasked) {
887 auto MaskReg = I.getOperand(CurOp++).getReg();
888 SrcOps.push_back(MaskReg);
889 }
890}
891
892bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
893 MachineInstr &I) const {
894 // Find the intrinsic ID.
895 unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
896 // Select the instruction.
897 switch (IntrinID) {
898 default:
899 return false;
900 case Intrinsic::riscv_vlm:
901 case Intrinsic::riscv_vle:
902 case Intrinsic::riscv_vle_mask:
903 case Intrinsic::riscv_vlse:
904 case Intrinsic::riscv_vlse_mask: {
905 bool IsMasked = IntrinID == Intrinsic::riscv_vle_mask ||
906 IntrinID == Intrinsic::riscv_vlse_mask;
907 bool IsStrided = IntrinID == Intrinsic::riscv_vlse ||
908 IntrinID == Intrinsic::riscv_vlse_mask;
909 LLT VT = MRI->getType(I.getOperand(0).getReg());
910 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
911
912 // Result vector
913 const Register DstReg = I.getOperand(0).getReg();
914
915 // Sources
916 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
917 unsigned CurOp = 2;
918 SmallVector<Register, 4> SrcOps; // Source registers.
919
920 // Passthru
921 if (HasPassthruOperand) {
922 auto PassthruReg = I.getOperand(CurOp++).getReg();
923 SrcOps.push_back(PassthruReg);
924 } else {
925 SrcOps.push_back(Register(RISCV::NoRegister));
926 }
927
928 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);
929
931 const RISCV::VLEPseudo *P =
932 RISCV::getVLEPseudo(IsMasked, IsStrided, /*FF*/ false, Log2SEW,
933 static_cast<unsigned>(LMUL));
934
935 MachineInstrBuilder PseudoMI =
936 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo), DstReg);
937 for (Register Reg : SrcOps)
938 PseudoMI.addReg(Reg);
939
940 // Select VL
941 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
942 for (auto &RenderFn : *VLOpFn)
943 RenderFn(PseudoMI);
944
945 // SEW
946 PseudoMI.addImm(Log2SEW);
947
948 // Policy
950 if (IsMasked)
951 Policy = I.getOperand(CurOp++).getImm();
952 PseudoMI.addImm(Policy);
953
954 // Memref
955 PseudoMI.cloneMemRefs(I);
956
957 I.eraseFromParent();
958 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
959 return true;
960 }
961 case Intrinsic::riscv_vloxei:
962 case Intrinsic::riscv_vloxei_mask:
963 case Intrinsic::riscv_vluxei:
964 case Intrinsic::riscv_vluxei_mask: {
965 bool IsMasked = IntrinID == Intrinsic::riscv_vloxei_mask ||
966 IntrinID == Intrinsic::riscv_vluxei_mask;
967 bool IsOrdered = IntrinID == Intrinsic::riscv_vloxei ||
968 IntrinID == Intrinsic::riscv_vloxei_mask;
969 LLT VT = MRI->getType(I.getOperand(0).getReg());
970 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
971
972 // Result vector
973 const Register DstReg = I.getOperand(0).getReg();
974
975 // Sources
976 bool HasPassthruOperand = IntrinID != Intrinsic::riscv_vlm;
977 unsigned CurOp = 2;
978 SmallVector<Register, 4> SrcOps; // Source registers.
979
980 // Passthru
981 if (HasPassthruOperand) {
982 auto PassthruReg = I.getOperand(CurOp++).getReg();
983 SrcOps.push_back(PassthruReg);
984 } else {
985 // Use NoRegister if there is no specified passthru.
986 SrcOps.push_back(Register());
987 }
988 LLT IndexVT;
989 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);
990
992 RISCVVType::VLMUL IndexLMUL =
994 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
995 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
996 reportFatalUsageError("The V extension does not support EEW=64 for index "
997 "values when XLEN=32");
998 }
999 const RISCV::VLX_VSXPseudo *P = RISCV::getVLXPseudo(
1000 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1001 static_cast<unsigned>(IndexLMUL));
1002
1003 MachineInstrBuilder PseudoMI =
1004 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo), DstReg);
1005 for (Register Reg : SrcOps)
1006 PseudoMI.addReg(Reg);
1007
1008 // Select VL
1009 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
1010 for (auto &RenderFn : *VLOpFn)
1011 RenderFn(PseudoMI);
1012
1013 // SEW
1014 PseudoMI.addImm(Log2SEW);
1015
1016 // Policy
1018 if (IsMasked)
1019 Policy = I.getOperand(CurOp++).getImm();
1020 PseudoMI.addImm(Policy);
1021
1022 // Memref
1023 PseudoMI.cloneMemRefs(I);
1024
1025 I.eraseFromParent();
1026 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
1027 return true;
1028 }
1029 case Intrinsic::riscv_vsm:
1030 case Intrinsic::riscv_vse:
1031 case Intrinsic::riscv_vse_mask:
1032 case Intrinsic::riscv_vsse:
1033 case Intrinsic::riscv_vsse_mask: {
1034 bool IsMasked = IntrinID == Intrinsic::riscv_vse_mask ||
1035 IntrinID == Intrinsic::riscv_vsse_mask;
1036 bool IsStrided = IntrinID == Intrinsic::riscv_vsse ||
1037 IntrinID == Intrinsic::riscv_vsse_mask;
1038 LLT VT = MRI->getType(I.getOperand(1).getReg());
1039 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1040
1041 // Sources
1042 unsigned CurOp = 1;
1043 SmallVector<Register, 4> SrcOps; // Source registers.
1044
1045 // Store value
1046 auto PassthruReg = I.getOperand(CurOp++).getReg();
1047 SrcOps.push_back(PassthruReg);
1048
1049 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, IsStrided);
1050
1052 const RISCV::VSEPseudo *P = RISCV::getVSEPseudo(
1053 IsMasked, IsStrided, Log2SEW, static_cast<unsigned>(LMUL));
1054
1055 MachineInstrBuilder PseudoMI =
1056 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo));
1057 for (Register Reg : SrcOps)
1058 PseudoMI.addReg(Reg);
1059
1060 // Select VL
1061 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
1062 for (auto &RenderFn : *VLOpFn)
1063 RenderFn(PseudoMI);
1064
1065 // SEW
1066 PseudoMI.addImm(Log2SEW);
1067
1068 // Memref
1069 PseudoMI.cloneMemRefs(I);
1070
1071 I.eraseFromParent();
1072 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
1073 return true;
1074 }
1075 case Intrinsic::riscv_vsoxei:
1076 case Intrinsic::riscv_vsoxei_mask:
1077 case Intrinsic::riscv_vsuxei:
1078 case Intrinsic::riscv_vsuxei_mask: {
1079 bool IsMasked = IntrinID == Intrinsic::riscv_vsoxei_mask ||
1080 IntrinID == Intrinsic::riscv_vsuxei_mask;
1081 bool IsOrdered = IntrinID == Intrinsic::riscv_vsoxei ||
1082 IntrinID == Intrinsic::riscv_vsoxei_mask;
1083 LLT VT = MRI->getType(I.getOperand(1).getReg());
1084 unsigned Log2SEW = Log2_32(VT.getScalarSizeInBits());
1085
1086 // Sources
1087 unsigned CurOp = 1;
1088 SmallVector<Register, 4> SrcOps; // Source registers.
1089
1090 // Store value
1091 auto PassthruReg = I.getOperand(CurOp++).getReg();
1092 SrcOps.push_back(PassthruReg);
1093
1094 LLT IndexVT;
1095 addVectorLoadStoreOperands(I, SrcOps, CurOp, IsMasked, true, &IndexVT);
1096
1098 RISCVVType::VLMUL IndexLMUL =
1100 unsigned IndexLog2EEW = Log2_32(IndexVT.getScalarSizeInBits());
1101 if (IndexLog2EEW == 6 && !Subtarget->is64Bit()) {
1102 reportFatalUsageError("The V extension does not support EEW=64 for index "
1103 "values when XLEN=32");
1104 }
1105 const RISCV::VLX_VSXPseudo *P = RISCV::getVSXPseudo(
1106 IsMasked, IsOrdered, IndexLog2EEW, static_cast<unsigned>(LMUL),
1107 static_cast<unsigned>(IndexLMUL));
1108
1109 MachineInstrBuilder PseudoMI =
1110 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(P->Pseudo));
1111 for (Register Reg : SrcOps)
1112 PseudoMI.addReg(Reg);
1113
1114 // Select VL
1115 auto VLOpFn = renderVLOp(I.getOperand(CurOp++));
1116 for (auto &RenderFn : *VLOpFn)
1117 RenderFn(PseudoMI);
1118
1119 // SEW
1120 PseudoMI.addImm(Log2SEW);
1121
1122 // Memref
1123 PseudoMI.cloneMemRefs(I);
1124
1125 I.eraseFromParent();
1126 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
1127 return true;
1128 }
1129 }
1130}
1131
1132bool RISCVInstructionSelector::selectIntrinsic(MachineInstr &I) const {
1133 // Find the intrinsic ID.
1134 unsigned IntrinID = cast<GIntrinsic>(I).getIntrinsicID();
1135 // Select the instruction.
1136 switch (IntrinID) {
1137 default:
1138 return false;
1139 case Intrinsic::riscv_vsetvli:
1140 case Intrinsic::riscv_vsetvlimax: {
1141
1142 bool VLMax = IntrinID == Intrinsic::riscv_vsetvlimax;
1143
1144 unsigned Offset = VLMax ? 2 : 3;
1145 unsigned SEW = RISCVVType::decodeVSEW(I.getOperand(Offset).getImm() & 0x7);
1146 RISCVVType::VLMUL VLMul =
1147 static_cast<RISCVVType::VLMUL>(I.getOperand(Offset + 1).getImm() & 0x7);
1148
1149 unsigned VTypeI = RISCVVType::encodeVTYPE(VLMul, SEW, /*TailAgnostic*/ true,
1150 /*MaskAgnostic*/ true);
1151
1152 Register DstReg = I.getOperand(0).getReg();
1153
1154 Register VLOperand;
1155 unsigned Opcode = RISCV::PseudoVSETVLI;
1156
1157 // Check if AVL is a constant that equals VLMAX.
1158 if (!VLMax) {
1159 Register AVLReg = I.getOperand(2).getReg();
1160 if (auto AVLConst = getIConstantVRegValWithLookThrough(AVLReg, *MRI)) {
1161 uint64_t AVL = AVLConst->Value.getZExtValue();
1162 if (auto VLEN = Subtarget->getRealVLen()) {
1163 if (*VLEN / RISCVVType::getSEWLMULRatio(SEW, VLMul) == AVL)
1164 VLMax = true;
1165 }
1166 }
1167
1168 if (mi_match(AVLReg, *MRI, m_AllOnes()))
1169 VLMax = true;
1170 }
1171
1172 if (VLMax) {
1173 VLOperand = Register(RISCV::X0);
1174 Opcode = RISCV::PseudoVSETVLIX0;
1175 } else {
1176 Register AVLReg = I.getOperand(2).getReg();
1177 VLOperand = AVLReg;
1178
1179 // Check if AVL is a small constant that can use PseudoVSETIVLI.
1180 if (auto AVLConst = getIConstantVRegValWithLookThrough(AVLReg, *MRI)) {
1181 uint64_t AVL = AVLConst->Value.getZExtValue();
1182 if (isUInt<5>(AVL)) {
1183 MachineInstr *PseudoMI =
1184 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1185 TII.get(RISCV::PseudoVSETIVLI), DstReg)
1186 .addImm(AVL)
1187 .addImm(VTypeI);
1188 I.eraseFromParent();
1189 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
1190 return true;
1191 }
1192 }
1193 }
1194
1195 MachineInstr *PseudoMI =
1196 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1197 .addReg(VLOperand)
1198 .addImm(VTypeI);
1199 I.eraseFromParent();
1200 constrainSelectedInstRegOperands(*PseudoMI, TII, TRI, RBI);
1201 return true;
1202 }
1203 }
1204}
1205
1206bool RISCVInstructionSelector::selectExtractSubvector(MachineInstr &MI) const {
1207 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT_SUBVECTOR);
1208
1209 Register DstReg = MI.getOperand(0).getReg();
1210 Register SrcReg = MI.getOperand(1).getReg();
1211
1212 LLT DstTy = MRI->getType(DstReg);
1213 LLT SrcTy = MRI->getType(SrcReg);
1214
1215 unsigned Idx = static_cast<unsigned>(MI.getOperand(2).getImm());
1216
1217 MVT DstMVT = getMVTForLLT(DstTy);
1218 MVT SrcMVT = getMVTForLLT(SrcTy);
1219
1220 unsigned SubRegIdx;
1221 std::tie(SubRegIdx, Idx) =
1223 SrcMVT, DstMVT, Idx, &TRI);
1224
1225 if (Idx != 0)
1226 return false;
1227
1228 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(DstMVT);
1229 const TargetRegisterClass *DstRC = TRI.getRegClass(DstRegClassID);
1230 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
1231 return false;
1232
1233 unsigned SrcRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(SrcMVT);
1234 const TargetRegisterClass *SrcRC = TRI.getRegClass(SrcRegClassID);
1235 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI))
1236 return false;
1237
1238 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(TargetOpcode::COPY),
1239 DstReg)
1240 .addReg(SrcReg, {}, SubRegIdx);
1241
1242 MI.eraseFromParent();
1243 return true;
1244}
1245
1246bool RISCVInstructionSelector::selectInsertSubVector(MachineInstr &MI) const {
1247 assert(MI.getOpcode() == TargetOpcode::G_INSERT_SUBVECTOR);
1248
1249 Register DstReg = MI.getOperand(0).getReg();
1250 Register VecReg = MI.getOperand(1).getReg();
1251 Register SubVecReg = MI.getOperand(2).getReg();
1252
1253 LLT VecTy = MRI->getType(VecReg);
1254 LLT SubVecTy = MRI->getType(SubVecReg);
1255
1256 MVT VecMVT = getMVTForLLT(VecTy);
1257 MVT SubVecMVT = getMVTForLLT(SubVecTy);
1258
1259 unsigned Idx = static_cast<unsigned>(MI.getOperand(3).getImm());
1260
1261 unsigned SubRegIdx;
1262 std::tie(SubRegIdx, Idx) =
1264 VecMVT, SubVecMVT, Idx, &TRI);
1265
1266 // If the Idx hasn't been completely eliminated then this is a subvector
1267 // insert which doesn't naturally align to a vector register. These must
1268 // be handled using instructions to manipulate the vector registers.
1269 if (Idx != 0)
1270 return false;
1271
1272 // Constrain dst
1273 unsigned DstRegClassID = RISCVTargetLowering::getRegClassIDForVecVT(VecMVT);
1274 const TargetRegisterClass *DstRC = TRI.getRegClass(DstRegClassID);
1275 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI))
1276 return false;
1277
1278 // If we haven't set a SubRegIdx, then we must be going between
1279 // equally-sized LMUL groups (e.g. VR -> VR). This can be done as a copy.
1280 if (SubRegIdx == RISCV::NoSubRegister) {
1282 DstRegClassID &&
1283 "Unexpected subvector insert");
1284 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(TargetOpcode::COPY),
1285 DstReg)
1286 .addReg(SubVecReg);
1287 MI.eraseFromParent();
1288 return true;
1289 }
1290
1291 // Use INSERT_SUBREG to insert the subvector into the vector at the
1292 // appropriate subregister index.
1293 MachineInstr *Ins = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1294 TII.get(TargetOpcode::INSERT_SUBREG), DstReg)
1295 .addReg(VecReg)
1296 .addReg(SubVecReg)
1297 .addImm(SubRegIdx);
1298
1299 MI.eraseFromParent();
1301 return true;
1302}
1303
1304bool RISCVInstructionSelector::select(MachineInstr &MI) {
1305 preISelLower(MI);
1306 const unsigned Opc = MI.getOpcode();
1307
1308 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
1309 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
1310 const Register DefReg = MI.getOperand(0).getReg();
1311 const LLT DefTy = MRI->getType(DefReg);
1312
1313 const RegClassOrRegBank &RegClassOrBank =
1314 MRI->getRegClassOrRegBank(DefReg);
1315
1316 const TargetRegisterClass *DefRC =
1318 if (!DefRC) {
1319 if (!DefTy.isValid()) {
1320 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
1321 return false;
1322 }
1323
1324 const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
1325 DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, STI.is64Bit());
1326 if (!DefRC) {
1327 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
1328 return false;
1329 }
1330 }
1331
1332 MI.setDesc(TII.get(TargetOpcode::PHI));
1333 return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI);
1334 }
1335
1336 // Certain non-generic instructions also need some special handling.
1337 if (MI.isCopy())
1338 return selectCopy(MI);
1339
1340 return true;
1341 }
1342
1343 if (selectImpl(MI, *CoverageInfo))
1344 return true;
1345
1346 switch (Opc) {
1347 case TargetOpcode::G_BITCAST:
1348 case TargetOpcode::G_ANYEXT:
1349 case TargetOpcode::G_PTRTOINT:
1350 case TargetOpcode::G_INTTOPTR:
1351 case TargetOpcode::G_TRUNC:
1352 case TargetOpcode::G_FREEZE:
1353 return selectCopy(MI);
1354 case TargetOpcode::G_CONSTANT: {
1355 Register DstReg = MI.getOperand(0).getReg();
1356 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
1357
1358 if (!materializeImm(DstReg, Imm, MI))
1359 return false;
1360
1361 MI.eraseFromParent();
1362 return true;
1363 }
1364 case TargetOpcode::G_ZEXT:
1365 case TargetOpcode::G_SEXT: {
1366 bool IsSigned = Opc != TargetOpcode::G_ZEXT;
1367 Register DstReg = MI.getOperand(0).getReg();
1368 Register SrcReg = MI.getOperand(1).getReg();
1369 LLT SrcTy = MRI->getType(SrcReg);
1370 unsigned SrcSize = SrcTy.getSizeInBits();
1371
1372 if (SrcTy.isVector())
1373 return false; // Should be handled by imported patterns.
1374
1375 assert((*RBI.getRegBank(DstReg, *MRI, TRI)).getID() ==
1376 RISCV::GPRBRegBankID &&
1377 "Unexpected ext regbank");
1378
1379 // Use addiw SrcReg, 0 (sext.w) for i32.
1380 if (IsSigned && SrcSize == 32) {
1381 MI.setDesc(TII.get(RISCV::ADDIW));
1382 MI.addOperand(MachineOperand::CreateImm(0));
1384 return true;
1385 }
1386
1387 // Use add.uw SrcReg, X0 (zext.w) for i32 with Zba.
1388 if (!IsSigned && SrcSize == 32 && STI.hasStdExtZba()) {
1389 MI.setDesc(TII.get(RISCV::ADD_UW));
1390 MI.addOperand(MachineOperand::CreateReg(RISCV::X0, /*isDef=*/false));
1392 return true;
1393 }
1394
1395 // Use sext.h/zext.h for i16 with Zbb.
1396 if (SrcSize == 16 &&
1397 (STI.hasStdExtZbb() || (!IsSigned && STI.hasStdExtZbkb()))) {
1398 MI.setDesc(TII.get(IsSigned ? RISCV::SEXT_H
1399 : STI.is64Bit() ? RISCV::ZEXT_H_RV64
1400 : RISCV::ZEXT_H_RV32));
1402 return true;
1403 }
1404
1405 // Fall back to shift pair.
1406 Register ShiftLeftReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1407 MachineInstr *ShiftLeft = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1408 TII.get(RISCV::SLLI), ShiftLeftReg)
1409 .addReg(SrcReg)
1410 .addImm(STI.getXLen() - SrcSize);
1411 constrainSelectedInstRegOperands(*ShiftLeft, TII, TRI, RBI);
1412 MachineInstr *ShiftRight =
1413 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1414 TII.get(IsSigned ? RISCV::SRAI : RISCV::SRLI), DstReg)
1415 .addReg(ShiftLeftReg)
1416 .addImm(STI.getXLen() - SrcSize);
1417 constrainSelectedInstRegOperands(*ShiftRight, TII, TRI, RBI);
1418 MI.eraseFromParent();
1419 return true;
1420 }
1421 case TargetOpcode::G_FCONSTANT: {
1422 // TODO: Use constant pool for complex constants.
1423 Register DstReg = MI.getOperand(0).getReg();
1424 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
1425 unsigned Size = MRI->getType(DstReg).getSizeInBits();
1426 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
1427 Register GPRReg;
1428 if (FPimm.isPosZero()) {
1429 GPRReg = RISCV::X0;
1430 } else {
1431 GPRReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1432 APInt Imm = FPimm.bitcastToAPInt();
1433 if (!materializeImm(GPRReg, Imm.getSExtValue(), MI))
1434 return false;
1435 }
1436
1437 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
1438 : Size == 32 ? RISCV::FMV_W_X
1439 : RISCV::FMV_H_X;
1440 MachineInstr *FMV = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1441 TII.get(Opcode), DstReg)
1442 .addReg(GPRReg);
1444 } else {
1445 // s64 on rv32
1446 assert(Size == 64 && !Subtarget->is64Bit() &&
1447 "Unexpected size or subtarget");
1448
1449 if (FPimm.isPosZero()) {
1450 // Optimize +0.0 to use fcvt.d.w
1451 MachineInstr *FCVT = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1452 TII.get(RISCV::FCVT_D_W), DstReg)
1453 .addReg(RISCV::X0)
1456
1457 MI.eraseFromParent();
1458 return true;
1459 }
1460
1461 // Split into two pieces and build through the stack.
1462 Register GPRRegHigh = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1463 Register GPRRegLow = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1464 APInt Imm = FPimm.bitcastToAPInt();
1465 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
1466 MI))
1467 return false;
1468 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MI))
1469 return false;
1470 MachineInstr *PairF64 =
1471 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1472 TII.get(RISCV::BuildPairF64Pseudo), DstReg)
1473 .addReg(GPRRegLow)
1474 .addReg(GPRRegHigh);
1475 constrainSelectedInstRegOperands(*PairF64, TII, TRI, RBI);
1476 }
1477
1478 MI.eraseFromParent();
1479 return true;
1480 }
1481 case TargetOpcode::G_GLOBAL_VALUE: {
1482 auto *GV = MI.getOperand(1).getGlobal();
1483 if (GV->isThreadLocal()) {
1484 // TODO: implement this case.
1485 return false;
1486 }
1487
1488 return selectAddr(MI, GV->isDSOLocal(), GV->hasExternalWeakLinkage());
1489 }
1490 case TargetOpcode::G_JUMP_TABLE:
1491 case TargetOpcode::G_CONSTANT_POOL:
1492 return selectAddr(MI);
1493 case TargetOpcode::G_BRCOND: {
1494 Register LHS, RHS;
1496 getOperandsForBranch(MI.getOperand(0).getReg(), CC, LHS, RHS, *MRI);
1497
1498 MachineInstr *Bcc = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1500 .addReg(LHS)
1501 .addReg(RHS)
1502 .addMBB(MI.getOperand(1).getMBB());
1503 MI.eraseFromParent();
1505 return true;
1506 }
1507 case TargetOpcode::G_SELECT:
1508 return selectSelect(MI);
1509 case TargetOpcode::G_FCMP:
1510 return selectFPCompare(MI);
1511 case TargetOpcode::G_FENCE: {
1512 AtomicOrdering FenceOrdering =
1513 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
1514 SyncScope::ID FenceSSID =
1515 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
1516 emitFence(FenceOrdering, FenceSSID, MI);
1517 MI.eraseFromParent();
1518 return true;
1519 }
1520 case TargetOpcode::G_IMPLICIT_DEF:
1521 return selectImplicitDef(MI);
1522 case TargetOpcode::G_UNMERGE_VALUES:
1523 return selectUnmergeValues(MI);
1524 case TargetOpcode::G_MERGE_VALUES:
1525 return selectMergeValues(MI);
1526 case TargetOpcode::G_LOAD:
1527 case TargetOpcode::G_STORE: {
1528 GLoadStore &LdSt = cast<GLoadStore>(MI);
1529 const Register ValReg = LdSt.getReg(0);
1530 const Register PtrReg = LdSt.getPointerReg();
1531 LLT PtrTy = MRI->getType(PtrReg);
1532
1533 const RegisterBank &RB = *RBI.getRegBank(ValReg, *MRI, TRI);
1534 if (RB.getID() != RISCV::GPRBRegBankID)
1535 return false;
1536
1537#ifndef NDEBUG
1538 const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, *MRI, TRI);
1539 // Check that the pointer register is valid.
1540 assert(PtrRB.getID() == RISCV::GPRBRegBankID &&
1541 "Load/Store pointer operand isn't a GPR");
1542 assert(PtrTy.isPointer() && "Load/Store pointer operand isn't a pointer");
1543#endif
1544
1545 // Can only handle AddressSpace 0.
1546 if (PtrTy.getAddressSpace() != 0)
1547 return false;
1548
1549 unsigned MemSize = LdSt.getMemSizeInBits().getValue();
1550 AtomicOrdering Order = LdSt.getMMO().getSuccessOrdering();
1551
1552 if (isStrongerThanMonotonic(Order)) {
1553 MI.setDesc(TII.get(selectZalasrLoadStoreOp(Opc, MemSize)));
1555 return true;
1556 }
1557
1558 const unsigned NewOpc =
1559 selectRegImmLoadStoreOp(STI, MI.getOpcode(), MemSize);
1560 if (NewOpc == MI.getOpcode())
1561 return false;
1562
1563 // Check if we can fold anything into the addressing mode.
1564 auto AddrModeFns = selectAddrRegImm(MI.getOperand(1));
1565 if (!AddrModeFns)
1566 return false;
1567
1568 // Folded something. Create a new instruction and return it.
1569 MachineInstrBuilder NewInst =
1570 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(NewOpc));
1571 NewInst.setMIFlags(MI.getFlags());
1572 if (isa<GStore>(MI))
1573 NewInst.addUse(ValReg);
1574 else
1575 NewInst.addDef(ValReg);
1576 NewInst.cloneMemRefs(MI);
1577 for (auto &Fn : *AddrModeFns)
1578 Fn(NewInst);
1579 MI.eraseFromParent();
1580
1581 constrainSelectedInstRegOperands(*NewInst, TII, TRI, RBI);
1582 return true;
1583 }
1584 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
1585 return selectIntrinsicWithSideEffects(MI);
1586 case TargetOpcode::G_INTRINSIC:
1587 return selectIntrinsic(MI);
1588 case TargetOpcode::G_EXTRACT_SUBVECTOR:
1589 return selectExtractSubvector(MI);
1590 case TargetOpcode::G_INSERT_SUBVECTOR:
1591 return selectInsertSubVector(MI);
1592 default:
1593 return false;
1594 }
1595}
1596
1597bool RISCVInstructionSelector::selectUnmergeValues(MachineInstr &MI) const {
1598 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
1599
1600 // Split Src into two s32 parts.
1601 if (MI.getNumOperands() != 3)
1602 return false;
1603 Register Src = MI.getOperand(2).getReg();
1605 Register Hi = MI.getOperand(1).getReg();
1606
1607 // RV32+Zilsd: split a GPRPair (i64) into its two s32 GPR halves.
1608 if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
1609 MRI->getType(Src) == LLT::scalar(64) && isRegInGprb(Src) &&
1610 isRegInGprb(Lo) && isRegInGprb(Hi)) {
1611 // GPRPairNoX0 rather than GPRPair: a sub-register index is only valid for
1612 // a class when every register in it supports the index, and X0_Pair has no
1613 // sub_gpr_odd. SelectionDAG gets the same effect through
1614 // InstrEmitter::ConstrainForSubReg.
1615 if (!RBI.constrainGenericRegister(Src, RISCV::GPRPairNoX0RegClass, *MRI))
1616 return false;
1617 // COPY with a sub-register index rather than EXTRACT_SUBREG, which is what
1618 // SelectionDAG emits too: InstrEmitter lowers an EXTRACT_SUBREG SDNode to
1619 // %dst = COPY %src:sub and never builds the pseudo itself. A literal
1620 // EXTRACT_SUBREG would also survive to ExpandPostRAPseudos, which treats
1621 // one as unreachable.
1622 auto ExtractHalf = [&](Register Dst, unsigned SubIdx) {
1623 RBI.constrainGenericRegister(Dst, RISCV::GPRRegClass, *MRI);
1624 MachineInstr &C = *BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1625 TII.get(TargetOpcode::COPY), Dst)
1626 .addReg(Src, {}, SubIdx);
1628 };
1629 ExtractHalf(Lo, RISCV::sub_gpr_even);
1630 ExtractHalf(Hi, RISCV::sub_gpr_odd);
1631 MI.eraseFromParent();
1632 return true;
1633 }
1634
1635 if (!Subtarget->hasStdExtZfa())
1636 return false;
1637
1638 if (!isRegInFprb(Src) || !isRegInGprb(Lo) || !isRegInGprb(Hi))
1639 return false;
1640
1641 MachineInstr *ExtractLo = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1642 TII.get(RISCV::FMV_X_W_FPR64), Lo)
1643 .addReg(Src);
1644 constrainSelectedInstRegOperands(*ExtractLo, TII, TRI, RBI);
1645
1646 MachineInstr *ExtractHi = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1647 TII.get(RISCV::FMVH_X_D), Hi)
1648 .addReg(Src);
1649 constrainSelectedInstRegOperands(*ExtractHi, TII, TRI, RBI);
1650
1651 MI.eraseFromParent();
1652 return true;
1653}
1654
1655bool RISCVInstructionSelector::selectMergeValues(MachineInstr &MI) const {
1656 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
1657
1658 if (MI.getNumOperands() != 3)
1659 return false;
1660 Register Dst = MI.getOperand(0).getReg();
1662 Register Hi = MI.getOperand(2).getReg();
1663
1664 // RV32+Zilsd: build a GPRPair (i64) from two s32 GPR halves.
1665 if (!Subtarget->is64Bit() && Subtarget->hasStdExtZilsd() &&
1666 MRI->getType(Dst) == LLT::scalar(64) && isRegInGprb(Dst) &&
1667 isRegInGprb(Lo) && isRegInGprb(Hi)) {
1668 // GPRPairNoX0 rather than GPRPair, because sub_gpr_odd is not valid for
1669 // X0_Pair. Constrain Dst up front: constrainSelectedInstRegOperands below
1670 // would otherwise infer GPRPair from the s64 type.
1671 if (!RBI.constrainGenericRegister(Dst, RISCV::GPRPairNoX0RegClass, *MRI))
1672 return false;
1673 MachineInstr &Seq = *BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1674 TII.get(TargetOpcode::REG_SEQUENCE), Dst)
1675 .addReg(Lo)
1676 .addImm(RISCV::sub_gpr_even)
1677 .addReg(Hi)
1678 .addImm(RISCV::sub_gpr_odd);
1679 MI.eraseFromParent();
1681 return true;
1682 }
1683 return false;
1684}
1685
1686bool RISCVInstructionSelector::replacePtrWithInt(MachineInstr &MI,
1687 unsigned OpIdx) {
1688 MachineOperand &Op = MI.getOperand(OpIdx);
1689 Register PtrReg = Op.getReg();
1690 assert(MRI->getType(PtrReg).isPointer() && "Operand is not a pointer!");
1691
1692 const LLT iXLen = LLT::integer(STI.getXLen());
1693 Register IntReg = MRI->createGenericVirtualRegister(iXLen);
1694 MRI->setRegBank(IntReg, RBI.getRegBank(RISCV::GPRBRegBankID));
1695 MachineInstr *PtrToInt = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
1696 TII.get(TargetOpcode::G_PTRTOINT), IntReg)
1697 .addReg(PtrReg);
1698 Op.setReg(IntReg);
1699 return select(*PtrToInt);
1700}
1701
1702void RISCVInstructionSelector::preISelLower(MachineInstr &MI) {
1703 switch (MI.getOpcode()) {
1704 case TargetOpcode::G_PTR_ADD: {
1705 Register DstReg = MI.getOperand(0).getReg();
1706 const LLT iXLen = LLT::integer(STI.getXLen());
1707
1708 replacePtrWithInt(MI, 1);
1709 MI.setDesc(TII.get(TargetOpcode::G_ADD));
1710 MRI->setType(DstReg, iXLen);
1711 break;
1712 }
1713 case TargetOpcode::G_PTRMASK: {
1714 Register DstReg = MI.getOperand(0).getReg();
1715 const LLT iXLen = LLT::integer(STI.getXLen());
1716 replacePtrWithInt(MI, 1);
1717 MI.setDesc(TII.get(TargetOpcode::G_AND));
1718 MRI->setType(DstReg, iXLen);
1719 break;
1720 }
1721 }
1722}
1723
1724void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
1725 const MachineInstr &MI,
1726 int OpIdx) const {
1727 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1728 "Expected G_CONSTANT");
1729 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
1730 MIB.addImm(-CstVal);
1731}
1732
1733void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
1734 const MachineInstr &MI,
1735 int OpIdx) const {
1736 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1737 "Expected G_CONSTANT");
1738 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
1739 MIB.addImm(STI.getXLen() - CstVal);
1740}
1741
1742void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
1743 const MachineInstr &MI,
1744 int OpIdx) const {
1745 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1746 "Expected G_CONSTANT");
1747 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
1748 MIB.addImm(32 - CstVal);
1749}
1750
1751void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
1752 const MachineInstr &MI,
1753 int OpIdx) const {
1754 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1755 "Expected G_CONSTANT");
1756 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
1757 MIB.addImm(CstVal + 1);
1758}
1759
1760void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
1761 const MachineInstr &MI,
1762 int OpIdx) const {
1763 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1764 "Expected G_CONSTANT");
1765 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
1767}
1768
1769void RISCVInstructionSelector::renderXLenSubTrailingOnes(
1770 MachineInstrBuilder &MIB, const MachineInstr &MI, int OpIdx) const {
1771 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1772 "Expected G_CONSTANT");
1773 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
1774 MIB.addImm(Subtarget->getXLen() - llvm::countr_one(C));
1775}
1776
1777void RISCVInstructionSelector::renderAddiPairImmSmall(MachineInstrBuilder &MIB,
1778 const MachineInstr &MI,
1779 int OpIdx) const {
1780 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1781 "Expected G_CONSTANT");
1782 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
1783 int64_t Adj = Imm < 0 ? -2048 : 2047;
1784 MIB.addImm(Imm - Adj);
1785}
1786
1787void RISCVInstructionSelector::renderAddiPairImmLarge(MachineInstrBuilder &MIB,
1788 const MachineInstr &MI,
1789 int OpIdx) const {
1790 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1791 "Expected G_CONSTANT");
1792 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue() < 0 ? -2048 : 2047;
1793 MIB.addImm(Imm);
1794}
1795
1796bool RISCVInstructionSelector::isRegInGprb(Register Reg) const {
1797 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::GPRBRegBankID;
1798}
1799
1800bool RISCVInstructionSelector::isRegInFprb(Register Reg) const {
1801 return RBI.getRegBank(Reg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID;
1802}
1803
1804// A G_PTR_ADD result is worth splitting into Hi (materialized) +
1805// Lo12 (folded offset) only if every user is a plain scalar load/store
1806// using it as the address. Otherwise the ADD is selected on its own with
1807// the full materialized constant, making the Hi materialization here redundant.
1808bool RISCVInstructionSelector::isWorthFoldingAdd(Register AddResult) const {
1809 for (const MachineInstr &User : MRI->use_nodbg_instructions(AddResult)) {
1810 auto *LdSt = dyn_cast<GLoadStore>(&User);
1811 if (!LdSt)
1812 return false;
1813 // Must be used as the pointer, not the stored value.
1814 if (LdSt->getPointerReg() != AddResult)
1815 return false;
1817 return false;
1818 // Only scalar integer/f16/f32/f64 memory (exclude vectors, f128, ...).
1819 LLT Ty = MRI->getType(User.getOperand(0).getReg());
1820 if (!Ty.isScalar() || Ty.getSizeInBits() > 64)
1821 return false;
1822 }
1823 return true;
1824}
1825
1826bool RISCVInstructionSelector::selectCopy(MachineInstr &MI) const {
1827 Register DstReg = MI.getOperand(0).getReg();
1828
1829 if (DstReg.isPhysical())
1830 return true;
1831
1832 const TargetRegisterClass *DstRC =
1833 TRI.getConstrainedRegClassForReg(DstReg, *MRI);
1834
1835 assert(DstRC &&
1836 "Register class not available for LLT, register bank combination");
1837
1838 // No need to constrain SrcReg. It will get constrained when
1839 // we hit another of its uses or its defs.
1840 // Copies do not have constraints.
1841 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
1842 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1843 << " operand\n");
1844 return false;
1845 }
1846
1847 MI.setDesc(TII.get(RISCV::COPY));
1848 return true;
1849}
1850
1851bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
1852 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
1853
1854 const Register DstReg = MI.getOperand(0).getReg();
1855 const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(
1856 MRI->getType(DstReg), *RBI.getRegBank(DstReg, *MRI, TRI), STI.is64Bit());
1857
1858 assert(DstRC &&
1859 "Register class not available for LLT, register bank combination");
1860
1861 if (!RBI.constrainGenericRegister(DstReg, *DstRC, *MRI)) {
1862 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
1863 << " operand\n");
1864 }
1865 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
1866 return true;
1867}
1868
1869bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
1870 MachineInstr &MI) const {
1871 if (Imm == 0) {
1872 MachineBasicBlock &MBB = *MI.getParent();
1873 DebugLoc DL = MI.getDebugLoc();
1874 BuildMI(MBB, MI, DL, TII.get(TargetOpcode::COPY), DstReg).addReg(RISCV::X0);
1875 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
1876 return true;
1877 }
1878
1880 return materializeInstSeq(DstReg, Seq, MI);
1881}
1882
1883bool RISCVInstructionSelector::materializeInstSeq(
1884 Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
1885 assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
1886
1887 MachineBasicBlock &MBB = *MI.getParent();
1888 DebugLoc DL = MI.getDebugLoc();
1889 unsigned NumInsts = Seq.size();
1890 Register SrcReg = RISCV::X0;
1891
1892 for (unsigned i = 0; i < NumInsts; i++) {
1893 Register TmpReg = i < NumInsts - 1
1894 ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
1895 : DstReg;
1896 const RISCVMatInt::Inst &I = Seq[i];
1897 MachineInstr *Result;
1898
1899 switch (I.getOpndKind()) {
1900 case RISCVMatInt::Imm:
1901 Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
1902 .addImm(I.getImm());
1903 break;
1904 case RISCVMatInt::RegX0:
1905 Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
1906 .addReg(SrcReg)
1907 .addReg(RISCV::X0);
1908 break;
1910 Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
1911 .addReg(SrcReg)
1912 .addReg(SrcReg);
1913 break;
1915 Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
1916 .addReg(SrcReg)
1917 .addImm(I.getImm());
1918 break;
1919 }
1920
1922
1923 SrcReg = TmpReg;
1924 }
1925
1926 return true;
1927}
1928
1929InstructionSelector::ComplexRendererFns
1930RISCVInstructionSelector::computeConstAddr(int64_t CVal, bool IsPrefetch,
1931 Register OrigBase) const {
1932 // Split the constant into a materialized high part (the base) and
1933 // a simm12 low part (the offset). For prefetch the low part
1934 // must additionally be a multiple of 32 (simm12_lsb00000).
1935 int64_t Lo12 = SignExtend64<12>(CVal);
1936 int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
1937 auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
1938 return {{[=](MachineInstrBuilder &MIB) {
1939 MIB.addReg(materializeConstBase(MIB, Plan, OrigBase));
1940 },
1941 [=](MachineInstrBuilder &MIB) { MIB.addImm(Plan.Lo12); }}};
1942 };
1943 if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
1944 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1945 return std::nullopt;
1946 ConstAddrPlan Plan;
1947 Plan.Lo12 = Lo12;
1948 if (Hi) {
1949 Plan.Kind = ConstAddrPlan::LUI;
1950 Plan.Hi20 = (Hi >> 12) & 0xfffff;
1951 }
1952 return emit(std::move(Plan));
1953 }
1954
1955 // Otherwise ask constant materialization how it would handle the constant
1956 // and fold the trailing ADDI into the offset.
1958 if (Seq.back().getOpcode() != RISCV::ADDI)
1959 return std::nullopt;
1960 Lo12 = Seq.back().getImm();
1961 if (IsPrefetch && (Lo12 & 0b11111) != 0)
1962 return std::nullopt;
1963 Seq.pop_back();
1964 if (Seq.empty())
1965 return std::nullopt;
1966 ConstAddrPlan Plan;
1967 Plan.Kind = ConstAddrPlan::InstSeq;
1968 Plan.Seq = std::move(Seq);
1969 Plan.Lo12 = Lo12;
1970 return emit(std::move(Plan));
1971}
1972
1974RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
1975 const ConstAddrPlan &Plan,
1976 Register OrigBase) const {
1977 MachineBasicBlock &MBB = *MIB->getParent();
1978 DebugLoc DL = MIB->getDebugLoc();
1979 MachineInstr &InsertPt = *MIB.getInstr();
1980
1981 Register HiReg = RISCV::X0;
1982 switch (Plan.Kind) {
1983 case ConstAddrPlan::X0:
1984 break;
1985 case ConstAddrPlan::LUI: {
1986 HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1987 MachineInstr *LUI = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::LUI), HiReg)
1988 .addImm(Plan.Hi20);
1990 break;
1991 }
1992 case ConstAddrPlan::InstSeq: {
1993 HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
1994 materializeInstSeq(HiReg, Plan.Seq, InsertPt);
1995 break;
1996 }
1997 }
1998
1999 // For G_PTR_ADD + large constant, add the original base to the materialized
2000 // high part.
2001 if (OrigBase.isValid() && HiReg != RISCV::X0) {
2002 Register BaseReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2003 MachineInstr *Add = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::ADD), BaseReg)
2004 .addReg(OrigBase)
2005 .addReg(HiReg);
2007 return BaseReg;
2008 }
2009 return OrigBase.isValid() ? OrigBase : HiReg;
2010}
2011
2012bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
2013 bool IsExternWeak) const {
2014 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
2015 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
2016 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
2017 "Unexpected opcode");
2018
2019 const MachineOperand &DispMO = MI.getOperand(1);
2020
2021 Register DefReg = MI.getOperand(0).getReg();
2022 const LLT DefTy = MRI->getType(DefReg);
2023
2024 // When HWASAN is used and tagging of global variables is enabled
2025 // they should be accessed via the GOT, since the tagged address of a global
2026 // is incompatible with existing code models. This also applies to non-pic
2027 // mode.
2028 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
2029 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
2030 // Use PC-relative addressing to access the symbol. This generates the
2031 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
2032 // %pcrel_lo(auipc)).
2033 MI.setDesc(TII.get(RISCV::PseudoLLA));
2035 return true;
2036 }
2037
2038 // Use PC-relative addressing to access the GOT for this symbol, then
2039 // load the address from the GOT. This generates the pattern (PseudoLGA
2040 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
2041 // %pcrel_lo(auipc))).
2042 MachineFunction &MF = *MI.getParent()->getParent();
2043 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2047 DefTy, Align(DefTy.getSizeInBits() / 8));
2048
2049 MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2050 TII.get(RISCV::PseudoLGA), DefReg)
2051 .addDisp(DispMO, 0)
2052 .addMemOperand(MemOp);
2053
2055
2056 MI.eraseFromParent();
2057 return true;
2058 }
2059
2060 switch (TM.getCodeModel()) {
2061 default: {
2063 "Unsupported code model for lowering", MI);
2064 return false;
2065 }
2066 case CodeModel::Small: {
2067 // Must lie within a single 2 GiB address range and must lie between
2068 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
2069 // (lui %hi(sym)) %lo(sym)).
2070 Register AddrHiDest = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2071 MachineInstr *AddrHi = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2072 TII.get(RISCV::LUI), AddrHiDest)
2073 .addDisp(DispMO, 0, RISCVII::MO_HI);
2074
2076
2077 MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2078 TII.get(RISCV::ADDI), DefReg)
2079 .addReg(AddrHiDest)
2080 .addDisp(DispMO, 0, RISCVII::MO_LO);
2081
2083
2084 MI.eraseFromParent();
2085 return true;
2086 }
2087 case CodeModel::Medium:
2088 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
2089 // relocation needs to reference a label that points to the auipc
2090 // instruction itself, not the global. This cannot be done inside the
2091 // instruction selector.
2092 if (IsExternWeak) {
2093 // An extern weak symbol may be undefined, i.e. have value 0, which may
2094 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
2095 // symbol. This generates the pattern (PseudoLGA sym), which expands to
2096 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2097 MachineFunction &MF = *MI.getParent()->getParent();
2098 MachineMemOperand *MemOp = MF.getMachineMemOperand(
2102 DefTy, Align(DefTy.getSizeInBits() / 8));
2103
2104 MachineInstr *Result = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2105 TII.get(RISCV::PseudoLGA), DefReg)
2106 .addDisp(DispMO, 0)
2107 .addMemOperand(MemOp);
2108
2110
2111 MI.eraseFromParent();
2112 return true;
2113 }
2114
2115 // Generate a sequence for accessing addresses within any 2GiB range
2116 // within the address space. This generates the pattern (PseudoLLA sym),
2117 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2118 MI.setDesc(TII.get(RISCV::PseudoLLA));
2120 return true;
2121 }
2122
2123 return false;
2124}
2125
2126bool RISCVInstructionSelector::selectSelect(MachineInstr &MI) const {
2127 auto &SelectMI = cast<GSelect>(MI);
2128
2129 Register LHS, RHS;
2131 getOperandsForBranch(SelectMI.getCondReg(), CC, LHS, RHS, *MRI);
2132
2133 Register DstReg = SelectMI.getReg(0);
2134
2135 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
2136 if (RBI.getRegBank(DstReg, *MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
2137 unsigned Size = MRI->getType(DstReg).getSizeInBits();
2138 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
2139 : RISCV::Select_FPR64_Using_CC_GPR;
2140 }
2141
2142 MachineInstr *Result =
2143 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), TII.get(Opc))
2144 .addDef(DstReg)
2145 .addReg(LHS)
2146 .addReg(RHS)
2147 .addImm(CC)
2148 .addReg(SelectMI.getTrueReg())
2149 .addReg(SelectMI.getFalseReg());
2150 MI.eraseFromParent();
2152 return true;
2153}
2154
2155// Convert an FCMP predicate to one of the supported F or D instructions.
2156static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
2157 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
2158 switch (Pred) {
2159 default:
2160 llvm_unreachable("Unsupported predicate");
2161 case CmpInst::FCMP_OLT:
2162 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
2163 case CmpInst::FCMP_OLE:
2164 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
2165 case CmpInst::FCMP_OEQ:
2166 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
2167 }
2168}
2169
2170// Try legalizing an FCMP by swapping or inverting the predicate to one that
2171// is supported.
2173 CmpInst::Predicate &Pred, bool &NeedInvert) {
2174 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
2175 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
2177 };
2178
2179 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
2180
2182 if (isLegalFCmpPredicate(InvPred)) {
2183 Pred = InvPred;
2184 std::swap(LHS, RHS);
2185 return true;
2186 }
2187
2189 NeedInvert = true;
2190 if (isLegalFCmpPredicate(InvPred)) {
2191 Pred = InvPred;
2192 return true;
2193 }
2194 InvPred = CmpInst::getSwappedPredicate(InvPred);
2195 if (isLegalFCmpPredicate(InvPred)) {
2196 Pred = InvPred;
2197 std::swap(LHS, RHS);
2198 return true;
2199 }
2200
2201 return false;
2202}
2203
2204// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
2205// the result in DstReg.
2206// FIXME: Maybe we should expand this earlier.
2207bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI) const {
2208 auto &CmpMI = cast<GFCmp>(MI);
2209 CmpInst::Predicate Pred = CmpMI.getCond();
2210
2211 Register DstReg = CmpMI.getReg(0);
2212 Register LHS = CmpMI.getLHSReg();
2213 Register RHS = CmpMI.getRHSReg();
2214
2215 unsigned Size = MRI->getType(LHS).getSizeInBits();
2216 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
2217
2218 Register TmpReg = DstReg;
2219
2220 bool NeedInvert = false;
2221 // First try swapping operands or inverting.
2222 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
2223 if (NeedInvert)
2224 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2225 MachineInstr *Cmp = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2226 TII.get(getFCmpOpcode(Pred, Size)), TmpReg)
2228 .addReg(LHS)
2229 .addReg(RHS);
2231 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
2232 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
2233 NeedInvert = Pred == CmpInst::FCMP_UEQ;
2234 Register Cmp1Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2235 MachineInstr *Cmp1 =
2236 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2237 TII.get(getFCmpOpcode(CmpInst::FCMP_OLT, Size)), Cmp1Reg)
2239 .addReg(LHS)
2240 .addReg(RHS);
2242 Register Cmp2Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2243 MachineInstr *Cmp2 =
2244 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2245 TII.get(getFCmpOpcode(CmpInst::FCMP_OLT, Size)), Cmp2Reg)
2247 .addReg(RHS)
2248 .addReg(LHS);
2250 if (NeedInvert)
2251 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2252 MachineInstr *Or = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2253 TII.get(RISCV::OR), TmpReg)
2254 .addReg(Cmp1Reg)
2255 .addReg(Cmp2Reg);
2257 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
2258 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
2259 // If LHS and RHS are the same, a single FEQ suffices.
2260 NeedInvert = Pred == CmpInst::FCMP_UNO;
2261 if (NeedInvert)
2262 TmpReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2263 if (LHS == RHS) {
2264 MachineInstr *Cmp =
2265 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2266 TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), TmpReg)
2268 .addReg(LHS)
2269 .addReg(LHS);
2271 } else {
2272 Register Cmp1Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2273 MachineInstr *Cmp1 =
2274 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2275 TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), Cmp1Reg)
2277 .addReg(LHS)
2278 .addReg(LHS);
2280 Register Cmp2Reg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
2281 MachineInstr *Cmp2 =
2282 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2283 TII.get(getFCmpOpcode(CmpInst::FCMP_OEQ, Size)), Cmp2Reg)
2285 .addReg(RHS)
2286 .addReg(RHS);
2288 MachineInstr *And = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2289 TII.get(RISCV::AND), TmpReg)
2290 .addReg(Cmp1Reg)
2291 .addReg(Cmp2Reg);
2293 }
2294 } else
2295 llvm_unreachable("Unhandled predicate");
2296
2297 // Emit an XORI to invert the result if needed.
2298 if (NeedInvert) {
2299 MachineInstr *Xor = BuildMI(*MI.getParent(), MI, MI.getDebugLoc(),
2300 TII.get(RISCV::XORI), DstReg)
2301 .addReg(TmpReg)
2302 .addImm(1);
2304 }
2305
2306 MI.eraseFromParent();
2307 return true;
2308}
2309
2310void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
2311 SyncScope::ID FenceSSID,
2312 MachineInstr &MI) const {
2313 MachineBasicBlock &MBB = *MI.getParent();
2314 DebugLoc DL = MI.getDebugLoc();
2315
2316 if (STI.hasStdExtZtso()) {
2317 // The only fence that needs an instruction is a sequentially-consistent
2318 // cross-thread fence.
2319 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
2320 FenceSSID == SyncScope::System) {
2321 // fence rw, rw
2322 BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE))
2325 return;
2326 }
2327
2328 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
2329 BuildMI(MBB, MI, DL, TII.get(TargetOpcode::MEMBARRIER));
2330 return;
2331 }
2332
2333 // singlethread fences only synchronize with signal handlers on the same
2334 // thread and thus only need to preserve instruction order, not actually
2335 // enforce memory ordering.
2336 if (FenceSSID == SyncScope::SingleThread) {
2337 BuildMI(MBB, MI, DL, TII.get(TargetOpcode::MEMBARRIER));
2338 return;
2339 }
2340
2341 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
2342 // Manual: Volume I.
2343 unsigned Pred, Succ;
2344 switch (FenceOrdering) {
2345 default:
2346 llvm_unreachable("Unexpected ordering");
2347 case AtomicOrdering::AcquireRelease:
2348 // fence acq_rel -> fence.tso
2349 BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE_TSO));
2350 return;
2351 case AtomicOrdering::Acquire:
2352 // fence acquire -> fence r, rw
2355 break;
2356 case AtomicOrdering::Release:
2357 // fence release -> fence rw, w
2359 Succ = RISCVFenceField::W;
2360 break;
2361 case AtomicOrdering::SequentiallyConsistent:
2362 // fence seq_cst -> fence rw, rw
2365 break;
2366 }
2367 BuildMI(MBB, MI, DL, TII.get(RISCV::FENCE)).addImm(Pred).addImm(Succ);
2368}
2369
2370namespace llvm {
2371InstructionSelector *
2374 const RISCVRegisterBankInfo &RBI) {
2375 return new RISCVInstructionSelector(TM, Subtarget, RBI);
2376}
2377} // end namespace llvm
const AArch64Subtarget & Subtarget
SDValue Pred
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
uint64_t uint64_t
uint64_t Imm
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static bool hasAllWUsers(const MachineInstr &OrigMI, const LoongArchSubtarget &ST, const MachineRegisterInfo &MRI)
static bool hasAllNBitUsers(const MachineInstr &OrigMI, const LoongArchSubtarget &ST, const MachineRegisterInfo &MRI, unsigned OrigBits)
#define I(x, y, z)
Definition MD5.cpp:57
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define P(N)
static StringRef getName(Value *V)
static bool isWorthFoldingAdd(SDValue Add)
static unsigned selectRegImmLoadStoreOp(const RISCVSubtarget &STI, unsigned GenericOpc, unsigned OpSize)
Select the RISC-V regimm opcode for the G_LOAD or G_STORE operation GenericOpc on STI,...
static unsigned selectZalasrLoadStoreOp(unsigned GenericOpc, unsigned OpSize)
Select the RISC-V Zalasr opcode for the G_LOAD or G_STORE operation GenericOpc, appropriate for the G...
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS, CmpInst::Predicate &Pred, bool &NeedInvert)
static void getOperandsForBranch(Register CondReg, RISCVCC::CondCode &CC, Register &LHS, Register &RHS, MachineRegisterInfo &MRI)
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
This file declares the targeting of the RegisterBankInfo class for RISC-V.
#define LLVM_DEBUG(...)
Definition Debug.h:119
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition APFloat.h:1475
bool isPosZero() const
Definition APFloat.h:1594
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition InstrTypes.h:740
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition InstrTypes.h:743
@ ICMP_SLT
signed less than
Definition InstrTypes.h:769
@ ICMP_SLE
signed less or equal
Definition InstrTypes.h:770
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition InstrTypes.h:746
@ ICMP_UGE
unsigned greater or equal
Definition InstrTypes.h:764
@ ICMP_UGT
unsigned greater than
Definition InstrTypes.h:763
@ ICMP_SGT
signed greater than
Definition InstrTypes.h:767
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition InstrTypes.h:748
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition InstrTypes.h:751
@ ICMP_ULT
unsigned less than
Definition InstrTypes.h:765
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition InstrTypes.h:747
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition InstrTypes.h:749
@ ICMP_NE
not equal
Definition InstrTypes.h:762
@ ICMP_SGE
signed greater or equal
Definition InstrTypes.h:768
@ ICMP_ULE
unsigned less or equal
Definition InstrTypes.h:766
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition InstrTypes.h:750
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition InstrTypes.h:890
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition InstrTypes.h:852
This is an important base class in LLVM.
Definition Constant.h:44
virtual void setupMF(MachineFunction &mf, GISelValueTracking *vt, CodeGenCoverage *covinfo=nullptr, ProfileSummaryInfo *psi=nullptr, BlockFrequencyInfo *bfi=nullptr)
Setup per-MF executor state.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr bool isVector() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr unsigned getAddressSpace() const
static LLT integer(unsigned SizeInBits)
TypeSize getValue() const
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addDisp(const MachineOperand &Disp, int64_t off, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
const MachineBasicBlock * getParent() const
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
const MachineOperand & getOperand(unsigned i) const
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
AtomicOrdering getSuccessOrdering() const
Return the atomic ordering requirements for this memory operation.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
iterator_range< use_nodbg_iterator > use_nodbg_operands(Register Reg) const
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
LLVM_ABI void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
LLVM_ABI void setType(Register VReg, LLT Ty)
Set the low-level type of VReg to Ty.
LLVM_ABI Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Analysis providing profile information.
This class provides the information for the target register banks.
unsigned getXLen() const
static std::pair< unsigned, unsigned > decomposeSubvectorInsertExtractToSubRegs(MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx, const RISCVRegisterInfo *TRI)
static unsigned getRegClassIDForVecVT(MVT VT)
static RISCVVType::VLMUL getLMUL(MVT VT)
static const TargetRegisterClass * constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI)
Constrain the (possibly generic) virtual register Reg to RC.
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
unsigned getID() const
Get the identifier of this register bank.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition Register.h:83
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
bool isPositionIndependent() const
CodeModel::Model getCodeModel() const
Returns the code model.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
operand_type_match m_Reg()
SpecificConstantMatch m_SpecificICst(const APInt &RequestedValue)
Matches a constant equal to RequestedValue.
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
UnaryOp_match< SrcTy, TargetOpcode::G_INTTOPTR > m_GIntToPtr(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
SpecificImmMatch m_SpecificImm(int64_t RequestedValue)
Matches an immediate operand equal to RequestedValue.
AllOnesConstantMatch m_AllOnes()
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
GFrameIndexMatch m_GFrameIndex(int &FI)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
OneUse_match< SubPat > m_OneUse(const SubPat &SP)
SrcImmOp_match< SrcTy, AnyImmMatch, TargetOpcode::G_SEXT_INREG > m_GSExtInReg(const SrcTy &Src)
Matches a G_SEXT_INREG, binding its source and immediate width.
unsigned getBrCond(CondCode CC, unsigned SelectOpc=0)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
SmallVector< Inst, 8 > InstSeq
Definition RISCVMatInt.h:43
static unsigned decodeVSEW(unsigned VSEW)
LLVM_ABI unsigned getSEWLMULRatio(unsigned SEW, VLMUL VLMul)
LLVM_ABI unsigned encodeVTYPE(VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic, bool AltFmt=false)
static constexpr int64_t VLMaxSentinel
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:63
@ System
Synchronized with respect to all concurrently executing threads.
Definition LLVMContext.h:66
@ User
could "use" a pointer
BaseReg
Stack frame base register. Bit 0 of FREInfo.Info.
Definition SFrame.h:77
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
PointerUnion< const TargetRegisterClass *, const RegisterBank * > RegClassOrRegBank
Convenient type to represent either a register class or a register bank.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
@ Known
Known to have no common set bits.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
bool isStrongerThanMonotonic(AtomicOrdering AO)
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition bit.h:315
LLVM_ABI void constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition Utils.cpp:163
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
LLVM_ABI MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
InstructionSelector * createRISCVInstructionSelector(const RISCVTargetMachine &TM, const RISCVSubtarget &Subtarget, const RISCVRegisterBankInfo &RBI)
LLVM_ABI std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition Utils.cpp:321
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition bit.h:204
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void reportGISelFailure(MachineFunction &MF, MachineOptimizationRemarkEmitter &MORE, MachineOptimizationRemarkMissed &R)
Report an ISel error as a missed optimization remark to the LLVMContext's diagnostic stream.
Definition Utils.cpp:265
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T maskTrailingZeros(unsigned N)
Create a bitmask with the N right-most bits set to 0, and all other bits set to 1.
Definition MathExtras.h:95
@ Or
Bitwise or logical OR of integers.
@ Xor
Bitwise or logical XOR of integers.
@ And
Bitwise or logical AND of integers.
@ Add
Sum of integers.
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition Utils.cpp:440
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
constexpr T maskTrailingOnes(unsigned N)
Create a bitmask with the N right-most bits set to 1, and all other bits set to 0.
Definition MathExtras.h:78
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define MORE()
Definition regcomp.c:247
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.