LLVM 20.0.0git
RISCVInstructionSelector.cpp
Go to the documentation of this file.
1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46 static const char *getName() { return DEBUG_TYPE; }
47
48private:
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
51
52 bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
53 bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
54
55 // tblgen-erated 'select' implementation, used as the initial selector for
56 // the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
58
59 // A lowering phase that runs before any selection attempts.
60 // Returns true if the instruction was modified.
61 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
63
64 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
66
67 // Custom selection methods
69 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
71 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
72 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
73 MachineRegisterInfo &MRI, bool IsLocal = true,
74 bool IsExternWeak = false) const;
75 bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
76 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
78 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
80 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
81 MachineIRBuilder &MIB) const;
86
87 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
88 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
89
90 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
91 template <unsigned ShAmt>
92 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
93 return selectSHXADDOp(Root, ShAmt);
94 }
95
96 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
97 unsigned ShAmt) const;
98 template <unsigned ShAmt>
99 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
100 return selectSHXADD_UWOp(Root, ShAmt);
101 }
102
103 // Custom renderers for tablegen
104 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
105 int OpIdx) const;
106 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
107 int OpIdx) const;
108 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
109 int OpIdx) const;
110 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
111 int OpIdx) const;
112 void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
113 int OpIdx) const;
114
115 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
116 int OpIdx) const;
117
118 const RISCVSubtarget &STI;
119 const RISCVInstrInfo &TII;
120 const RISCVRegisterInfo &TRI;
121 const RISCVRegisterBankInfo &RBI;
122 const RISCVTargetMachine &TM;
123
124 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
125 // uses "STI." in the code generated by TableGen. We need to unify the name of
126 // Subtarget variable.
127 const RISCVSubtarget *Subtarget = &STI;
128
129#define GET_GLOBALISEL_PREDICATES_DECL
130#include "RISCVGenGlobalISel.inc"
131#undef GET_GLOBALISEL_PREDICATES_DECL
132
133#define GET_GLOBALISEL_TEMPORARIES_DECL
134#include "RISCVGenGlobalISel.inc"
135#undef GET_GLOBALISEL_TEMPORARIES_DECL
136};
137
138} // end anonymous namespace
139
140#define GET_GLOBALISEL_IMPL
141#include "RISCVGenGlobalISel.inc"
142#undef GET_GLOBALISEL_IMPL
143
144RISCVInstructionSelector::RISCVInstructionSelector(
145 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
146 const RISCVRegisterBankInfo &RBI)
147 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
148 TM(TM),
149
151#include "RISCVGenGlobalISel.inc"
154#include "RISCVGenGlobalISel.inc"
156{
157}
158
160RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
161 if (!Root.isReg())
162 return std::nullopt;
163
164 using namespace llvm::MIPatternMatch;
165 MachineRegisterInfo &MRI = MF->getRegInfo();
166
167 Register RootReg = Root.getReg();
168 Register ShAmtReg = RootReg;
169 const LLT ShiftLLT = MRI.getType(RootReg);
170 unsigned ShiftWidth = ShiftLLT.getSizeInBits();
171 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
172 // Peek through zext.
173 Register ZExtSrcReg;
174 if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
175 ShAmtReg = ZExtSrcReg;
176 }
177
178 APInt AndMask;
179 Register AndSrcReg;
180 // Try to combine the following pattern (applicable to other shift
181 // instructions as well as 32-bit ones):
182 //
183 // %4:gprb(s64) = G_AND %3, %2
184 // %5:gprb(s64) = G_LSHR %1, %4(s64)
185 //
186 // According to RISC-V's ISA manual, SLL, SRL, and SRA ignore other bits than
187 // the lowest log2(XLEN) bits of register rs2. As for the above pattern, if
188 // the lowest log2(XLEN) bits of register rd and rs2 of G_AND are the same,
189 // then it can be eliminated. Given register rs1 or rs2 holding a constant
190 // (the and mask), there are two cases G_AND can be erased:
191 //
192 // 1. the lowest log2(XLEN) bits of the and mask are all set
193 // 2. the bits of the register being masked are already unset (zero set)
194 if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
195 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
196 if (ShMask.isSubsetOf(AndMask)) {
197 ShAmtReg = AndSrcReg;
198 } else {
199 // SimplifyDemandedBits may have optimized the mask so try restoring any
200 // bits that are known zero.
201 KnownBits Known = KB->getKnownBits(AndSrcReg);
202 if (ShMask.isSubsetOf(AndMask | Known.Zero))
203 ShAmtReg = AndSrcReg;
204 }
205 }
206
207 APInt Imm;
209 if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
210 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
211 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
212 // to avoid the ADD.
213 ShAmtReg = Reg;
214 } else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
215 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
216 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
217 // to generate a NEG instead of a SUB of a constant.
218 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
219 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
220 return {{[=](MachineInstrBuilder &MIB) {
221 MachineIRBuilder(*MIB.getInstr())
222 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
223 MIB.addReg(ShAmtReg);
224 }}};
225 }
226 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
227 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
228 // to generate a NOT instead of a SUB of a constant.
229 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
230 return {{[=](MachineInstrBuilder &MIB) {
231 MachineIRBuilder(*MIB.getInstr())
232 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
233 .addImm(-1);
234 MIB.addReg(ShAmtReg);
235 }}};
236 }
237 }
238
239 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
240}
241
243RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
244 unsigned ShAmt) const {
245 using namespace llvm::MIPatternMatch;
246 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
248
249 if (!Root.isReg())
250 return std::nullopt;
251 Register RootReg = Root.getReg();
252
253 const unsigned XLen = STI.getXLen();
254 APInt Mask, C2;
255 Register RegY;
256 std::optional<bool> LeftShift;
257 // (and (shl y, c2), mask)
258 if (mi_match(RootReg, MRI,
259 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
260 LeftShift = true;
261 // (and (lshr y, c2), mask)
262 else if (mi_match(RootReg, MRI,
263 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
264 LeftShift = false;
265
266 if (LeftShift.has_value()) {
267 if (*LeftShift)
268 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
269 else
270 Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
271
272 if (Mask.isShiftedMask()) {
273 unsigned Leading = XLen - Mask.getActiveBits();
274 unsigned Trailing = Mask.countr_zero();
275 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
276 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
277 if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
278 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
279 return {{[=](MachineInstrBuilder &MIB) {
280 MachineIRBuilder(*MIB.getInstr())
281 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
282 .addImm(Trailing - C2.getLimitedValue());
283 MIB.addReg(DstReg);
284 }}};
285 }
286
287 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
288 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
289 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
290 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
291 return {{[=](MachineInstrBuilder &MIB) {
292 MachineIRBuilder(*MIB.getInstr())
293 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
294 .addImm(Leading + Trailing);
295 MIB.addReg(DstReg);
296 }}};
297 }
298 }
299 }
300
301 LeftShift.reset();
302
303 // (shl (and y, mask), c2)
304 if (mi_match(RootReg, MRI,
305 m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
306 m_ICst(C2))))
307 LeftShift = true;
308 // (lshr (and y, mask), c2)
309 else if (mi_match(RootReg, MRI,
311 m_ICst(C2))))
312 LeftShift = false;
313
314 if (LeftShift.has_value() && Mask.isShiftedMask()) {
315 unsigned Leading = XLen - Mask.getActiveBits();
316 unsigned Trailing = Mask.countr_zero();
317
318 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
319 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
320 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
321 (Trailing + C2.getLimitedValue()) == ShAmt;
322 if (!Cond)
323 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
324 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
325 Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
326 (Trailing - C2.getLimitedValue()) == ShAmt;
327
328 if (Cond) {
329 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
330 return {{[=](MachineInstrBuilder &MIB) {
331 MachineIRBuilder(*MIB.getInstr())
332 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
333 .addImm(Trailing);
334 MIB.addReg(DstReg);
335 }}};
336 }
337 }
338
339 return std::nullopt;
340}
341
343RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
344 unsigned ShAmt) const {
345 using namespace llvm::MIPatternMatch;
346 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
348
349 if (!Root.isReg())
350 return std::nullopt;
351 Register RootReg = Root.getReg();
352
353 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
354 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
355 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
356 APInt Mask, C2;
357 Register RegX;
358 if (mi_match(
359 RootReg, MRI,
361 m_ICst(Mask))))) {
362 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
363
364 if (Mask.isShiftedMask()) {
365 unsigned Leading = Mask.countl_zero();
366 unsigned Trailing = Mask.countr_zero();
367 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
368 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
369 return {{[=](MachineInstrBuilder &MIB) {
370 MachineIRBuilder(*MIB.getInstr())
371 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
372 .addImm(C2.getLimitedValue() - ShAmt);
373 MIB.addReg(DstReg);
374 }}};
375 }
376 }
377 }
378
379 return std::nullopt;
380}
381
383RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
384 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
386
387 if (!Root.isReg())
388 return std::nullopt;
389
390 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
391 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
392 return {{
393 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
394 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
395 }};
396 }
397
398 if (isBaseWithConstantOffset(Root, MRI)) {
399 MachineOperand &LHS = RootDef->getOperand(1);
400 MachineOperand &RHS = RootDef->getOperand(2);
401 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
402 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
403
404 int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
405 if (isInt<12>(RHSC)) {
406 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
407 return {{
408 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
409 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
410 }};
411
412 return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
413 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
414 }
415 }
416
417 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
418 // the combiner?
419 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
420 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
421}
422
423/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
424/// CC Must be an ICMP Predicate.
425static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
426 switch (CC) {
427 default:
428 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
429 case CmpInst::Predicate::ICMP_EQ:
430 return RISCVCC::COND_EQ;
431 case CmpInst::Predicate::ICMP_NE:
432 return RISCVCC::COND_NE;
433 case CmpInst::Predicate::ICMP_ULT:
434 return RISCVCC::COND_LTU;
435 case CmpInst::Predicate::ICMP_SLT:
436 return RISCVCC::COND_LT;
437 case CmpInst::Predicate::ICMP_UGE:
438 return RISCVCC::COND_GEU;
439 case CmpInst::Predicate::ICMP_SGE:
440 return RISCVCC::COND_GE;
441 }
442}
443
446 Register &RHS) {
447 // Try to fold an ICmp. If that fails, use a NE compare with X0.
449 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
450 LHS = CondReg;
451 RHS = RISCV::X0;
453 return;
454 }
455
456 // We found an ICmp, do some canonicalizations.
457
458 // Adjust comparisons to use comparison with 0 if possible.
460 switch (Pred) {
461 case CmpInst::Predicate::ICMP_SGT:
462 // Convert X > -1 to X >= 0
463 if (*Constant == -1) {
465 RHS = RISCV::X0;
466 return;
467 }
468 break;
469 case CmpInst::Predicate::ICMP_SLT:
470 // Convert X < 1 to 0 >= X
471 if (*Constant == 1) {
473 RHS = LHS;
474 LHS = RISCV::X0;
475 return;
476 }
477 break;
478 default:
479 break;
480 }
481 }
482
483 switch (Pred) {
484 default:
485 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
486 case CmpInst::Predicate::ICMP_EQ:
487 case CmpInst::Predicate::ICMP_NE:
488 case CmpInst::Predicate::ICMP_ULT:
489 case CmpInst::Predicate::ICMP_SLT:
490 case CmpInst::Predicate::ICMP_UGE:
491 case CmpInst::Predicate::ICMP_SGE:
492 // These CCs are supported directly by RISC-V branches.
493 break;
494 case CmpInst::Predicate::ICMP_SGT:
495 case CmpInst::Predicate::ICMP_SLE:
496 case CmpInst::Predicate::ICMP_UGT:
497 case CmpInst::Predicate::ICMP_ULE:
498 // These CCs are not supported directly by RISC-V branches, but changing the
499 // direction of the CC and swapping LHS and RHS are.
500 Pred = CmpInst::getSwappedPredicate(Pred);
501 std::swap(LHS, RHS);
502 break;
503 }
504
505 CC = getRISCVCCFromICmp(Pred);
506 return;
507}
508
509bool RISCVInstructionSelector::select(MachineInstr &MI) {
510 MachineBasicBlock &MBB = *MI.getParent();
513 MachineIRBuilder MIB(MI);
514
515 preISelLower(MI, MIB, MRI);
516 const unsigned Opc = MI.getOpcode();
517
518 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
519 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
520 const Register DefReg = MI.getOperand(0).getReg();
521 const LLT DefTy = MRI.getType(DefReg);
522
523 const RegClassOrRegBank &RegClassOrBank =
524 MRI.getRegClassOrRegBank(DefReg);
525
526 const TargetRegisterClass *DefRC =
527 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
528 if (!DefRC) {
529 if (!DefTy.isValid()) {
530 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
531 return false;
532 }
533
534 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
535 DefRC = getRegClassForTypeOnBank(DefTy, RB);
536 if (!DefRC) {
537 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
538 return false;
539 }
540 }
541
542 MI.setDesc(TII.get(TargetOpcode::PHI));
543 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
544 }
545
546 // Certain non-generic instructions also need some special handling.
547 if (MI.isCopy())
548 return selectCopy(MI, MRI);
549
550 return true;
551 }
552
553 if (selectImpl(MI, *CoverageInfo))
554 return true;
555
556 switch (Opc) {
557 case TargetOpcode::G_ANYEXT:
558 case TargetOpcode::G_PTRTOINT:
559 case TargetOpcode::G_INTTOPTR:
560 case TargetOpcode::G_TRUNC:
561 case TargetOpcode::G_FREEZE:
562 return selectCopy(MI, MRI);
563 case TargetOpcode::G_CONSTANT: {
564 Register DstReg = MI.getOperand(0).getReg();
565 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
566
567 if (!materializeImm(DstReg, Imm, MIB))
568 return false;
569
570 MI.eraseFromParent();
571 return true;
572 }
573 case TargetOpcode::G_FCONSTANT: {
574 // TODO: Use constant pool for complext constants.
575 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
576 Register DstReg = MI.getOperand(0).getReg();
577 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
578 APInt Imm = FPimm.bitcastToAPInt();
579 unsigned Size = MRI.getType(DstReg).getSizeInBits();
580 if (Size == 16 || Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
581 Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
582 if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
583 return false;
584
585 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X
586 : Size == 32 ? RISCV::FMV_W_X
587 : RISCV::FMV_H_X;
588 auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
589 if (!FMV.constrainAllUses(TII, TRI, RBI))
590 return false;
591 } else {
592 assert(Size == 64 && !Subtarget->is64Bit() &&
593 "Unexpected size or subtarget");
594 // Split into two pieces and build through the stack.
595 Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
596 Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
597 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
598 MIB))
599 return false;
600 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
601 return false;
602 MachineInstrBuilder PairF64 = MIB.buildInstr(
603 RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
604 if (!PairF64.constrainAllUses(TII, TRI, RBI))
605 return false;
606 }
607
608 MI.eraseFromParent();
609 return true;
610 }
611 case TargetOpcode::G_GLOBAL_VALUE: {
612 auto *GV = MI.getOperand(1).getGlobal();
613 if (GV->isThreadLocal()) {
614 // TODO: implement this case.
615 return false;
616 }
617
618 return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
619 GV->hasExternalWeakLinkage());
620 }
621 case TargetOpcode::G_JUMP_TABLE:
622 case TargetOpcode::G_CONSTANT_POOL:
623 return selectAddr(MI, MIB, MRI);
624 case TargetOpcode::G_BRCOND: {
627 getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
628
629 auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
630 .addMBB(MI.getOperand(1).getMBB());
631 MI.eraseFromParent();
632 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
633 }
634 case TargetOpcode::G_BRJT: {
635 // FIXME: Move to legalization?
636 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
637 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
638 assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
639 "Unsupported jump-table entry size");
640 assert(
644 "Unexpected jump-table entry kind");
645
646 auto SLL =
647 MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {MI.getOperand(2)})
648 .addImm(Log2_32(EntrySize));
649 if (!SLL.constrainAllUses(TII, TRI, RBI))
650 return false;
651
652 // TODO: Use SHXADD. Moving to legalization would fix this automatically.
653 auto ADD = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
654 {MI.getOperand(0), SLL.getReg(0)});
655 if (!ADD.constrainAllUses(TII, TRI, RBI))
656 return false;
657
658 unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
659 auto Dest =
660 MIB.buildInstr(LdOpc, {&RISCV::GPRRegClass}, {ADD.getReg(0)})
661 .addImm(0)
662 .addMemOperand(MF.getMachineMemOperand(
664 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout()))));
665 if (!Dest.constrainAllUses(TII, TRI, RBI))
666 return false;
667
668 // If the Kind is EK_LabelDifference32, the table stores an offset from
669 // the location of the table. Add the table address to get an absolute
670 // address.
672 Dest = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
673 {Dest.getReg(0), MI.getOperand(0)});
674 if (!Dest.constrainAllUses(TII, TRI, RBI))
675 return false;
676 }
677
678 auto Branch =
679 MIB.buildInstr(RISCV::PseudoBRIND, {}, {Dest.getReg(0)}).addImm(0);
680 if (!Branch.constrainAllUses(TII, TRI, RBI))
681 return false;
682
683 MI.eraseFromParent();
684 return true;
685 }
686 case TargetOpcode::G_BRINDIRECT:
687 MI.setDesc(TII.get(RISCV::PseudoBRIND));
688 MI.addOperand(MachineOperand::CreateImm(0));
690 case TargetOpcode::G_SEXT_INREG:
691 return selectSExtInreg(MI, MIB);
692 case TargetOpcode::G_FRAME_INDEX: {
693 // TODO: We may want to replace this code with the SelectionDAG patterns,
694 // which fail to get imported because it uses FrameAddrRegImm, which is a
695 // ComplexPattern
696 MI.setDesc(TII.get(RISCV::ADDI));
697 MI.addOperand(MachineOperand::CreateImm(0));
699 }
700 case TargetOpcode::G_SELECT:
701 return selectSelect(MI, MIB, MRI);
702 case TargetOpcode::G_FCMP:
703 return selectFPCompare(MI, MIB, MRI);
704 case TargetOpcode::G_FENCE: {
705 AtomicOrdering FenceOrdering =
706 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
707 SyncScope::ID FenceSSID =
708 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
709 emitFence(FenceOrdering, FenceSSID, MIB);
710 MI.eraseFromParent();
711 return true;
712 }
713 case TargetOpcode::G_IMPLICIT_DEF:
714 return selectImplicitDef(MI, MIB, MRI);
715 case TargetOpcode::G_MERGE_VALUES:
716 return selectMergeValues(MI, MIB, MRI);
717 case TargetOpcode::G_UNMERGE_VALUES:
718 return selectUnmergeValues(MI, MIB, MRI);
719 default:
720 return false;
721 }
722}
723
724bool RISCVInstructionSelector::selectMergeValues(
726 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
727
728 // Build a F64 Pair from operands
729 if (MI.getNumOperands() != 3)
730 return false;
731 Register Dst = MI.getOperand(0).getReg();
732 Register Lo = MI.getOperand(1).getReg();
733 Register Hi = MI.getOperand(2).getReg();
734 if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
735 return false;
736 MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
738}
739
740bool RISCVInstructionSelector::selectUnmergeValues(
742 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
743
744 // Split F64 Src into two s32 parts
745 if (MI.getNumOperands() != 3)
746 return false;
747 Register Src = MI.getOperand(2).getReg();
748 Register Lo = MI.getOperand(0).getReg();
749 Register Hi = MI.getOperand(1).getReg();
750 if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
751 return false;
752 MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
754}
755
756bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
757 MachineIRBuilder &MIB,
759 Register PtrReg = Op.getReg();
760 assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
761
762 const LLT sXLen = LLT::scalar(STI.getXLen());
763 auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
764 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
765 Op.setReg(PtrToInt.getReg(0));
766 return select(*PtrToInt);
767}
768
769void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
770 MachineIRBuilder &MIB,
772 switch (MI.getOpcode()) {
773 case TargetOpcode::G_PTR_ADD: {
774 Register DstReg = MI.getOperand(0).getReg();
775 const LLT sXLen = LLT::scalar(STI.getXLen());
776
777 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
778 MI.setDesc(TII.get(TargetOpcode::G_ADD));
779 MRI.setType(DstReg, sXLen);
780 break;
781 }
782 case TargetOpcode::G_PTRMASK: {
783 Register DstReg = MI.getOperand(0).getReg();
784 const LLT sXLen = LLT::scalar(STI.getXLen());
785 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
786 MI.setDesc(TII.get(TargetOpcode::G_AND));
787 MRI.setType(DstReg, sXLen);
788 }
789 }
790}
791
792void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
793 const MachineInstr &MI,
794 int OpIdx) const {
795 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
796 "Expected G_CONSTANT");
797 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
798 MIB.addImm(-CstVal);
799}
800
801void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
802 const MachineInstr &MI,
803 int OpIdx) const {
804 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
805 "Expected G_CONSTANT");
806 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
807 MIB.addImm(STI.getXLen() - CstVal);
808}
809
810void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
811 const MachineInstr &MI,
812 int OpIdx) const {
813 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
814 "Expected G_CONSTANT");
815 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
816 MIB.addImm(32 - CstVal);
817}
818
819void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
820 const MachineInstr &MI,
821 int OpIdx) const {
822 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
823 "Expected G_CONSTANT");
824 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
825 MIB.addImm(CstVal + 1);
826}
827
828void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
829 const MachineInstr &MI,
830 int OpIdx) const {
831 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
832 "Expected G_CONSTANT");
833 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
834 MIB.addImm(CstVal);
835}
836
837void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
838 const MachineInstr &MI,
839 int OpIdx) const {
840 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
841 "Expected G_CONSTANT");
842 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
844}
845
846const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
847 LLT Ty, const RegisterBank &RB) const {
848 if (RB.getID() == RISCV::GPRBRegBankID) {
849 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
850 return &RISCV::GPRRegClass;
851 }
852
853 if (RB.getID() == RISCV::FPRBRegBankID) {
854 if (Ty.getSizeInBits() == 16)
855 return &RISCV::FPR16RegClass;
856 if (Ty.getSizeInBits() == 32)
857 return &RISCV::FPR32RegClass;
858 if (Ty.getSizeInBits() == 64)
859 return &RISCV::FPR64RegClass;
860 }
861
862 if (RB.getID() == RISCV::VRBRegBankID) {
863 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
864 return &RISCV::VRRegClass;
865
866 if (Ty.getSizeInBits().getKnownMinValue() == 128)
867 return &RISCV::VRM2RegClass;
868
869 if (Ty.getSizeInBits().getKnownMinValue() == 256)
870 return &RISCV::VRM4RegClass;
871
872 if (Ty.getSizeInBits().getKnownMinValue() == 512)
873 return &RISCV::VRM8RegClass;
874 }
875
876 return nullptr;
877}
878
879bool RISCVInstructionSelector::isRegInGprb(Register Reg,
880 MachineRegisterInfo &MRI) const {
881 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
882}
883
884bool RISCVInstructionSelector::isRegInFprb(Register Reg,
885 MachineRegisterInfo &MRI) const {
886 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
887}
888
889bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
890 MachineRegisterInfo &MRI) const {
891 Register DstReg = MI.getOperand(0).getReg();
892
893 if (DstReg.isPhysical())
894 return true;
895
896 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
897 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
898 assert(DstRC &&
899 "Register class not available for LLT, register bank combination");
900
901 // No need to constrain SrcReg. It will get constrained when
902 // we hit another of its uses or its defs.
903 // Copies do not have constraints.
904 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
905 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
906 << " operand\n");
907 return false;
908 }
909
910 MI.setDesc(TII.get(RISCV::COPY));
911 return true;
912}
913
914bool RISCVInstructionSelector::selectImplicitDef(
916 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
917
918 const Register DstReg = MI.getOperand(0).getReg();
919 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
920 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
921
922 assert(DstRC &&
923 "Register class not available for LLT, register bank combination");
924
925 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
926 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
927 << " operand\n");
928 }
929 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
930 return true;
931}
932
933bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
934 MachineIRBuilder &MIB) const {
936
937 if (Imm == 0) {
938 MIB.buildCopy(DstReg, Register(RISCV::X0));
939 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
940 return true;
941 }
942
944 unsigned NumInsts = Seq.size();
945 Register SrcReg = RISCV::X0;
946
947 for (unsigned i = 0; i < NumInsts; i++) {
948 Register TmpReg = i < NumInsts - 1
949 ? MRI.createVirtualRegister(&RISCV::GPRRegClass)
950 : DstReg;
951 const RISCVMatInt::Inst &I = Seq[i];
953
954 switch (I.getOpndKind()) {
955 case RISCVMatInt::Imm:
956 // clang-format off
957 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
958 .addImm(I.getImm());
959 // clang-format on
960 break;
962 Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
963 {SrcReg, Register(RISCV::X0)});
964 break;
966 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
967 break;
969 Result =
970 MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
971 break;
972 }
973
974 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
975 return false;
976
977 SrcReg = TmpReg;
978 }
979
980 return true;
981}
982
983bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
984 MachineIRBuilder &MIB,
986 bool IsLocal,
987 bool IsExternWeak) const {
988 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
989 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
990 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
991 "Unexpected opcode");
992
993 const MachineOperand &DispMO = MI.getOperand(1);
994
995 Register DefReg = MI.getOperand(0).getReg();
996 const LLT DefTy = MRI.getType(DefReg);
997
998 // When HWASAN is used and tagging of global variables is enabled
999 // they should be accessed via the GOT, since the tagged address of a global
1000 // is incompatible with existing code models. This also applies to non-pic
1001 // mode.
1002 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
1003 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
1004 // Use PC-relative addressing to access the symbol. This generates the
1005 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
1006 // %pcrel_lo(auipc)).
1007 MI.setDesc(TII.get(RISCV::PseudoLLA));
1009 }
1010
1011 // Use PC-relative addressing to access the GOT for this symbol, then
1012 // load the address from the GOT. This generates the pattern (PseudoLGA
1013 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
1014 // %pcrel_lo(auipc))).
1015 MachineFunction &MF = *MI.getParent()->getParent();
1020 DefTy, Align(DefTy.getSizeInBits() / 8));
1021
1022 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1023 .addDisp(DispMO, 0)
1025
1026 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1027 return false;
1028
1029 MI.eraseFromParent();
1030 return true;
1031 }
1032
1033 switch (TM.getCodeModel()) {
1034 default: {
1035 reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1036 getName(), "Unsupported code model for lowering", MI);
1037 return false;
1038 }
1039 case CodeModel::Small: {
1040 // Must lie within a single 2 GiB address range and must lie between
1041 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1042 // (lui %hi(sym)) %lo(sym)).
1043 Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1044 MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1045 .addDisp(DispMO, 0, RISCVII::MO_HI);
1046
1047 if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1048 return false;
1049
1050 auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1051 .addDisp(DispMO, 0, RISCVII::MO_LO);
1052
1053 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1054 return false;
1055
1056 MI.eraseFromParent();
1057 return true;
1058 }
1059 case CodeModel::Medium:
1060 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1061 // relocation needs to reference a label that points to the auipc
1062 // instruction itself, not the global. This cannot be done inside the
1063 // instruction selector.
1064 if (IsExternWeak) {
1065 // An extern weak symbol may be undefined, i.e. have value 0, which may
1066 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1067 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1068 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1069 MachineFunction &MF = *MI.getParent()->getParent();
1074 DefTy, Align(DefTy.getSizeInBits() / 8));
1075
1076 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1077 .addDisp(DispMO, 0)
1079
1080 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1081 return false;
1082
1083 MI.eraseFromParent();
1084 return true;
1085 }
1086
1087 // Generate a sequence for accessing addresses within any 2GiB range
1088 // within the address space. This generates the pattern (PseudoLLA sym),
1089 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1090 MI.setDesc(TII.get(RISCV::PseudoLLA));
1092 }
1093
1094 return false;
1095}
1096
1097bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
1098 MachineIRBuilder &MIB) const {
1099 Register DstReg = MI.getOperand(0).getReg();
1100 Register SrcReg = MI.getOperand(1).getReg();
1101 unsigned SrcSize = MI.getOperand(2).getImm();
1102
1103 MachineInstr *NewMI;
1104 if (SrcSize == 32) {
1105 assert(Subtarget->is64Bit() && "Unexpected extend");
1106 // addiw rd, rs, 0 (i.e. sext.w rd, rs)
1107 NewMI = MIB.buildInstr(RISCV::ADDIW, {DstReg}, {SrcReg}).addImm(0U);
1108 } else {
1109 assert(Subtarget->hasStdExtZbb() && "Unexpected extension");
1110 assert((SrcSize == 8 || SrcSize == 16) && "Unexpected size");
1111 unsigned Opc = SrcSize == 16 ? RISCV::SEXT_H : RISCV::SEXT_B;
1112 NewMI = MIB.buildInstr(Opc, {DstReg}, {SrcReg});
1113 }
1114
1115 if (!constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI))
1116 return false;
1117
1118 MI.eraseFromParent();
1119 return true;
1120}
1121
1122bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1123 MachineIRBuilder &MIB,
1124 MachineRegisterInfo &MRI) const {
1125 auto &SelectMI = cast<GSelect>(MI);
1126
1127 Register LHS, RHS;
1129 getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
1130
1131 Register DstReg = SelectMI.getReg(0);
1132
1133 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1134 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1135 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1136 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1137 : RISCV::Select_FPR64_Using_CC_GPR;
1138 }
1139
1140 MachineInstr *Result = MIB.buildInstr(Opc)
1141 .addDef(DstReg)
1142 .addReg(LHS)
1143 .addReg(RHS)
1144 .addImm(CC)
1145 .addReg(SelectMI.getTrueReg())
1146 .addReg(SelectMI.getFalseReg());
1147 MI.eraseFromParent();
1148 return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1149}
1150
1151// Convert an FCMP predicate to one of the supported F or D instructions.
1152static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1153 assert((Size == 16 || Size == 32 || Size == 64) && "Unsupported size");
1154 switch (Pred) {
1155 default:
1156 llvm_unreachable("Unsupported predicate");
1157 case CmpInst::FCMP_OLT:
1158 return Size == 16 ? RISCV::FLT_H : Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1159 case CmpInst::FCMP_OLE:
1160 return Size == 16 ? RISCV::FLE_H : Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1161 case CmpInst::FCMP_OEQ:
1162 return Size == 16 ? RISCV::FEQ_H : Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1163 }
1164}
1165
1166// Try legalizing an FCMP by swapping or inverting the predicate to one that
1167// is supported.
1169 CmpInst::Predicate &Pred, bool &NeedInvert) {
1170 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1171 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1172 Pred == CmpInst::FCMP_OEQ;
1173 };
1174
1175 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1176
1178 if (isLegalFCmpPredicate(InvPred)) {
1179 Pred = InvPred;
1180 std::swap(LHS, RHS);
1181 return true;
1182 }
1183
1184 InvPred = CmpInst::getInversePredicate(Pred);
1185 NeedInvert = true;
1186 if (isLegalFCmpPredicate(InvPred)) {
1187 Pred = InvPred;
1188 return true;
1189 }
1190 InvPred = CmpInst::getSwappedPredicate(InvPred);
1191 if (isLegalFCmpPredicate(InvPred)) {
1192 Pred = InvPred;
1193 std::swap(LHS, RHS);
1194 return true;
1195 }
1196
1197 return false;
1198}
1199
1200// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1201// the result in DstReg.
1202// FIXME: Maybe we should expand this earlier.
1203bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1204 MachineIRBuilder &MIB,
1205 MachineRegisterInfo &MRI) const {
1206 auto &CmpMI = cast<GFCmp>(MI);
1207 CmpInst::Predicate Pred = CmpMI.getCond();
1208
1209 Register DstReg = CmpMI.getReg(0);
1210 Register LHS = CmpMI.getLHSReg();
1211 Register RHS = CmpMI.getRHSReg();
1212
1213 unsigned Size = MRI.getType(LHS).getSizeInBits();
1214 assert((Size == 16 || Size == 32 || Size == 64) && "Unexpected size");
1215
1216 Register TmpReg = DstReg;
1217
1218 bool NeedInvert = false;
1219 // First try swapping operands or inverting.
1220 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1221 if (NeedInvert)
1222 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1223 auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1224 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1225 return false;
1226 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1227 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1228 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1230 {&RISCV::GPRRegClass}, {LHS, RHS});
1231 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1232 return false;
1234 {&RISCV::GPRRegClass}, {RHS, LHS});
1235 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1236 return false;
1237 if (NeedInvert)
1238 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1239 auto Or =
1240 MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1241 if (!Or.constrainAllUses(TII, TRI, RBI))
1242 return false;
1243 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1244 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1245 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1246 NeedInvert = Pred == CmpInst::FCMP_UNO;
1248 {&RISCV::GPRRegClass}, {LHS, LHS});
1249 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1250 return false;
1252 {&RISCV::GPRRegClass}, {RHS, RHS});
1253 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1254 return false;
1255 if (NeedInvert)
1256 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1257 auto And =
1258 MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1259 if (!And.constrainAllUses(TII, TRI, RBI))
1260 return false;
1261 } else
1262 llvm_unreachable("Unhandled predicate");
1263
1264 // Emit an XORI to invert the result if needed.
1265 if (NeedInvert) {
1266 auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1267 if (!Xor.constrainAllUses(TII, TRI, RBI))
1268 return false;
1269 }
1270
1271 MI.eraseFromParent();
1272 return true;
1273}
1274
1275void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1276 SyncScope::ID FenceSSID,
1277 MachineIRBuilder &MIB) const {
1278 if (STI.hasStdExtZtso()) {
1279 // The only fence that needs an instruction is a sequentially-consistent
1280 // cross-thread fence.
1281 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1282 FenceSSID == SyncScope::System) {
1283 // fence rw, rw
1284 MIB.buildInstr(RISCV::FENCE, {}, {})
1287 return;
1288 }
1289
1290 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1291 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1292 return;
1293 }
1294
1295 // singlethread fences only synchronize with signal handlers on the same
1296 // thread and thus only need to preserve instruction order, not actually
1297 // enforce memory ordering.
1298 if (FenceSSID == SyncScope::SingleThread) {
1299 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1300 return;
1301 }
1302
1303 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1304 // Manual: Volume I.
1305 unsigned Pred, Succ;
1306 switch (FenceOrdering) {
1307 default:
1308 llvm_unreachable("Unexpected ordering");
1309 case AtomicOrdering::AcquireRelease:
1310 // fence acq_rel -> fence.tso
1311 MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1312 return;
1313 case AtomicOrdering::Acquire:
1314 // fence acquire -> fence r, rw
1315 Pred = RISCVFenceField::R;
1317 break;
1318 case AtomicOrdering::Release:
1319 // fence release -> fence rw, w
1321 Succ = RISCVFenceField::W;
1322 break;
1323 case AtomicOrdering::SequentiallyConsistent:
1324 // fence seq_cst -> fence rw, rw
1327 break;
1328 }
1329 MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1330}
1331
1332namespace llvm {
1335 const RISCVSubtarget &Subtarget,
1336 const RISCVRegisterBankInfo &RBI) {
1337 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1338}
1339} // end namespace llvm
unsigned const MachineRegisterInfo * MRI
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static StringRef getName(Value *V)
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI, RISCVCC::CondCode &CC, Register &LHS, Register &RHS)
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS, CmpInst::Predicate &Pred, bool &NeedInvert)
const SmallVectorImpl< MachineOperand > & Cond
This file declares the targeting of the RegisterBankInfo class for RISC-V.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
APInt bitcastToAPInt() const
Definition: APFloat.h:1266
Class for arbitrary precision integers.
Definition: APInt.h:78
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1446
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1089
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:453
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:757
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:760
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:763
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:765
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:768
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:764
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:766
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:767
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:909
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:871
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:161
This is an important base class in LLVM.
Definition: Constant.h:42
This class represents an Operation in the Expression.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
virtual bool select(MachineInstr &I)=0
Select the (possibly generic) instruction I to only use target-specific opcodes.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isValid() const
Definition: LowLevelType.h:145
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:569
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:346
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:579
unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
unsigned getEntryAlignment(const DataLayout &TD) const
getEntryAlignment - Return the alignment of each entry in the jump table.
JTEntryKind getEntryKind() const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
T get() const
Returns the value of the specified pointer type.
Definition: PointerUnion.h:155
T dyn_cast() const
Returns the current pointer if it is of the specified pointer type, otherwise returns null.
Definition: PointerUnion.h:162
This class provides the information for the target register banks.
This class implements the register bank concept.
Definition: RegisterBank.h:28
unsigned getID() const
Get the identifier of this register bank.
Definition: RegisterBank.h:45
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
size_t size() const
Definition: SmallVector.h:91
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
unsigned getID() const
Return the register class ID number.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:246
operand_type_match m_Reg()
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
unsigned getBrCond(CondCode CC, bool Imm=false)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ System
Synchronized with respect to all concurrently executing threads.
Definition: LLVMContext.h:57
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:155
InstructionSelector * createRISCVInstructionSelector(const RISCVTargetMachine &TM, const RISCVSubtarget &Subtarget, const RISCVRegisterBankInfo &RBI)
std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition: Utils.cpp:314
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:340
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:291
void reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, MachineOptimizationRemarkEmitter &MORE, MachineOptimizationRemarkMissed &R)
Report an ISel error as a missed optimization remark to the LLVMContext's diagnostic stream.
Definition: Utils.cpp:275
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Xor
Bitwise or logical XOR of integers.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define MORE()
Definition: regcomp.c:252
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Matching combinators.
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.