LLVM 19.0.0git
RISCVInstructionSelector.cpp
Go to the documentation of this file.
1//===-- RISCVInstructionSelector.cpp -----------------------------*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// RISC-V.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
16#include "RISCVSubtarget.h"
17#include "RISCVTargetMachine.h"
25#include "llvm/IR/IntrinsicsRISCV.h"
26#include "llvm/Support/Debug.h"
27
28#define DEBUG_TYPE "riscv-isel"
29
30using namespace llvm;
31using namespace MIPatternMatch;
32
33#define GET_GLOBALISEL_PREDICATE_BITSET
34#include "RISCVGenGlobalISel.inc"
35#undef GET_GLOBALISEL_PREDICATE_BITSET
36
37namespace {
38
39class RISCVInstructionSelector : public InstructionSelector {
40public:
41 RISCVInstructionSelector(const RISCVTargetMachine &TM,
42 const RISCVSubtarget &STI,
43 const RISCVRegisterBankInfo &RBI);
44
45 bool select(MachineInstr &MI) override;
46 static const char *getName() { return DEBUG_TYPE; }
47
48private:
50 getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB) const;
51
52 bool isRegInGprb(Register Reg, MachineRegisterInfo &MRI) const;
53 bool isRegInFprb(Register Reg, MachineRegisterInfo &MRI) const;
54
55 // tblgen-erated 'select' implementation, used as the initial selector for
56 // the patterns that don't require complex C++.
57 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
58
59 // A lowering phase that runs before any selection attempts.
60 // Returns true if the instruction was modified.
61 void preISelLower(MachineInstr &MI, MachineIRBuilder &MIB,
63
64 bool replacePtrWithInt(MachineOperand &Op, MachineIRBuilder &MIB,
66
67 // Custom selection methods
69 bool selectImplicitDef(MachineInstr &MI, MachineIRBuilder &MIB,
71 bool materializeImm(Register Reg, int64_t Imm, MachineIRBuilder &MIB) const;
72 bool selectAddr(MachineInstr &MI, MachineIRBuilder &MIB,
73 MachineRegisterInfo &MRI, bool IsLocal = true,
74 bool IsExternWeak = false) const;
75 bool selectSExtInreg(MachineInstr &MI, MachineIRBuilder &MIB) const;
76 bool selectSelect(MachineInstr &MI, MachineIRBuilder &MIB,
78 bool selectFPCompare(MachineInstr &MI, MachineIRBuilder &MIB,
80 bool selectIntrinsicWithSideEffects(MachineInstr &MI, MachineIRBuilder &MIB,
82 void emitFence(AtomicOrdering FenceOrdering, SyncScope::ID FenceSSID,
83 MachineIRBuilder &MIB) const;
88
89 ComplexRendererFns selectShiftMask(MachineOperand &Root) const;
90 ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
91
92 ComplexRendererFns selectSHXADDOp(MachineOperand &Root, unsigned ShAmt) const;
93 template <unsigned ShAmt>
94 ComplexRendererFns selectSHXADDOp(MachineOperand &Root) const {
95 return selectSHXADDOp(Root, ShAmt);
96 }
97
98 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root,
99 unsigned ShAmt) const;
100 template <unsigned ShAmt>
101 ComplexRendererFns selectSHXADD_UWOp(MachineOperand &Root) const {
102 return selectSHXADD_UWOp(Root, ShAmt);
103 }
104
105 // Custom renderers for tablegen
106 void renderNegImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
107 int OpIdx) const;
108 void renderImmSubFromXLen(MachineInstrBuilder &MIB, const MachineInstr &MI,
109 int OpIdx) const;
110 void renderImmSubFrom32(MachineInstrBuilder &MIB, const MachineInstr &MI,
111 int OpIdx) const;
112 void renderImmPlus1(MachineInstrBuilder &MIB, const MachineInstr &MI,
113 int OpIdx) const;
114 void renderImm(MachineInstrBuilder &MIB, const MachineInstr &MI,
115 int OpIdx) const;
116
117 void renderTrailingZeros(MachineInstrBuilder &MIB, const MachineInstr &MI,
118 int OpIdx) const;
119
120 const RISCVSubtarget &STI;
121 const RISCVInstrInfo &TII;
122 const RISCVRegisterInfo &TRI;
123 const RISCVRegisterBankInfo &RBI;
124 const RISCVTargetMachine &TM;
125
126 // FIXME: This is necessary because DAGISel uses "Subtarget->" and GlobalISel
127 // uses "STI." in the code generated by TableGen. We need to unify the name of
128 // Subtarget variable.
129 const RISCVSubtarget *Subtarget = &STI;
130
131#define GET_GLOBALISEL_PREDICATES_DECL
132#include "RISCVGenGlobalISel.inc"
133#undef GET_GLOBALISEL_PREDICATES_DECL
134
135#define GET_GLOBALISEL_TEMPORARIES_DECL
136#include "RISCVGenGlobalISel.inc"
137#undef GET_GLOBALISEL_TEMPORARIES_DECL
138};
139
140} // end anonymous namespace
141
142#define GET_GLOBALISEL_IMPL
143#include "RISCVGenGlobalISel.inc"
144#undef GET_GLOBALISEL_IMPL
145
146RISCVInstructionSelector::RISCVInstructionSelector(
147 const RISCVTargetMachine &TM, const RISCVSubtarget &STI,
148 const RISCVRegisterBankInfo &RBI)
149 : STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI),
150 TM(TM),
151
153#include "RISCVGenGlobalISel.inc"
156#include "RISCVGenGlobalISel.inc"
158{
159}
160
162RISCVInstructionSelector::selectShiftMask(MachineOperand &Root) const {
163 if (!Root.isReg())
164 return std::nullopt;
165
166 using namespace llvm::MIPatternMatch;
167 MachineRegisterInfo &MRI = MF->getRegInfo();
168
169 Register RootReg = Root.getReg();
170 Register ShAmtReg = RootReg;
171 const LLT ShiftLLT = MRI.getType(RootReg);
172 unsigned ShiftWidth = ShiftLLT.getSizeInBits();
173 assert(isPowerOf2_32(ShiftWidth) && "Unexpected max shift amount!");
174 // Peek through zext.
175 Register ZExtSrcReg;
176 if (mi_match(ShAmtReg, MRI, m_GZExt(m_Reg(ZExtSrcReg)))) {
177 ShAmtReg = ZExtSrcReg;
178 }
179
180 APInt AndMask;
181 Register AndSrcReg;
182 if (mi_match(ShAmtReg, MRI, m_GAnd(m_Reg(AndSrcReg), m_ICst(AndMask)))) {
183 APInt ShMask(AndMask.getBitWidth(), ShiftWidth - 1);
184 if (ShMask.isSubsetOf(AndMask)) {
185 ShAmtReg = AndSrcReg;
186 } else {
187 // SimplifyDemandedBits may have optimized the mask so try restoring any
188 // bits that are known zero.
189 KnownBits Known = KB->getKnownBits(ShAmtReg);
190 if (ShMask.isSubsetOf(AndMask | Known.Zero))
191 ShAmtReg = AndSrcReg;
192 }
193 }
194
195 APInt Imm;
197 if (mi_match(ShAmtReg, MRI, m_GAdd(m_Reg(Reg), m_ICst(Imm)))) {
198 if (Imm != 0 && Imm.urem(ShiftWidth) == 0)
199 // If we are shifting by X+N where N == 0 mod Size, then just shift by X
200 // to avoid the ADD.
201 ShAmtReg = Reg;
202 } else if (mi_match(ShAmtReg, MRI, m_GSub(m_ICst(Imm), m_Reg(Reg)))) {
203 if (Imm != 0 && Imm.urem(ShiftWidth) == 0) {
204 // If we are shifting by N-X where N == 0 mod Size, then just shift by -X
205 // to generate a NEG instead of a SUB of a constant.
206 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
207 unsigned NegOpc = Subtarget->is64Bit() ? RISCV::SUBW : RISCV::SUB;
208 return {{[=](MachineInstrBuilder &MIB) {
209 MachineIRBuilder(*MIB.getInstr())
210 .buildInstr(NegOpc, {ShAmtReg}, {Register(RISCV::X0), Reg});
211 MIB.addReg(ShAmtReg);
212 }}};
213 }
214 if (Imm.urem(ShiftWidth) == ShiftWidth - 1) {
215 // If we are shifting by N-X where N == -1 mod Size, then just shift by ~X
216 // to generate a NOT instead of a SUB of a constant.
217 ShAmtReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
218 return {{[=](MachineInstrBuilder &MIB) {
219 MachineIRBuilder(*MIB.getInstr())
220 .buildInstr(RISCV::XORI, {ShAmtReg}, {Reg})
221 .addImm(-1);
222 MIB.addReg(ShAmtReg);
223 }}};
224 }
225 }
226
227 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(ShAmtReg); }}};
228}
229
231RISCVInstructionSelector::selectSHXADDOp(MachineOperand &Root,
232 unsigned ShAmt) const {
233 using namespace llvm::MIPatternMatch;
234 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
236
237 if (!Root.isReg())
238 return std::nullopt;
239 Register RootReg = Root.getReg();
240
241 const unsigned XLen = STI.getXLen();
242 APInt Mask, C2;
243 Register RegY;
244 std::optional<bool> LeftShift;
245 // (and (shl y, c2), mask)
246 if (mi_match(RootReg, MRI,
247 m_GAnd(m_GShl(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
248 LeftShift = true;
249 // (and (lshr y, c2), mask)
250 else if (mi_match(RootReg, MRI,
251 m_GAnd(m_GLShr(m_Reg(RegY), m_ICst(C2)), m_ICst(Mask))))
252 LeftShift = false;
253
254 if (LeftShift.has_value()) {
255 if (*LeftShift)
256 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
257 else
258 Mask &= maskTrailingOnes<uint64_t>(XLen - C2.getLimitedValue());
259
260 if (Mask.isShiftedMask()) {
261 unsigned Leading = XLen - Mask.getActiveBits();
262 unsigned Trailing = Mask.countr_zero();
263 // Given (and (shl y, c2), mask) in which mask has no leading zeros and
264 // c3 trailing zeros. We can use an SRLI by c3 - c2 followed by a SHXADD.
265 if (*LeftShift && Leading == 0 && C2.ult(Trailing) && Trailing == ShAmt) {
266 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
267 return {{[=](MachineInstrBuilder &MIB) {
268 MachineIRBuilder(*MIB.getInstr())
269 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
270 .addImm(Trailing - C2.getLimitedValue());
271 MIB.addReg(DstReg);
272 }}};
273 }
274
275 // Given (and (lshr y, c2), mask) in which mask has c2 leading zeros and
276 // c3 trailing zeros. We can use an SRLI by c2 + c3 followed by a SHXADD.
277 if (!*LeftShift && Leading == C2 && Trailing == ShAmt) {
278 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
279 return {{[=](MachineInstrBuilder &MIB) {
280 MachineIRBuilder(*MIB.getInstr())
281 .buildInstr(RISCV::SRLI, {DstReg}, {RegY})
282 .addImm(Leading + Trailing);
283 MIB.addReg(DstReg);
284 }}};
285 }
286 }
287 }
288
289 LeftShift.reset();
290
291 // (shl (and y, mask), c2)
292 if (mi_match(RootReg, MRI,
293 m_GShl(m_OneNonDBGUse(m_GAnd(m_Reg(RegY), m_ICst(Mask))),
294 m_ICst(C2))))
295 LeftShift = true;
296 // (lshr (and y, mask), c2)
297 else if (mi_match(RootReg, MRI,
299 m_ICst(C2))))
300 LeftShift = false;
301
302 if (LeftShift.has_value() && Mask.isShiftedMask()) {
303 unsigned Leading = XLen - Mask.getActiveBits();
304 unsigned Trailing = Mask.countr_zero();
305
306 // Given (shl (and y, mask), c2) in which mask has 32 leading zeros and
307 // c3 trailing zeros. If c1 + c3 == ShAmt, we can emit SRLIW + SHXADD.
308 bool Cond = *LeftShift && Leading == 32 && Trailing > 0 &&
309 (Trailing + C2.getLimitedValue()) == ShAmt;
310 if (!Cond)
311 // Given (lshr (and y, mask), c2) in which mask has 32 leading zeros and
312 // c3 trailing zeros. If c3 - c1 == ShAmt, we can emit SRLIW + SHXADD.
313 Cond = !*LeftShift && Leading == 32 && C2.ult(Trailing) &&
314 (Trailing - C2.getLimitedValue()) == ShAmt;
315
316 if (Cond) {
317 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
318 return {{[=](MachineInstrBuilder &MIB) {
319 MachineIRBuilder(*MIB.getInstr())
320 .buildInstr(RISCV::SRLIW, {DstReg}, {RegY})
321 .addImm(Trailing);
322 MIB.addReg(DstReg);
323 }}};
324 }
325 }
326
327 return std::nullopt;
328}
329
331RISCVInstructionSelector::selectSHXADD_UWOp(MachineOperand &Root,
332 unsigned ShAmt) const {
333 using namespace llvm::MIPatternMatch;
334 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
336
337 if (!Root.isReg())
338 return std::nullopt;
339 Register RootReg = Root.getReg();
340
341 // Given (and (shl x, c2), mask) in which mask is a shifted mask with
342 // 32 - ShAmt leading zeros and c2 trailing zeros. We can use SLLI by
343 // c2 - ShAmt followed by SHXADD_UW with ShAmt for x amount.
344 APInt Mask, C2;
345 Register RegX;
346 if (mi_match(
347 RootReg, MRI,
349 m_ICst(Mask))))) {
350 Mask &= maskTrailingZeros<uint64_t>(C2.getLimitedValue());
351
352 if (Mask.isShiftedMask()) {
353 unsigned Leading = Mask.countl_zero();
354 unsigned Trailing = Mask.countr_zero();
355 if (Leading == 32 - ShAmt && C2 == Trailing && Trailing > ShAmt) {
356 Register DstReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
357 return {{[=](MachineInstrBuilder &MIB) {
358 MachineIRBuilder(*MIB.getInstr())
359 .buildInstr(RISCV::SLLI, {DstReg}, {RegX})
360 .addImm(C2.getLimitedValue() - ShAmt);
361 MIB.addReg(DstReg);
362 }}};
363 }
364 }
365 }
366
367 return std::nullopt;
368}
369
371RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
372 MachineFunction &MF = *Root.getParent()->getParent()->getParent();
374
375 if (!Root.isReg())
376 return std::nullopt;
377
378 MachineInstr *RootDef = MRI.getVRegDef(Root.getReg());
379 if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
380 return {{
381 [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
382 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
383 }};
384 }
385
386 if (isBaseWithConstantOffset(Root, MRI)) {
387 MachineOperand &LHS = RootDef->getOperand(1);
388 MachineOperand &RHS = RootDef->getOperand(2);
389 MachineInstr *LHSDef = MRI.getVRegDef(LHS.getReg());
390 MachineInstr *RHSDef = MRI.getVRegDef(RHS.getReg());
391
392 int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
393 if (isInt<12>(RHSC)) {
394 if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
395 return {{
396 [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
397 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
398 }};
399
400 return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
401 [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
402 }
403 }
404
405 // TODO: Need to get the immediate from a G_PTR_ADD. Should this be done in
406 // the combiner?
407 return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
408 [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
409}
410
411/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
412/// CC Must be an ICMP Predicate.
413static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
414 switch (CC) {
415 default:
416 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
417 case CmpInst::Predicate::ICMP_EQ:
418 return RISCVCC::COND_EQ;
419 case CmpInst::Predicate::ICMP_NE:
420 return RISCVCC::COND_NE;
421 case CmpInst::Predicate::ICMP_ULT:
422 return RISCVCC::COND_LTU;
423 case CmpInst::Predicate::ICMP_SLT:
424 return RISCVCC::COND_LT;
425 case CmpInst::Predicate::ICMP_UGE:
426 return RISCVCC::COND_GEU;
427 case CmpInst::Predicate::ICMP_SGE:
428 return RISCVCC::COND_GE;
429 }
430}
431
434 Register &RHS) {
435 // Try to fold an ICmp. If that fails, use a NE compare with X0.
437 if (!mi_match(CondReg, MRI, m_GICmp(m_Pred(Pred), m_Reg(LHS), m_Reg(RHS)))) {
438 LHS = CondReg;
439 RHS = RISCV::X0;
441 return;
442 }
443
444 // We found an ICmp, do some canonicalizations.
445
446 // Adjust comparisons to use comparison with 0 if possible.
448 switch (Pred) {
449 case CmpInst::Predicate::ICMP_SGT:
450 // Convert X > -1 to X >= 0
451 if (*Constant == -1) {
453 RHS = RISCV::X0;
454 return;
455 }
456 break;
457 case CmpInst::Predicate::ICMP_SLT:
458 // Convert X < 1 to 0 >= X
459 if (*Constant == 1) {
461 RHS = LHS;
462 LHS = RISCV::X0;
463 return;
464 }
465 break;
466 default:
467 break;
468 }
469 }
470
471 switch (Pred) {
472 default:
473 llvm_unreachable("Expected ICMP CmpInst::Predicate.");
474 case CmpInst::Predicate::ICMP_EQ:
475 case CmpInst::Predicate::ICMP_NE:
476 case CmpInst::Predicate::ICMP_ULT:
477 case CmpInst::Predicate::ICMP_SLT:
478 case CmpInst::Predicate::ICMP_UGE:
479 case CmpInst::Predicate::ICMP_SGE:
480 // These CCs are supported directly by RISC-V branches.
481 break;
482 case CmpInst::Predicate::ICMP_SGT:
483 case CmpInst::Predicate::ICMP_SLE:
484 case CmpInst::Predicate::ICMP_UGT:
485 case CmpInst::Predicate::ICMP_ULE:
486 // These CCs are not supported directly by RISC-V branches, but changing the
487 // direction of the CC and swapping LHS and RHS are.
488 Pred = CmpInst::getSwappedPredicate(Pred);
489 std::swap(LHS, RHS);
490 break;
491 }
492
493 CC = getRISCVCCFromICmp(Pred);
494 return;
495}
496
497bool RISCVInstructionSelector::select(MachineInstr &MI) {
498 MachineBasicBlock &MBB = *MI.getParent();
501 MachineIRBuilder MIB(MI);
502
503 preISelLower(MI, MIB, MRI);
504 const unsigned Opc = MI.getOpcode();
505
506 if (!MI.isPreISelOpcode() || Opc == TargetOpcode::G_PHI) {
507 if (Opc == TargetOpcode::PHI || Opc == TargetOpcode::G_PHI) {
508 const Register DefReg = MI.getOperand(0).getReg();
509 const LLT DefTy = MRI.getType(DefReg);
510
511 const RegClassOrRegBank &RegClassOrBank =
512 MRI.getRegClassOrRegBank(DefReg);
513
514 const TargetRegisterClass *DefRC =
515 RegClassOrBank.dyn_cast<const TargetRegisterClass *>();
516 if (!DefRC) {
517 if (!DefTy.isValid()) {
518 LLVM_DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n");
519 return false;
520 }
521
522 const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>();
523 DefRC = getRegClassForTypeOnBank(DefTy, RB);
524 if (!DefRC) {
525 LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n");
526 return false;
527 }
528 }
529
530 MI.setDesc(TII.get(TargetOpcode::PHI));
531 return RBI.constrainGenericRegister(DefReg, *DefRC, MRI);
532 }
533
534 // Certain non-generic instructions also need some special handling.
535 if (MI.isCopy())
536 return selectCopy(MI, MRI);
537
538 return true;
539 }
540
541 if (selectImpl(MI, *CoverageInfo))
542 return true;
543
544 switch (Opc) {
545 case TargetOpcode::G_ANYEXT:
546 case TargetOpcode::G_PTRTOINT:
547 case TargetOpcode::G_INTTOPTR:
548 case TargetOpcode::G_TRUNC:
549 return selectCopy(MI, MRI);
550 case TargetOpcode::G_CONSTANT: {
551 Register DstReg = MI.getOperand(0).getReg();
552 int64_t Imm = MI.getOperand(1).getCImm()->getSExtValue();
553
554 if (!materializeImm(DstReg, Imm, MIB))
555 return false;
556
557 MI.eraseFromParent();
558 return true;
559 }
560 case TargetOpcode::G_FCONSTANT: {
561 // TODO: Use constant pool for complext constants.
562 // TODO: Optimize +0.0 to use fcvt.d.w for s64 on rv32.
563 Register DstReg = MI.getOperand(0).getReg();
564 const APFloat &FPimm = MI.getOperand(1).getFPImm()->getValueAPF();
565 APInt Imm = FPimm.bitcastToAPInt();
566 unsigned Size = MRI.getType(DstReg).getSizeInBits();
567 if (Size == 32 || (Size == 64 && Subtarget->is64Bit())) {
568 Register GPRReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
569 if (!materializeImm(GPRReg, Imm.getSExtValue(), MIB))
570 return false;
571
572 unsigned Opcode = Size == 64 ? RISCV::FMV_D_X : RISCV::FMV_W_X;
573 auto FMV = MIB.buildInstr(Opcode, {DstReg}, {GPRReg});
574 if (!FMV.constrainAllUses(TII, TRI, RBI))
575 return false;
576 } else {
577 assert(Size == 64 && !Subtarget->is64Bit() &&
578 "Unexpected size or subtarget");
579 // Split into two pieces and build through the stack.
580 Register GPRRegHigh = MRI.createVirtualRegister(&RISCV::GPRRegClass);
581 Register GPRRegLow = MRI.createVirtualRegister(&RISCV::GPRRegClass);
582 if (!materializeImm(GPRRegHigh, Imm.extractBits(32, 32).getSExtValue(),
583 MIB))
584 return false;
585 if (!materializeImm(GPRRegLow, Imm.trunc(32).getSExtValue(), MIB))
586 return false;
587 MachineInstrBuilder PairF64 = MIB.buildInstr(
588 RISCV::BuildPairF64Pseudo, {DstReg}, {GPRRegLow, GPRRegHigh});
589 if (!PairF64.constrainAllUses(TII, TRI, RBI))
590 return false;
591 }
592
593 MI.eraseFromParent();
594 return true;
595 }
596 case TargetOpcode::G_GLOBAL_VALUE: {
597 auto *GV = MI.getOperand(1).getGlobal();
598 if (GV->isThreadLocal()) {
599 // TODO: implement this case.
600 return false;
601 }
602
603 return selectAddr(MI, MIB, MRI, GV->isDSOLocal(),
604 GV->hasExternalWeakLinkage());
605 }
606 case TargetOpcode::G_JUMP_TABLE:
607 case TargetOpcode::G_CONSTANT_POOL:
608 return selectAddr(MI, MIB, MRI);
609 case TargetOpcode::G_BRCOND: {
612 getOperandsForBranch(MI.getOperand(0).getReg(), MRI, CC, LHS, RHS);
613
614 auto Bcc = MIB.buildInstr(RISCVCC::getBrCond(CC), {}, {LHS, RHS})
615 .addMBB(MI.getOperand(1).getMBB());
616 MI.eraseFromParent();
617 return constrainSelectedInstRegOperands(*Bcc, TII, TRI, RBI);
618 }
619 case TargetOpcode::G_BRJT: {
620 // FIXME: Move to legalization?
621 const MachineJumpTableInfo *MJTI = MF.getJumpTableInfo();
622 unsigned EntrySize = MJTI->getEntrySize(MF.getDataLayout());
623 assert((EntrySize == 4 || (Subtarget->is64Bit() && EntrySize == 8)) &&
624 "Unsupported jump-table entry size");
625 assert(
629 "Unexpected jump-table entry kind");
630
631 auto SLL =
632 MIB.buildInstr(RISCV::SLLI, {&RISCV::GPRRegClass}, {MI.getOperand(2)})
633 .addImm(Log2_32(EntrySize));
634 if (!SLL.constrainAllUses(TII, TRI, RBI))
635 return false;
636
637 // TODO: Use SHXADD. Moving to legalization would fix this automatically.
638 auto ADD = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
639 {MI.getOperand(0), SLL.getReg(0)});
640 if (!ADD.constrainAllUses(TII, TRI, RBI))
641 return false;
642
643 unsigned LdOpc = EntrySize == 8 ? RISCV::LD : RISCV::LW;
644 auto Dest =
645 MIB.buildInstr(LdOpc, {&RISCV::GPRRegClass}, {ADD.getReg(0)})
646 .addImm(0)
647 .addMemOperand(MF.getMachineMemOperand(
649 EntrySize, Align(MJTI->getEntryAlignment(MF.getDataLayout()))));
650 if (!Dest.constrainAllUses(TII, TRI, RBI))
651 return false;
652
653 // If the Kind is EK_LabelDifference32, the table stores an offset from
654 // the location of the table. Add the table address to get an absolute
655 // address.
657 Dest = MIB.buildInstr(RISCV::ADD, {&RISCV::GPRRegClass},
658 {Dest.getReg(0), MI.getOperand(0)});
659 if (!Dest.constrainAllUses(TII, TRI, RBI))
660 return false;
661 }
662
663 auto Branch =
664 MIB.buildInstr(RISCV::PseudoBRIND, {}, {Dest.getReg(0)}).addImm(0);
665 if (!Branch.constrainAllUses(TII, TRI, RBI))
666 return false;
667
668 MI.eraseFromParent();
669 return true;
670 }
671 case TargetOpcode::G_BRINDIRECT:
672 MI.setDesc(TII.get(RISCV::PseudoBRIND));
673 MI.addOperand(MachineOperand::CreateImm(0));
675 case TargetOpcode::G_SEXT_INREG:
676 return selectSExtInreg(MI, MIB);
677 case TargetOpcode::G_FRAME_INDEX: {
678 // TODO: We may want to replace this code with the SelectionDAG patterns,
679 // which fail to get imported because it uses FrameAddrRegImm, which is a
680 // ComplexPattern
681 MI.setDesc(TII.get(RISCV::ADDI));
682 MI.addOperand(MachineOperand::CreateImm(0));
684 }
685 case TargetOpcode::G_SELECT:
686 return selectSelect(MI, MIB, MRI);
687 case TargetOpcode::G_FCMP:
688 return selectFPCompare(MI, MIB, MRI);
689 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
690 return selectIntrinsicWithSideEffects(MI, MIB, MRI);
691 case TargetOpcode::G_FENCE: {
692 AtomicOrdering FenceOrdering =
693 static_cast<AtomicOrdering>(MI.getOperand(0).getImm());
694 SyncScope::ID FenceSSID =
695 static_cast<SyncScope::ID>(MI.getOperand(1).getImm());
696 emitFence(FenceOrdering, FenceSSID, MIB);
697 MI.eraseFromParent();
698 return true;
699 }
700 case TargetOpcode::G_IMPLICIT_DEF:
701 return selectImplicitDef(MI, MIB, MRI);
702 case TargetOpcode::G_MERGE_VALUES:
703 return selectMergeValues(MI, MIB, MRI);
704 case TargetOpcode::G_UNMERGE_VALUES:
705 return selectUnmergeValues(MI, MIB, MRI);
706 default:
707 return false;
708 }
709}
710
711bool RISCVInstructionSelector::selectMergeValues(
713 assert(MI.getOpcode() == TargetOpcode::G_MERGE_VALUES);
714
715 // Build a F64 Pair from operands
716 if (MI.getNumOperands() != 3)
717 return false;
718 Register Dst = MI.getOperand(0).getReg();
719 Register Lo = MI.getOperand(1).getReg();
720 Register Hi = MI.getOperand(2).getReg();
721 if (!isRegInFprb(Dst, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
722 return false;
723 MI.setDesc(TII.get(RISCV::BuildPairF64Pseudo));
725}
726
727bool RISCVInstructionSelector::selectUnmergeValues(
729 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
730
731 // Split F64 Src into two s32 parts
732 if (MI.getNumOperands() != 3)
733 return false;
734 Register Src = MI.getOperand(2).getReg();
735 Register Lo = MI.getOperand(0).getReg();
736 Register Hi = MI.getOperand(1).getReg();
737 if (!isRegInFprb(Src, MRI) || !isRegInGprb(Lo, MRI) || !isRegInGprb(Hi, MRI))
738 return false;
739 MI.setDesc(TII.get(RISCV::SplitF64Pseudo));
741}
742
743bool RISCVInstructionSelector::replacePtrWithInt(MachineOperand &Op,
744 MachineIRBuilder &MIB,
746 Register PtrReg = Op.getReg();
747 assert(MRI.getType(PtrReg).isPointer() && "Operand is not a pointer!");
748
749 const LLT sXLen = LLT::scalar(STI.getXLen());
750 auto PtrToInt = MIB.buildPtrToInt(sXLen, PtrReg);
751 MRI.setRegBank(PtrToInt.getReg(0), RBI.getRegBank(RISCV::GPRBRegBankID));
752 Op.setReg(PtrToInt.getReg(0));
753 return select(*PtrToInt);
754}
755
756void RISCVInstructionSelector::preISelLower(MachineInstr &MI,
757 MachineIRBuilder &MIB,
759 switch (MI.getOpcode()) {
760 case TargetOpcode::G_PTR_ADD: {
761 Register DstReg = MI.getOperand(0).getReg();
762 const LLT sXLen = LLT::scalar(STI.getXLen());
763
764 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
765 MI.setDesc(TII.get(TargetOpcode::G_ADD));
766 MRI.setType(DstReg, sXLen);
767 break;
768 }
769 case TargetOpcode::G_PTRMASK: {
770 Register DstReg = MI.getOperand(0).getReg();
771 const LLT sXLen = LLT::scalar(STI.getXLen());
772 replacePtrWithInt(MI.getOperand(1), MIB, MRI);
773 MI.setDesc(TII.get(TargetOpcode::G_AND));
774 MRI.setType(DstReg, sXLen);
775 }
776 }
777}
778
779void RISCVInstructionSelector::renderNegImm(MachineInstrBuilder &MIB,
780 const MachineInstr &MI,
781 int OpIdx) const {
782 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
783 "Expected G_CONSTANT");
784 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
785 MIB.addImm(-CstVal);
786}
787
788void RISCVInstructionSelector::renderImmSubFromXLen(MachineInstrBuilder &MIB,
789 const MachineInstr &MI,
790 int OpIdx) const {
791 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
792 "Expected G_CONSTANT");
793 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
794 MIB.addImm(STI.getXLen() - CstVal);
795}
796
797void RISCVInstructionSelector::renderImmSubFrom32(MachineInstrBuilder &MIB,
798 const MachineInstr &MI,
799 int OpIdx) const {
800 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
801 "Expected G_CONSTANT");
802 uint64_t CstVal = MI.getOperand(1).getCImm()->getZExtValue();
803 MIB.addImm(32 - CstVal);
804}
805
806void RISCVInstructionSelector::renderImmPlus1(MachineInstrBuilder &MIB,
807 const MachineInstr &MI,
808 int OpIdx) const {
809 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
810 "Expected G_CONSTANT");
811 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
812 MIB.addImm(CstVal + 1);
813}
814
815void RISCVInstructionSelector::renderImm(MachineInstrBuilder &MIB,
816 const MachineInstr &MI,
817 int OpIdx) const {
818 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
819 "Expected G_CONSTANT");
820 int64_t CstVal = MI.getOperand(1).getCImm()->getSExtValue();
821 MIB.addImm(CstVal);
822}
823
824void RISCVInstructionSelector::renderTrailingZeros(MachineInstrBuilder &MIB,
825 const MachineInstr &MI,
826 int OpIdx) const {
827 assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
828 "Expected G_CONSTANT");
829 uint64_t C = MI.getOperand(1).getCImm()->getZExtValue();
831}
832
833const TargetRegisterClass *RISCVInstructionSelector::getRegClassForTypeOnBank(
834 LLT Ty, const RegisterBank &RB) const {
835 if (RB.getID() == RISCV::GPRBRegBankID) {
836 if (Ty.getSizeInBits() <= 32 || (STI.is64Bit() && Ty.getSizeInBits() == 64))
837 return &RISCV::GPRRegClass;
838 }
839
840 if (RB.getID() == RISCV::FPRBRegBankID) {
841 if (Ty.getSizeInBits() == 32)
842 return &RISCV::FPR32RegClass;
843 if (Ty.getSizeInBits() == 64)
844 return &RISCV::FPR64RegClass;
845 }
846
847 if (RB.getID() == RISCV::VRBRegBankID) {
848 if (Ty.getSizeInBits().getKnownMinValue() <= 64)
849 return &RISCV::VRRegClass;
850
851 if (Ty.getSizeInBits().getKnownMinValue() == 128)
852 return &RISCV::VRM2RegClass;
853
854 if (Ty.getSizeInBits().getKnownMinValue() == 256)
855 return &RISCV::VRM4RegClass;
856
857 if (Ty.getSizeInBits().getKnownMinValue() == 512)
858 return &RISCV::VRM8RegClass;
859 }
860
861 return nullptr;
862}
863
864bool RISCVInstructionSelector::isRegInGprb(Register Reg,
865 MachineRegisterInfo &MRI) const {
866 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::GPRBRegBankID;
867}
868
869bool RISCVInstructionSelector::isRegInFprb(Register Reg,
870 MachineRegisterInfo &MRI) const {
871 return RBI.getRegBank(Reg, MRI, TRI)->getID() == RISCV::FPRBRegBankID;
872}
873
874bool RISCVInstructionSelector::selectCopy(MachineInstr &MI,
875 MachineRegisterInfo &MRI) const {
876 Register DstReg = MI.getOperand(0).getReg();
877
878 if (DstReg.isPhysical())
879 return true;
880
881 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
882 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
883 assert(DstRC &&
884 "Register class not available for LLT, register bank combination");
885
886 // No need to constrain SrcReg. It will get constrained when
887 // we hit another of its uses or its defs.
888 // Copies do not have constraints.
889 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
890 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
891 << " operand\n");
892 return false;
893 }
894
895 MI.setDesc(TII.get(RISCV::COPY));
896 return true;
897}
898
899bool RISCVInstructionSelector::selectImplicitDef(
901 assert(MI.getOpcode() == TargetOpcode::G_IMPLICIT_DEF);
902
903 const Register DstReg = MI.getOperand(0).getReg();
904 const TargetRegisterClass *DstRC = getRegClassForTypeOnBank(
905 MRI.getType(DstReg), *RBI.getRegBank(DstReg, MRI, TRI));
906
907 assert(DstRC &&
908 "Register class not available for LLT, register bank combination");
909
910 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
911 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(MI.getOpcode())
912 << " operand\n");
913 }
914 MI.setDesc(TII.get(TargetOpcode::IMPLICIT_DEF));
915 return true;
916}
917
918bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
919 MachineIRBuilder &MIB) const {
921
922 if (Imm == 0) {
923 MIB.buildCopy(DstReg, Register(RISCV::X0));
924 RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, MRI);
925 return true;
926 }
927
929 unsigned NumInsts = Seq.size();
930 Register SrcReg = RISCV::X0;
931
932 for (unsigned i = 0; i < NumInsts; i++) {
933 Register TmpReg = i < NumInsts - 1
934 ? MRI.createVirtualRegister(&RISCV::GPRRegClass)
935 : DstReg;
936 const RISCVMatInt::Inst &I = Seq[i];
938
939 switch (I.getOpndKind()) {
940 case RISCVMatInt::Imm:
941 // clang-format off
942 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {})
943 .addImm(I.getImm());
944 // clang-format on
945 break;
947 Result = MIB.buildInstr(I.getOpcode(), {TmpReg},
948 {SrcReg, Register(RISCV::X0)});
949 break;
951 Result = MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg, SrcReg});
952 break;
954 Result =
955 MIB.buildInstr(I.getOpcode(), {TmpReg}, {SrcReg}).addImm(I.getImm());
956 break;
957 }
958
959 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
960 return false;
961
962 SrcReg = TmpReg;
963 }
964
965 return true;
966}
967
968bool RISCVInstructionSelector::selectAddr(MachineInstr &MI,
969 MachineIRBuilder &MIB,
971 bool IsLocal,
972 bool IsExternWeak) const {
973 assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
974 MI.getOpcode() == TargetOpcode::G_JUMP_TABLE ||
975 MI.getOpcode() == TargetOpcode::G_CONSTANT_POOL) &&
976 "Unexpected opcode");
977
978 const MachineOperand &DispMO = MI.getOperand(1);
979
980 Register DefReg = MI.getOperand(0).getReg();
981 const LLT DefTy = MRI.getType(DefReg);
982
983 // When HWASAN is used and tagging of global variables is enabled
984 // they should be accessed via the GOT, since the tagged address of a global
985 // is incompatible with existing code models. This also applies to non-pic
986 // mode.
987 if (TM.isPositionIndependent() || Subtarget->allowTaggedGlobals()) {
988 if (IsLocal && !Subtarget->allowTaggedGlobals()) {
989 // Use PC-relative addressing to access the symbol. This generates the
990 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
991 // %pcrel_lo(auipc)).
992 MI.setDesc(TII.get(RISCV::PseudoLLA));
994 }
995
996 // Use PC-relative addressing to access the GOT for this symbol, then
997 // load the address from the GOT. This generates the pattern (PseudoLGA
998 // sym), which expands to (ld (addi (auipc %got_pcrel_hi(sym))
999 // %pcrel_lo(auipc))).
1000 MachineFunction &MF = *MI.getParent()->getParent();
1005 DefTy, Align(DefTy.getSizeInBits() / 8));
1006
1007 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1008 .addDisp(DispMO, 0)
1010
1011 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1012 return false;
1013
1014 MI.eraseFromParent();
1015 return true;
1016 }
1017
1018 switch (TM.getCodeModel()) {
1019 default: {
1020 reportGISelFailure(const_cast<MachineFunction &>(*MF), *TPC, *MORE,
1021 getName(), "Unsupported code model for lowering", MI);
1022 return false;
1023 }
1024 case CodeModel::Small: {
1025 // Must lie within a single 2 GiB address range and must lie between
1026 // absolute addresses -2 GiB and +2 GiB. This generates the pattern (addi
1027 // (lui %hi(sym)) %lo(sym)).
1028 Register AddrHiDest = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1029 MachineInstr *AddrHi = MIB.buildInstr(RISCV::LUI, {AddrHiDest}, {})
1030 .addDisp(DispMO, 0, RISCVII::MO_HI);
1031
1032 if (!constrainSelectedInstRegOperands(*AddrHi, TII, TRI, RBI))
1033 return false;
1034
1035 auto Result = MIB.buildInstr(RISCV::ADDI, {DefReg}, {AddrHiDest})
1036 .addDisp(DispMO, 0, RISCVII::MO_LO);
1037
1038 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1039 return false;
1040
1041 MI.eraseFromParent();
1042 return true;
1043 }
1044 case CodeModel::Medium:
1045 // Emit LGA/LLA instead of the sequence it expands to because the pcrel_lo
1046 // relocation needs to reference a label that points to the auipc
1047 // instruction itself, not the global. This cannot be done inside the
1048 // instruction selector.
1049 if (IsExternWeak) {
1050 // An extern weak symbol may be undefined, i.e. have value 0, which may
1051 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
1052 // symbol. This generates the pattern (PseudoLGA sym), which expands to
1053 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
1054 MachineFunction &MF = *MI.getParent()->getParent();
1059 DefTy, Align(DefTy.getSizeInBits() / 8));
1060
1061 auto Result = MIB.buildInstr(RISCV::PseudoLGA, {DefReg}, {})
1062 .addDisp(DispMO, 0)
1064
1065 if (!constrainSelectedInstRegOperands(*Result, TII, TRI, RBI))
1066 return false;
1067
1068 MI.eraseFromParent();
1069 return true;
1070 }
1071
1072 // Generate a sequence for accessing addresses within any 2GiB range
1073 // within the address space. This generates the pattern (PseudoLLA sym),
1074 // which expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
1075 MI.setDesc(TII.get(RISCV::PseudoLLA));
1077 }
1078
1079 return false;
1080}
1081
1082bool RISCVInstructionSelector::selectSExtInreg(MachineInstr &MI,
1083 MachineIRBuilder &MIB) const {
1084 if (!STI.isRV64())
1085 return false;
1086
1087 const MachineOperand &Size = MI.getOperand(2);
1088 // Only Size == 32 (i.e. shift by 32 bits) is acceptable at this point.
1089 if (!Size.isImm() || Size.getImm() != 32)
1090 return false;
1091
1092 const MachineOperand &Src = MI.getOperand(1);
1093 const MachineOperand &Dst = MI.getOperand(0);
1094 // addiw rd, rs, 0 (i.e. sext.w rd, rs)
1095 MachineInstr *NewMI =
1096 MIB.buildInstr(RISCV::ADDIW, {Dst.getReg()}, {Src.getReg()}).addImm(0U);
1097
1098 if (!constrainSelectedInstRegOperands(*NewMI, TII, TRI, RBI))
1099 return false;
1100
1101 MI.eraseFromParent();
1102 return true;
1103}
1104
1105bool RISCVInstructionSelector::selectSelect(MachineInstr &MI,
1106 MachineIRBuilder &MIB,
1107 MachineRegisterInfo &MRI) const {
1108 auto &SelectMI = cast<GSelect>(MI);
1109
1110 Register LHS, RHS;
1112 getOperandsForBranch(SelectMI.getCondReg(), MRI, CC, LHS, RHS);
1113
1114 Register DstReg = SelectMI.getReg(0);
1115
1116 unsigned Opc = RISCV::Select_GPR_Using_CC_GPR;
1117 if (RBI.getRegBank(DstReg, MRI, TRI)->getID() == RISCV::FPRBRegBankID) {
1118 unsigned Size = MRI.getType(DstReg).getSizeInBits();
1119 Opc = Size == 32 ? RISCV::Select_FPR32_Using_CC_GPR
1120 : RISCV::Select_FPR64_Using_CC_GPR;
1121 }
1122
1123 MachineInstr *Result = MIB.buildInstr(Opc)
1124 .addDef(DstReg)
1125 .addReg(LHS)
1126 .addReg(RHS)
1127 .addImm(CC)
1128 .addReg(SelectMI.getTrueReg())
1129 .addReg(SelectMI.getFalseReg());
1130 MI.eraseFromParent();
1131 return constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
1132}
1133
1134// Convert an FCMP predicate to one of the supported F or D instructions.
1135static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size) {
1136 assert((Size == 32 || Size == 64) && "Unsupported size");
1137 switch (Pred) {
1138 default:
1139 llvm_unreachable("Unsupported predicate");
1140 case CmpInst::FCMP_OLT:
1141 return Size == 32 ? RISCV::FLT_S : RISCV::FLT_D;
1142 case CmpInst::FCMP_OLE:
1143 return Size == 32 ? RISCV::FLE_S : RISCV::FLE_D;
1144 case CmpInst::FCMP_OEQ:
1145 return Size == 32 ? RISCV::FEQ_S : RISCV::FEQ_D;
1146 }
1147}
1148
1149// Try legalizing an FCMP by swapping or inverting the predicate to one that
1150// is supported.
1152 CmpInst::Predicate &Pred, bool &NeedInvert) {
1153 auto isLegalFCmpPredicate = [](CmpInst::Predicate Pred) {
1154 return Pred == CmpInst::FCMP_OLT || Pred == CmpInst::FCMP_OLE ||
1155 Pred == CmpInst::FCMP_OEQ;
1156 };
1157
1158 assert(!isLegalFCmpPredicate(Pred) && "Predicate already legal?");
1159
1161 if (isLegalFCmpPredicate(InvPred)) {
1162 Pred = InvPred;
1163 std::swap(LHS, RHS);
1164 return true;
1165 }
1166
1167 InvPred = CmpInst::getInversePredicate(Pred);
1168 NeedInvert = true;
1169 if (isLegalFCmpPredicate(InvPred)) {
1170 Pred = InvPred;
1171 return true;
1172 }
1173 InvPred = CmpInst::getSwappedPredicate(InvPred);
1174 if (isLegalFCmpPredicate(InvPred)) {
1175 Pred = InvPred;
1176 std::swap(LHS, RHS);
1177 return true;
1178 }
1179
1180 return false;
1181}
1182
1183// Emit a sequence of instructions to compare LHS and RHS using Pred. Return
1184// the result in DstReg.
1185// FIXME: Maybe we should expand this earlier.
1186bool RISCVInstructionSelector::selectFPCompare(MachineInstr &MI,
1187 MachineIRBuilder &MIB,
1188 MachineRegisterInfo &MRI) const {
1189 auto &CmpMI = cast<GFCmp>(MI);
1190 CmpInst::Predicate Pred = CmpMI.getCond();
1191
1192 Register DstReg = CmpMI.getReg(0);
1193 Register LHS = CmpMI.getLHSReg();
1194 Register RHS = CmpMI.getRHSReg();
1195
1196 unsigned Size = MRI.getType(LHS).getSizeInBits();
1197 assert((Size == 32 || Size == 64) && "Unexpected size");
1198
1199 Register TmpReg = DstReg;
1200
1201 bool NeedInvert = false;
1202 // First try swapping operands or inverting.
1203 if (legalizeFCmpPredicate(LHS, RHS, Pred, NeedInvert)) {
1204 if (NeedInvert)
1205 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1206 auto Cmp = MIB.buildInstr(getFCmpOpcode(Pred, Size), {TmpReg}, {LHS, RHS});
1207 if (!Cmp.constrainAllUses(TII, TRI, RBI))
1208 return false;
1209 } else if (Pred == CmpInst::FCMP_ONE || Pred == CmpInst::FCMP_UEQ) {
1210 // fcmp one LHS, RHS => (OR (FLT LHS, RHS), (FLT RHS, LHS))
1211 NeedInvert = Pred == CmpInst::FCMP_UEQ;
1213 {&RISCV::GPRRegClass}, {LHS, RHS});
1214 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1215 return false;
1217 {&RISCV::GPRRegClass}, {RHS, LHS});
1218 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1219 return false;
1220 if (NeedInvert)
1221 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1222 auto Or =
1223 MIB.buildInstr(RISCV::OR, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1224 if (!Or.constrainAllUses(TII, TRI, RBI))
1225 return false;
1226 } else if (Pred == CmpInst::FCMP_ORD || Pred == CmpInst::FCMP_UNO) {
1227 // fcmp ord LHS, RHS => (AND (FEQ LHS, LHS), (FEQ RHS, RHS))
1228 // FIXME: If LHS and RHS are the same we can use a single FEQ.
1229 NeedInvert = Pred == CmpInst::FCMP_UNO;
1231 {&RISCV::GPRRegClass}, {LHS, LHS});
1232 if (!Cmp1.constrainAllUses(TII, TRI, RBI))
1233 return false;
1235 {&RISCV::GPRRegClass}, {RHS, RHS});
1236 if (!Cmp2.constrainAllUses(TII, TRI, RBI))
1237 return false;
1238 if (NeedInvert)
1239 TmpReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
1240 auto And =
1241 MIB.buildInstr(RISCV::AND, {TmpReg}, {Cmp1.getReg(0), Cmp2.getReg(0)});
1242 if (!And.constrainAllUses(TII, TRI, RBI))
1243 return false;
1244 } else
1245 llvm_unreachable("Unhandled predicate");
1246
1247 // Emit an XORI to invert the result if needed.
1248 if (NeedInvert) {
1249 auto Xor = MIB.buildInstr(RISCV::XORI, {DstReg}, {TmpReg}).addImm(1);
1250 if (!Xor.constrainAllUses(TII, TRI, RBI))
1251 return false;
1252 }
1253
1254 MI.eraseFromParent();
1255 return true;
1256}
1257
1258bool RISCVInstructionSelector::selectIntrinsicWithSideEffects(
1260 assert(MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS &&
1261 "Unexpected opcode");
1262 // Find the intrinsic ID.
1263 unsigned IntrinID = cast<GIntrinsic>(MI).getIntrinsicID();
1264
1265 // Select the instruction.
1266 switch (IntrinID) {
1267 default:
1268 return false;
1269 case Intrinsic::trap:
1270 MIB.buildInstr(RISCV::UNIMP, {}, {});
1271 break;
1272 case Intrinsic::debugtrap:
1273 MIB.buildInstr(RISCV::EBREAK, {}, {});
1274 break;
1275 }
1276
1277 MI.eraseFromParent();
1278 return true;
1279}
1280
1281void RISCVInstructionSelector::emitFence(AtomicOrdering FenceOrdering,
1282 SyncScope::ID FenceSSID,
1283 MachineIRBuilder &MIB) const {
1284 if (STI.hasStdExtZtso()) {
1285 // The only fence that needs an instruction is a sequentially-consistent
1286 // cross-thread fence.
1287 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
1288 FenceSSID == SyncScope::System) {
1289 // fence rw, rw
1290 MIB.buildInstr(RISCV::FENCE, {}, {})
1293 return;
1294 }
1295
1296 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
1297 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1298 return;
1299 }
1300
1301 // singlethread fences only synchronize with signal handlers on the same
1302 // thread and thus only need to preserve instruction order, not actually
1303 // enforce memory ordering.
1304 if (FenceSSID == SyncScope::SingleThread) {
1305 MIB.buildInstr(TargetOpcode::MEMBARRIER, {}, {});
1306 return;
1307 }
1308
1309 // Refer to Table A.6 in the version 2.3 draft of the RISC-V Instruction Set
1310 // Manual: Volume I.
1311 unsigned Pred, Succ;
1312 switch (FenceOrdering) {
1313 default:
1314 llvm_unreachable("Unexpected ordering");
1315 case AtomicOrdering::AcquireRelease:
1316 // fence acq_rel -> fence.tso
1317 MIB.buildInstr(RISCV::FENCE_TSO, {}, {});
1318 return;
1319 case AtomicOrdering::Acquire:
1320 // fence acquire -> fence r, rw
1321 Pred = RISCVFenceField::R;
1323 break;
1324 case AtomicOrdering::Release:
1325 // fence release -> fence rw, w
1327 Succ = RISCVFenceField::W;
1328 break;
1329 case AtomicOrdering::SequentiallyConsistent:
1330 // fence seq_cst -> fence rw, rw
1333 break;
1334 }
1335 MIB.buildInstr(RISCV::FENCE, {}, {}).addImm(Pred).addImm(Succ);
1336}
1337
1338namespace llvm {
1341 RISCVSubtarget &Subtarget,
1342 RISCVRegisterBankInfo &RBI) {
1343 return new RISCVInstructionSelector(TM, Subtarget, RBI);
1344}
1345} // end namespace llvm
unsigned const MachineRegisterInfo * MRI
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
Provides analysis for querying information about KnownBits during GISel passes.
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
static StringRef getName(Value *V)
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static void getOperandsForBranch(Register CondReg, MachineRegisterInfo &MRI, RISCVCC::CondCode &CC, Register &LHS, Register &RHS)
static unsigned getFCmpOpcode(CmpInst::Predicate Pred, unsigned Size)
static bool legalizeFCmpPredicate(Register &LHS, Register &RHS, CmpInst::Predicate &Pred, bool &NeedInvert)
const SmallVectorImpl< MachineOperand > & Cond
This file declares the targeting of the RegisterBankInfo class for RISC-V.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
Class for arbitrary precision integers.
Definition: APInt.h:76
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1433
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1083
uint64_t getLimitedValue(uint64_t Limit=UINT64_MAX) const
If this value is smaller than the specified limit, return it, otherwise return the limit value.
Definition: APInt.h:453
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:780
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:783
@ FCMP_OLT
0 1 0 0 True if ordered and less than
Definition: InstrTypes.h:786
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
Definition: InstrTypes.h:788
@ FCMP_UEQ
1 0 0 1 True if unordered or equal
Definition: InstrTypes.h:791
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
Definition: InstrTypes.h:787
@ FCMP_ORD
0 1 1 1 True if ordered (no nans)
Definition: InstrTypes.h:789
@ FCMP_UNO
1 0 0 0 True if unordered: isnan(X) | isnan(Y)
Definition: InstrTypes.h:790
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:932
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:894
int64_t getSExtValue() const
Return the constant as a 64-bit integer value after it has been sign extended as appropriate for the ...
Definition: Constants.h:152
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
std::optional< SmallVector< std::function< void(MachineInstrBuilder &)>, 4 > > ComplexRendererFns
virtual bool select(MachineInstr &I)=0
Select the (possibly generic) instruction I to only use target-specific opcodes.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isValid() const
Definition: LowLevelType.h:137
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:185
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
bool constrainAllUses(const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI) const
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:543
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:326
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:553
unsigned getEntrySize(const DataLayout &TD) const
getEntrySize - Return the size of each entry in the jump table.
@ EK_LabelDifference32
EK_LabelDifference32 - Each entry is the address of the block minus the address of the jump table.
@ EK_Custom32
EK_Custom32 - Each entry is a 32-bit value that is custom lowered by the TargetLowering::LowerCustomJ...
@ EK_BlockAddress
EK_BlockAddress - Each entry is a plain address of block, e.g.: .word LBB123.
unsigned getEntryAlignment(const DataLayout &TD) const
getEntryAlignment - Return the alignment of each entry in the jump table.
JTEntryKind getEntryKind() const
A description of a memory reference used in the backend.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
T get() const
Returns the value of the specified pointer type.
Definition: PointerUnion.h:155
T dyn_cast() const
Returns the current pointer if it is of the specified pointer type, otherwise returns null.
Definition: PointerUnion.h:162
This class provides the information for the target register banks.
This class implements the register bank concept.
Definition: RegisterBank.h:28
unsigned getID() const
Get the identifier of this register bank.
Definition: RegisterBank.h:45
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
size_t size() const
Definition: SmallVector.h:91
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
unsigned getID() const
Return the register class ID number.
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
operand_type_match m_Reg()
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP > m_GICmp(const Pred &P, const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
unsigned getBrCond(CondCode CC)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition: LLVMContext.h:54
@ System
Synchronized with respect to all concurrently executing threads.
Definition: LLVMContext.h:57
Reg
All possible values of the reg field in the ModR/M byte.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:153
std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition: Utils.cpp:305
int countr_zero(T Val)
Count number of 0's from the least significant bit to the most stopping at the first 1.
Definition: bit.h:215
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
void reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, MachineOptimizationRemarkEmitter &MORE, MachineOptimizationRemarkMissed &R)
Report an ISel error as a missed optimization remark to the LLVMContext's diagnostic stream.
Definition: Utils.cpp:273
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Xor
Bitwise or logical XOR of integers.
InstructionSelector * createRISCVInstructionSelector(const RISCVTargetMachine &TM, RISCVSubtarget &Subtarget, RISCVRegisterBankInfo &RBI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define MORE()
Definition: regcomp.c:252
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Matching combinators.
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.