LLVM 24.0.0git
RISCVInstrInfo.cpp
Go to the documentation of this file.
1//===-- RISCVInstrInfo.cpp - RISC-V Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISC-V implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
16#include "RISCV.h"
18#include "RISCVSubtarget.h"
19#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Statistic.h"
32#include "llvm/IR/Module.h"
33#include "llvm/MC/MCDwarf.h"
37
38using namespace llvm;
39
40#define GEN_CHECK_COMPRESS_INSTR
41#include "RISCVGenCompressInstEmitter.inc"
42
43#define GET_INSTRINFO_CTOR_DTOR
44#include "RISCVGenInstrInfo.inc"
45
46#define DEBUG_TYPE "riscv-instr-info"
47STATISTIC(NumVRegSpilled,
48 "Number of registers within vector register groups spilled");
49STATISTIC(NumVRegReloaded,
50 "Number of registers within vector register groups reloaded");
51
53
54using namespace RISCV;
55
56#define GET_RISCVVPseudosTable_IMPL
57#include "RISCVGenSearchableTables.inc"
58
59} // namespace llvm::RISCVVPseudosTable
60
61namespace llvm::RISCV {
62
63#define GET_RISCVMaskedPseudosTable_IMPL
64#include "RISCVGenSearchableTables.inc"
65
66} // end namespace llvm::RISCV
67
69 : RISCVGenInstrInfo(STI, RegInfo, RISCV::ADJCALLSTACKDOWN,
70 RISCV::ADJCALLSTACKUP),
71 RegInfo(STI.getHwMode(), STI.getUserReservedXRegs()), STI(STI) {}
72
73#define GET_INSTRINFO_HELPERS
74#include "RISCVGenInstrInfo.inc"
75
77 if (STI.hasStdExtZca())
78 return MCInstBuilder(RISCV::C_NOP);
79 return MCInstBuilder(RISCV::ADDI)
80 .addReg(RISCV::X0)
81 .addReg(RISCV::X0)
82 .addImm(0);
83}
84
86 int &FrameIndex) const {
88 return isLoadFromStackSlot(MI, FrameIndex, Dummy);
89}
90
91static std::optional<unsigned> getLMULForRVVWholeLoadStore(unsigned Opcode) {
92 switch (Opcode) {
93 default:
94 return std::nullopt;
95 case RISCV::VS1R_V:
96 case RISCV::VL1RE8_V:
97 case RISCV::VL1RE16_V:
98 case RISCV::VL1RE32_V:
99 case RISCV::VL1RE64_V:
100 return 1;
101 case RISCV::VS2R_V:
102 case RISCV::VL2RE8_V:
103 case RISCV::VL2RE16_V:
104 case RISCV::VL2RE32_V:
105 case RISCV::VL2RE64_V:
106 return 2;
107 case RISCV::VS4R_V:
108 case RISCV::VL4RE8_V:
109 case RISCV::VL4RE16_V:
110 case RISCV::VL4RE32_V:
111 case RISCV::VL4RE64_V:
112 return 4;
113 case RISCV::VS8R_V:
114 case RISCV::VL8RE8_V:
115 case RISCV::VL8RE16_V:
116 case RISCV::VL8RE32_V:
117 case RISCV::VL8RE64_V:
118 return 8;
119 }
120}
121
123 int &FrameIndex,
124 TypeSize &MemBytes) const {
125 switch (MI.getOpcode()) {
126 default:
127 return 0;
128 case RISCV::LB:
129 case RISCV::LBU:
130 MemBytes = TypeSize::getFixed(1);
131 break;
132 case RISCV::LH:
133 case RISCV::LH_INX:
134 case RISCV::LHU:
135 case RISCV::FLH:
136 MemBytes = TypeSize::getFixed(2);
137 break;
138 case RISCV::LW:
139 case RISCV::LW_INX:
140 case RISCV::FLW:
141 case RISCV::LWU:
142 MemBytes = TypeSize::getFixed(4);
143 break;
144 case RISCV::LD:
145 case RISCV::LD_RV32:
146 case RISCV::FLD:
147 MemBytes = TypeSize::getFixed(8);
148 break;
149 case RISCV::VL1RE8_V:
150 case RISCV::VL2RE8_V:
151 case RISCV::VL4RE8_V:
152 case RISCV::VL8RE8_V:
153 if (!MI.getOperand(1).isFI())
154 return Register();
155 FrameIndex = MI.getOperand(1).getIndex();
156 unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());
158 return MI.getOperand(0).getReg();
159 }
160
161 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
162 MI.getOperand(2).getImm() == 0) {
163 FrameIndex = MI.getOperand(1).getIndex();
164 return MI.getOperand(0).getReg();
165 }
166
167 return 0;
168}
169
171 int &FrameIndex) const {
172 TypeSize Dummy = TypeSize::getZero();
173 return isStoreToStackSlot(MI, FrameIndex, Dummy);
174}
175
177 int &FrameIndex,
178 TypeSize &MemBytes) const {
179 switch (MI.getOpcode()) {
180 default:
181 return 0;
182 case RISCV::SB:
183 MemBytes = TypeSize::getFixed(1);
184 break;
185 case RISCV::SH:
186 case RISCV::SH_INX:
187 case RISCV::FSH:
188 MemBytes = TypeSize::getFixed(2);
189 break;
190 case RISCV::SW:
191 case RISCV::SW_INX:
192 case RISCV::FSW:
193 MemBytes = TypeSize::getFixed(4);
194 break;
195 case RISCV::SD:
196 case RISCV::SD_RV32:
197 case RISCV::FSD:
198 MemBytes = TypeSize::getFixed(8);
199 break;
200 case RISCV::VS1R_V:
201 case RISCV::VS2R_V:
202 case RISCV::VS4R_V:
203 case RISCV::VS8R_V:
204 if (!MI.getOperand(1).isFI())
205 return Register();
206 FrameIndex = MI.getOperand(1).getIndex();
207 unsigned LMUL = *getLMULForRVVWholeLoadStore(MI.getOpcode());
209 return MI.getOperand(0).getReg();
210 }
211
212 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
213 MI.getOperand(2).getImm() == 0) {
214 FrameIndex = MI.getOperand(1).getIndex();
215 return MI.getOperand(0).getReg();
216 }
217
218 return 0;
219}
220
222 const MachineInstr &MI) const {
223 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {
224 case RISCV::VMV_V_X:
225 case RISCV::VFMV_V_F:
226 case RISCV::VMV_V_I:
227 case RISCV::VMV_S_X:
228 case RISCV::VFMV_S_F:
229 case RISCV::VID_V:
230 return MI.getOperand(1).isUndef();
231 default:
233 }
234}
235
236static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
237 unsigned NumRegs) {
238 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
239}
240
242 const MachineBasicBlock &MBB,
245 RISCVVType::VLMUL LMul) {
246 if (STI.getCLOpts().prefer_whole_register_move)
247 return false;
248
249 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
250 "Unexpected COPY instruction.");
251 Register SrcReg = MBBI->getOperand(1).getReg();
253
254 bool FoundDef = false;
255 bool FirstVSetVLI = false;
256 unsigned FirstSEW = 0;
257 while (MBBI != MBB.begin()) {
258 --MBBI;
259 if (MBBI->isMetaInstruction())
260 continue;
261
262 if (RISCVInstrInfo::isVectorConfigInstr(*MBBI)) {
263 // There is a vsetvli between COPY and source define instruction.
264 // vy = def_vop ... (producing instruction)
265 // ...
266 // vsetvli
267 // ...
268 // vx = COPY vy
269 if (!FoundDef) {
270 if (!FirstVSetVLI) {
271 FirstVSetVLI = true;
272 unsigned FirstVType = MBBI->getOperand(2).getImm();
273 RISCVVType::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);
274 FirstSEW = RISCVVType::getSEW(FirstVType);
275 // The first encountered vsetvli must have the same lmul as the
276 // register class of COPY.
277 if (FirstLMul != LMul)
278 return false;
279 }
280 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
281 // define instruction.
282 if (!RISCVInstrInfo::isVLPreservingConfig(*MBBI))
283 return false;
284 continue;
285 }
286
287 // MBBI is the first vsetvli before the producing instruction.
288 unsigned VType = MBBI->getOperand(2).getImm();
289 // If there is a vsetvli between COPY and the producing instruction.
290 if (FirstVSetVLI) {
291 // If SEW is different, return false.
292 if (RISCVVType::getSEW(VType) != FirstSEW)
293 return false;
294 }
295
296 // If the vsetvli is tail undisturbed, keep the whole register move.
297 if (!RISCVVType::isTailAgnostic(VType))
298 return false;
299
300 // The checking is conservative. We only have register classes for
301 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
302 // for fractional LMUL operations. However, we could not use the vsetvli
303 // lmul for widening operations. The result of widening operation is
304 // 2 x LMUL.
305 return LMul == RISCVVType::getVLMUL(VType);
306 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
307 return false;
308 } else if (MBBI->getNumDefs()) {
309 // Check all the instructions which will change VL.
310 // For example, vleff has implicit def VL.
311 if (MBBI->modifiesRegister(RISCV::VL, /*TRI=*/nullptr))
312 return false;
313
314 // Only converting whole register copies to vmv.v.v when the defining
315 // value appears in the explicit operands.
316 for (const MachineOperand &MO : MBBI->explicit_operands()) {
317 if (!MO.isReg() || !MO.isDef())
318 continue;
319 if (!FoundDef && TRI->regsOverlap(MO.getReg(), SrcReg)) {
320 // We only permit the source of COPY has the same LMUL as the defined
321 // operand.
322 // There are cases we need to keep the whole register copy if the LMUL
323 // is different.
324 // For example,
325 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
326 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
327 // # The COPY may be created by vlmul_trunc intrinsic.
328 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
329 //
330 // After widening, the valid value will be 4 x e32 elements. If we
331 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
332 // FIXME: The COPY of subregister of Zvlsseg register will not be able
333 // to convert to vmv.v.[v|i] under the constraint.
334 if (MO.getReg() != SrcReg)
335 return false;
336
337 // In widening reduction instructions with LMUL_1 input vector case,
338 // only checking the LMUL is insufficient due to reduction result is
339 // always LMUL_1.
340 // For example,
341 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
342 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
343 // $v26 = COPY killed renamable $v8
344 // After widening, The valid value will be 1 x e16 elements. If we
345 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
346 uint64_t TSFlags = MBBI->getDesc().TSFlags;
348 return false;
349
350 // If the producing instruction does not depend on vsetvli, do not
351 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
352 if (!RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasVLOp(TSFlags))
353 return false;
354
355 // Found the definition.
356 FoundDef = true;
357 DefMBBI = MBBI;
358 break;
359 }
360 }
361 }
362 }
363
364 return false;
365}
366
369 const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc,
370 const TargetRegisterClass *RegClass) const {
371 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
373 unsigned NF = RISCVRI::getNF(RegClass->TSFlags);
374
375 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
376 uint16_t DstEncoding = TRI->getEncodingValue(DstReg);
377 auto [LMulVal, Fractional] = RISCVVType::decodeVLMUL(LMul);
378 assert(!Fractional && "It is impossible be fractional lmul here.");
379 unsigned NumRegs = NF * LMulVal;
380 bool ReversedCopy =
381 forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NumRegs);
382 if (ReversedCopy) {
383 // If the src and dest overlap when copying a tuple, we need to copy the
384 // registers in reverse.
385 SrcEncoding += NumRegs - 1;
386 DstEncoding += NumRegs - 1;
387 }
388
389 unsigned I = 0;
390 auto GetCopyInfo = [&](uint16_t SrcEncoding, uint16_t DstEncoding)
391 -> std::tuple<RISCVVType::VLMUL, const TargetRegisterClass &, unsigned,
392 unsigned, unsigned> {
393 if (ReversedCopy) {
394 // For reversed copying, if there are enough aligned registers(8/4/2), we
395 // can do a larger copy(LMUL8/4/2).
396 // Besides, we have already known that DstEncoding is larger than
397 // SrcEncoding in forwardCopyWillClobberTuple, so the difference between
398 // DstEncoding and SrcEncoding should be >= LMUL value we try to use to
399 // avoid clobbering.
400 uint16_t Diff = DstEncoding - SrcEncoding;
401 if (I + 8 <= NumRegs && Diff >= 8 && SrcEncoding % 8 == 7 &&
402 DstEncoding % 8 == 7)
403 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
404 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
405 if (I + 4 <= NumRegs && Diff >= 4 && SrcEncoding % 4 == 3 &&
406 DstEncoding % 4 == 3)
407 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
408 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
409 if (I + 2 <= NumRegs && Diff >= 2 && SrcEncoding % 2 == 1 &&
410 DstEncoding % 2 == 1)
411 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
412 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
413 // Or we should do LMUL1 copying.
414 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
415 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
416 }
417
418 // For forward copying, if source register encoding and destination register
419 // encoding are aligned to 8/4/2, we can do a LMUL8/4/2 copying.
420 if (I + 8 <= NumRegs && SrcEncoding % 8 == 0 && DstEncoding % 8 == 0)
421 return {RISCVVType::LMUL_8, RISCV::VRM8RegClass, RISCV::VMV8R_V,
422 RISCV::PseudoVMV_V_V_M8, RISCV::PseudoVMV_V_I_M8};
423 if (I + 4 <= NumRegs && SrcEncoding % 4 == 0 && DstEncoding % 4 == 0)
424 return {RISCVVType::LMUL_4, RISCV::VRM4RegClass, RISCV::VMV4R_V,
425 RISCV::PseudoVMV_V_V_M4, RISCV::PseudoVMV_V_I_M4};
426 if (I + 2 <= NumRegs && SrcEncoding % 2 == 0 && DstEncoding % 2 == 0)
427 return {RISCVVType::LMUL_2, RISCV::VRM2RegClass, RISCV::VMV2R_V,
428 RISCV::PseudoVMV_V_V_M2, RISCV::PseudoVMV_V_I_M2};
429 // Or we should do LMUL1 copying.
430 return {RISCVVType::LMUL_1, RISCV::VRRegClass, RISCV::VMV1R_V,
431 RISCV::PseudoVMV_V_V_M1, RISCV::PseudoVMV_V_I_M1};
432 };
433
434 while (I != NumRegs) {
435 // For non-segment copying, we only do this once as the registers are always
436 // aligned.
437 // For segment copying, we may do this several times. If the registers are
438 // aligned to larger LMUL, we can eliminate some copyings.
439 auto [LMulCopied, RegClass, Opc, VVOpc, VIOpc] =
440 GetCopyInfo(SrcEncoding, DstEncoding);
441 auto [NumCopied, _] = RISCVVType::decodeVLMUL(LMulCopied);
442
444 if (LMul == LMulCopied &&
445 isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
446 Opc = VVOpc;
447 if (DefMBBI->getOpcode() == VIOpc)
448 Opc = VIOpc;
449 }
450
451 // Emit actual copying.
452 // For reversed copying, the encoding should be decreased.
453 MCRegister ActualSrcReg = TRI->findVRegWithEncoding(
454 RegClass, ReversedCopy ? (SrcEncoding - NumCopied + 1) : SrcEncoding);
455 MCRegister ActualDstReg = TRI->findVRegWithEncoding(
456 RegClass, ReversedCopy ? (DstEncoding - NumCopied + 1) : DstEncoding);
457
458 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), ActualDstReg);
459 bool UseVMV_V_I = RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_I;
460 bool UseVMV = UseVMV_V_I || RISCV::getRVVMCOpcode(Opc) == RISCV::VMV_V_V;
461 if (UseVMV)
462 MIB.addReg(ActualDstReg, RegState::Undef);
463 if (UseVMV_V_I)
464 MIB = MIB.add(DefMBBI->getOperand(2));
465 else
466 MIB = MIB.addReg(ActualSrcReg, getKillRegState(KillSrc));
467 if (UseVMV) {
468 const MCInstrDesc &Desc = DefMBBI->getDesc();
469 MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
470 unsigned Log2SEW =
471 DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc)).getImm();
472 MIB.addImm(Log2SEW ? Log2SEW : 3); // SEW
473 MIB.addImm(0); // tu, mu
474 MIB.addReg(RISCV::VL, RegState::Implicit);
475 MIB.addReg(RISCV::VTYPE, RegState::Implicit);
476 }
477 // Add an implicit read of the original source to silence the verifier
478 // in the cases where some of the smaller VRs we're copying from might be
479 // undef, caused by the fact that the original, larger source VR might not
480 // be fully initialized at the time this COPY happens.
481 MIB.addReg(SrcReg, RegState::Implicit);
482
483 // If we are copying reversely, we should decrease the encoding.
484 SrcEncoding += (ReversedCopy ? -NumCopied : NumCopied);
485 DstEncoding += (ReversedCopy ? -NumCopied : NumCopied);
486 I += NumCopied;
487 }
488}
489
492 const DebugLoc &DL, Register DstReg,
493 Register SrcReg, bool KillSrc,
494 bool RenamableDest, bool RenamableSrc) const {
495 const RISCVRegisterInfo *TRI = STI.getRegisterInfo();
496 RegState KillFlag = getKillRegState(KillSrc);
497
498 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
499 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
500 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc))
501 .addImm(0);
502 return;
503 }
504
505 if (RISCV::GPRF16RegClass.contains(DstReg, SrcReg)) {
506 BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR16INX), DstReg)
507 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
508 return;
509 }
510
511 if (RISCV::GPRF32RegClass.contains(DstReg, SrcReg)) {
512 BuildMI(MBB, MBBI, DL, get(RISCV::PseudoMV_FPR32INX), DstReg)
513 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
514 return;
515 }
516
517 if (RISCV::GPRPairRegClass.contains(DstReg, SrcReg)) {
518 if (!STI.is64Bit()) {
519 if (STI.hasStdExtZdinx()) {
520 // On RV32_Zdinx, FMV.D will move a pair of registers to another pair of
521 // registers, in one instruction.
522 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D_IN32X), DstReg)
523 .addReg(SrcReg, getRenamableRegState(RenamableSrc))
524 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
525 return;
526 }
527
528 if (STI.hasStdExtP()) {
529 // On RV32P, `padd.dw` is a GPR Pair Add
530 BuildMI(MBB, MBBI, DL, get(RISCV::PADD_DW), DstReg)
531 .addReg(RISCV::X0_Pair)
532 .addReg(SrcReg, KillFlag | getRenamableRegState(RenamableSrc));
533 return;
534 }
535 }
536
537 auto [EvenReg, OddReg] = TRI->getGPRPairSubRegs(SrcReg);
538 assert(DstReg != RISCV::X0_Pair && "Cannot write to X0_Pair");
539
540 // Emit an ADDI for both parts of GPRPair.
541 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
542 TRI->getSubReg(DstReg, RISCV::sub_gpr_even))
543 .addReg(EvenReg, KillFlag)
544 .addImm(0);
545 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI),
546 TRI->getSubReg(DstReg, RISCV::sub_gpr_odd))
547 .addReg(OddReg, KillFlag)
548 .addImm(0);
549 return;
550 }
551
552 // Handle copy from csr
553 if (RISCV::VCSRRegClass.contains(SrcReg) &&
554 RISCV::GPRRegClass.contains(DstReg)) {
555 BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
556 .addImm(RISCVSysReg::lookupSysRegByName(TRI->getName(SrcReg))->Encoding)
557 .addReg(RISCV::X0);
558 return;
559 }
560
561 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
562 unsigned Opc;
563 if (STI.hasStdExtZfh()) {
564 Opc = RISCV::FSGNJ_H;
565 } else {
566 assert(STI.hasStdExtF() &&
567 (STI.hasStdExtZfhmin() || STI.hasStdExtZfbfmin()) &&
568 "Unexpected extensions");
569 // Zfhmin/Zfbfmin doesn't have FSGNJ_H, replace FSGNJ_H with FSGNJ_S.
570 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
571 &RISCV::FPR32RegClass);
572 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
573 &RISCV::FPR32RegClass);
574 Opc = RISCV::FSGNJ_S;
575 }
576 BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
577 .addReg(SrcReg, KillFlag)
578 .addReg(SrcReg, KillFlag);
579 return;
580 }
581
582 if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
583 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_S), DstReg)
584 .addReg(SrcReg, KillFlag)
585 .addReg(SrcReg, KillFlag);
586 return;
587 }
588
589 if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
590 BuildMI(MBB, MBBI, DL, get(RISCV::FSGNJ_D), DstReg)
591 .addReg(SrcReg, KillFlag)
592 .addReg(SrcReg, KillFlag);
593 return;
594 }
595
596 if (RISCV::FPR16RegClass.contains(DstReg) &&
597 RISCV::GPRRegClass.contains(SrcReg)) {
598 assert(STI.hasHalfFPLoadStoreMove() && "Unexpected extensions");
599 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_H_X), DstReg)
600 .addReg(SrcReg, KillFlag);
601 return;
602 }
603
604 if (RISCV::GPRRegClass.contains(DstReg) &&
605 RISCV::FPR16RegClass.contains(SrcReg)) {
606 assert(STI.hasHalfFPLoadStoreMove() && "Unexpected extensions");
607 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_H), DstReg)
608 .addReg(SrcReg, KillFlag);
609 return;
610 }
611
612 if (RISCV::FPR32RegClass.contains(DstReg) &&
613 RISCV::GPRRegClass.contains(SrcReg)) {
614 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_W_X), DstReg)
615 .addReg(SrcReg, KillFlag);
616 return;
617 }
618
619 if (RISCV::GPRRegClass.contains(DstReg) &&
620 RISCV::FPR32RegClass.contains(SrcReg)) {
621 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_W), DstReg)
622 .addReg(SrcReg, KillFlag);
623 return;
624 }
625
626 if (RISCV::FPR64RegClass.contains(DstReg) &&
627 RISCV::GPRRegClass.contains(SrcReg)) {
628 assert(STI.getXLen() == 64 && "Unexpected GPR size");
629 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_D_X), DstReg)
630 .addReg(SrcReg, KillFlag);
631 return;
632 }
633
634 if (RISCV::GPRRegClass.contains(DstReg) &&
635 RISCV::FPR64RegClass.contains(SrcReg)) {
636 assert(STI.getXLen() == 64 && "Unexpected GPR size");
637 BuildMI(MBB, MBBI, DL, get(RISCV::FMV_X_D), DstReg)
638 .addReg(SrcReg, KillFlag);
639 return;
640 }
641
642 // VR->VR copies.
643 const TargetRegisterClass *RegClass =
644 TRI->getCommonMinimalPhysRegClass(SrcReg, DstReg);
645 if (RISCVRegisterInfo::isRVVRegClass(RegClass)) {
646 copyPhysRegVector(MBB, MBBI, DL, DstReg, SrcReg, KillSrc, RegClass);
647 return;
648 }
649
650 llvm_unreachable("Impossible reg-to-reg copy");
651}
652
655 Register SrcReg, bool IsKill, int FI,
656 const TargetRegisterClass *RC,
657 Register VReg,
658 MachineInstr::MIFlag Flags) const {
659 MachineFunction *MF = MBB.getParent();
660 MachineFrameInfo &MFI = MF->getFrameInfo();
661 Align Alignment = MFI.getObjectAlign(FI);
662
663 unsigned Opcode;
664 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
665 Opcode = RegInfo.getRegSizeInBits(RISCV::GPRRegClass) == 32 ? RISCV::SW
666 : RISCV::SD;
667 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
668 Opcode = RISCV::SH_INX;
669 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
670 Opcode = RISCV::SW_INX;
671 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
672 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
673 Alignment >= STI.getZilsdAlign()) {
674 Opcode = RISCV::SD_RV32;
675 } else {
676 Opcode = RISCV::PseudoRV32ZdinxSD;
677 }
678 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
679 Opcode = RISCV::FSH;
680 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
681 Opcode = RISCV::FSW;
682 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
683 Opcode = RISCV::FSD;
684 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
685 Opcode = RISCV::VS1R_V;
686 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
687 Opcode = RISCV::VS2R_V;
688 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
689 Opcode = RISCV::VS4R_V;
690 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
691 Opcode = RISCV::VS8R_V;
692 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
693 Opcode = RISCV::PseudoVSPILL2_M1;
694 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
695 Opcode = RISCV::PseudoVSPILL2_M2;
696 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
697 Opcode = RISCV::PseudoVSPILL2_M4;
698 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
699 Opcode = RISCV::PseudoVSPILL3_M1;
700 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
701 Opcode = RISCV::PseudoVSPILL3_M2;
702 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
703 Opcode = RISCV::PseudoVSPILL4_M1;
704 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
705 Opcode = RISCV::PseudoVSPILL4_M2;
706 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
707 Opcode = RISCV::PseudoVSPILL5_M1;
708 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
709 Opcode = RISCV::PseudoVSPILL6_M1;
710 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
711 Opcode = RISCV::PseudoVSPILL7_M1;
712 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
713 Opcode = RISCV::PseudoVSPILL8_M1;
714 else
715 llvm_unreachable("Can't store this register to stack slot");
716
720 TypeSize::getScalable(MFI.getObjectSize(FI)), Alignment);
721
723 BuildMI(MBB, I, DebugLoc(), get(Opcode))
724 .addReg(SrcReg, getKillRegState(IsKill))
725 .addFrameIndex(FI)
726 .addMemOperand(MMO)
727 .setMIFlag(Flags);
728 NumVRegSpilled += RegInfo.getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;
729 } else {
732 MFI.getObjectSize(FI), Alignment);
733
734 BuildMI(MBB, I, DebugLoc(), get(Opcode))
735 .addReg(SrcReg, getKillRegState(IsKill))
736 .addFrameIndex(FI)
737 .addImm(0)
738 .addMemOperand(MMO)
739 .setMIFlag(Flags);
740 }
741}
742
745 Register DstReg, int FI,
746 const TargetRegisterClass *RC,
747 Register VReg, unsigned SubReg,
748 MachineInstr::MIFlag Flags) const {
749 MachineFunction *MF = MBB.getParent();
750 MachineFrameInfo &MFI = MF->getFrameInfo();
751 Align Alignment = MFI.getObjectAlign(FI);
752 DebugLoc DL =
753 Flags & MachineInstr::FrameDestroy ? MBB.findDebugLoc(I) : DebugLoc();
754
755 unsigned Opcode;
756 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
757 Opcode = RegInfo.getRegSizeInBits(RISCV::GPRRegClass) == 32 ? RISCV::LW
758 : RISCV::LD;
759 } else if (RISCV::GPRF16RegClass.hasSubClassEq(RC)) {
760 Opcode = RISCV::LH_INX;
761 } else if (RISCV::GPRF32RegClass.hasSubClassEq(RC)) {
762 Opcode = RISCV::LW_INX;
763 } else if (RISCV::GPRPairRegClass.hasSubClassEq(RC)) {
764 if (!STI.is64Bit() && STI.hasStdExtZilsd() &&
765 Alignment >= STI.getZilsdAlign()) {
766 Opcode = RISCV::LD_RV32;
767 } else {
768 Opcode = RISCV::PseudoRV32ZdinxLD;
769 }
770 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
771 Opcode = RISCV::FLH;
772 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
773 Opcode = RISCV::FLW;
774 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
775 Opcode = RISCV::FLD;
776 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
777 Opcode = RISCV::VL1RE8_V;
778 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
779 Opcode = RISCV::VL2RE8_V;
780 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
781 Opcode = RISCV::VL4RE8_V;
782 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
783 Opcode = RISCV::VL8RE8_V;
784 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
785 Opcode = RISCV::PseudoVRELOAD2_M1;
786 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
787 Opcode = RISCV::PseudoVRELOAD2_M2;
788 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
789 Opcode = RISCV::PseudoVRELOAD2_M4;
790 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
791 Opcode = RISCV::PseudoVRELOAD3_M1;
792 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
793 Opcode = RISCV::PseudoVRELOAD3_M2;
794 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
795 Opcode = RISCV::PseudoVRELOAD4_M1;
796 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
797 Opcode = RISCV::PseudoVRELOAD4_M2;
798 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
799 Opcode = RISCV::PseudoVRELOAD5_M1;
800 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
801 Opcode = RISCV::PseudoVRELOAD6_M1;
802 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
803 Opcode = RISCV::PseudoVRELOAD7_M1;
804 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
805 Opcode = RISCV::PseudoVRELOAD8_M1;
806 else
807 llvm_unreachable("Can't load this register from stack slot");
808
812 TypeSize::getScalable(MFI.getObjectSize(FI)), Alignment);
813
815 BuildMI(MBB, I, DL, get(Opcode), DstReg)
816 .addFrameIndex(FI)
817 .addMemOperand(MMO)
818 .setMIFlag(Flags);
819 NumVRegReloaded += RegInfo.getRegSizeInBits(*RC) / RISCV::RVVBitsPerBlock;
820 } else {
823 MFI.getObjectSize(FI), Alignment);
824
825 BuildMI(MBB, I, DL, get(Opcode), DstReg)
826 .addFrameIndex(FI)
827 .addImm(0)
828 .addMemOperand(MMO)
829 .setMIFlag(Flags);
830 }
831}
832std::optional<unsigned> getFoldedOpcode(MachineFunction &MF, MachineInstr &MI,
834 const RISCVSubtarget &ST) {
835
836 // The below optimizations narrow the load so they are only valid for little
837 // endian.
838 // TODO: Support big endian by adding an offset into the frame object?
839 if (MF.getDataLayout().isBigEndian())
840 return std::nullopt;
841
842 // Fold load from stack followed by sext.b/sext.h/sext.w/zext.b/zext.h/zext.w.
843 if (Ops.size() != 1 || Ops[0] != 1)
844 return std::nullopt;
845
846 switch (MI.getOpcode()) {
847 default:
848 if (RISCVInstrInfo::isSEXT_W(MI))
849 return RISCV::LW;
850 if (RISCVInstrInfo::isZEXT_W(MI))
851 return RISCV::LWU;
852 if (RISCVInstrInfo::isZEXT_B(MI))
853 return RISCV::LBU;
854 break;
855 case RISCV::SEXT_H:
856 return RISCV::LH;
857 case RISCV::SEXT_B:
858 return RISCV::LB;
859 case RISCV::ZEXT_H_RV32:
860 case RISCV::ZEXT_H_RV64:
861 return RISCV::LHU;
862 // A GPR reload feeding an int->FP move can be an FP load from the slot.
863 case RISCV::FMV_H_X:
864 return RISCV::FLH;
865 case RISCV::FMV_W_X:
866 return RISCV::FLW;
867 case RISCV::FMV_D_X:
868 return RISCV::FLD;
869 }
870
871 switch (RISCV::getRVVMCOpcode(MI.getOpcode())) {
872 default:
873 return std::nullopt;
874 case RISCV::VMV_X_S: {
875 unsigned Log2SEW =
876 MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
877 if (ST.getXLen() < (1U << Log2SEW))
878 return std::nullopt;
879 switch (Log2SEW) {
880 case 3:
881 return RISCV::LB;
882 case 4:
883 return RISCV::LH;
884 case 5:
885 return RISCV::LW;
886 case 6:
887 return RISCV::LD;
888 default:
889 llvm_unreachable("Unexpected SEW");
890 }
891 }
892 case RISCV::VFMV_F_S: {
893 unsigned Log2SEW =
894 MI.getOperand(RISCVII::getSEWOpNum(MI.getDesc())).getImm();
895 switch (Log2SEW) {
896 case 4:
897 return RISCV::FLH;
898 case 5:
899 return RISCV::FLW;
900 case 6:
901 return RISCV::FLD;
902 default:
903 llvm_unreachable("Unexpected SEW");
904 }
905 }
906 }
907}
908
909// This is the version used during InlineSpiller::spillAroundUses
912 ArrayRef<unsigned> Ops, int FrameIndex,
913 MachineInstr *&CopyMI, LiveIntervals *LIS,
914 VirtRegMap *VRM) const {
916 std::optional<unsigned> LoadOpc = getFoldedOpcode(MF, MI, Ops, STI);
917 if (!LoadOpc)
918 return nullptr;
919 Register DstReg = MI.getOperand(0).getReg();
920 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(*LoadOpc),
921 DstReg)
922 .addFrameIndex(FrameIndex)
923 .addImm(0);
924}
925
926static unsigned getLoadPredicatedOpcode(unsigned Opcode) {
927 switch (Opcode) {
928 case RISCV::LB:
929 return RISCV::PseudoCCLB;
930 case RISCV::LBU:
931 return RISCV::PseudoCCLBU;
932 case RISCV::LH:
933 return RISCV::PseudoCCLH;
934 case RISCV::LHU:
935 return RISCV::PseudoCCLHU;
936 case RISCV::LW:
937 return RISCV::PseudoCCLW;
938 case RISCV::LWU:
939 return RISCV::PseudoCCLWU;
940 case RISCV::LD:
941 return RISCV::PseudoCCLD;
942 case RISCV::QC_E_LB:
943 return RISCV::PseudoCCQC_E_LB;
944 case RISCV::QC_E_LBU:
945 return RISCV::PseudoCCQC_E_LBU;
946 case RISCV::QC_E_LH:
947 return RISCV::PseudoCCQC_E_LH;
948 case RISCV::QC_E_LHU:
949 return RISCV::PseudoCCQC_E_LHU;
950 case RISCV::QC_E_LW:
951 return RISCV::PseudoCCQC_E_LW;
952 default:
953 return 0;
954 }
955}
956
959 MachineInstr &LoadMI, MachineInstr *&CopyMI, LiveIntervals *LIS,
960 VirtRegMap *VRM) const {
962 // For now, only handle RISCV::PseudoCCMOVGPR.
963 if (MI.getOpcode() != RISCV::PseudoCCMOVGPR)
964 return nullptr;
965
966 unsigned PredOpc = getLoadPredicatedOpcode(LoadMI.getOpcode());
967
968 if (!STI.hasShortForwardBranchILoad() || !PredOpc)
969 return nullptr;
970
972 if (Ops.size() != 1 || (Ops[0] != 1 && Ops[0] != 2))
973 return nullptr;
974
975 bool Invert = Ops[0] == 2;
976 const MachineOperand &FalseReg = MI.getOperand(!Invert ? 2 : 1);
977 Register DestReg = MI.getOperand(0).getReg();
978 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
979 if (!MRI.constrainRegClass(DestReg, PreviousClass))
980 return nullptr;
981
982 // Create a new predicated version of DefMI.
983 MachineInstrBuilder NewMI = BuildMI(*MI.getParent(), InsertPt,
984 MI.getDebugLoc(), get(PredOpc), DestReg);
985
986 // Copy the false register.
987 NewMI.add(FalseReg);
988
989 // Copy all the DefMI operands.
990 const MCInstrDesc &DefDesc = LoadMI.getDesc();
991 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
992 NewMI.add(LoadMI.getOperand(i));
993
994 // Add branch opcode, inverting if necessary.
995 unsigned BCC = MI.getOperand(MI.getNumExplicitOperands() - 3).getImm();
996 if (!Invert)
998 NewMI.addImm(BCC);
999
1000 // Copy condition portion
1001 NewMI.add({MI.getOperand(MI.getNumExplicitOperands() - 2),
1002 MI.getOperand(MI.getNumExplicitOperands() - 1)});
1003 NewMI.cloneMemRefs(LoadMI);
1004 return NewMI;
1005}
1006
1009 const DebugLoc &DL, Register DstReg, uint64_t Val,
1010 MachineInstr::MIFlag Flag, bool DstRenamable,
1011 bool DstIsDead) const {
1012 Register SrcReg = RISCV::X0;
1013
1014 // For RV32, allow a sign or unsigned 32 bit value.
1015 if (!STI.is64Bit() && !isInt<32>(Val)) {
1016 // If have a uimm32 it will still fit in a register so we can allow it.
1017 if (!isUInt<32>(Val))
1018 report_fatal_error("Should only materialize 32-bit constants for RV32");
1019
1020 // Sign extend for generateInstSeq.
1021 Val = SignExtend64<32>(Val);
1022 }
1023
1025 assert(!Seq.empty());
1026
1027 bool SrcRenamable = false;
1028 unsigned Num = 0;
1029
1030 for (const RISCVMatInt::Inst &Inst : Seq) {
1031 bool LastItem = ++Num == Seq.size();
1032 RegState DstRegState = getDeadRegState(DstIsDead && LastItem) |
1033 getRenamableRegState(DstRenamable);
1034 RegState SrcRegState = getKillRegState(SrcReg != RISCV::X0) |
1035 getRenamableRegState(SrcRenamable);
1036 switch (Inst.getOpndKind()) {
1037 case RISCVMatInt::Imm:
1038 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
1039 .addDef(DstReg, DstRegState)
1040 .addImm(Inst.getImm())
1041 .setMIFlag(Flag);
1042 break;
1043 case RISCVMatInt::RegX0:
1044 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
1045 .addDef(DstReg, DstRegState)
1046 .addReg(SrcReg, SrcRegState)
1047 .addReg(RISCV::X0)
1048 .setMIFlag(Flag);
1049 break;
1051 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
1052 .addDef(DstReg, DstRegState)
1053 .addReg(SrcReg, SrcRegState)
1054 .addReg(SrcReg, SrcRegState)
1055 .setMIFlag(Flag);
1056 break;
1058 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()))
1059 .addDef(DstReg, DstRegState)
1060 .addReg(SrcReg, SrcRegState)
1061 .addImm(Inst.getImm())
1062 .setMIFlag(Flag);
1063 break;
1064 }
1065
1066 // Only the first instruction has X0 as its source.
1067 SrcReg = DstReg;
1068 SrcRenamable = DstRenamable;
1069 }
1070}
1071
1073 switch (Opc) {
1074 default:
1075 return RISCVCC::COND_INVALID;
1076 case RISCV::BEQ:
1077 case RISCV::BEQI:
1078 case RISCV::CV_BEQIMM:
1079 case RISCV::QC_BEQI:
1080 case RISCV::QC_E_BEQI:
1081 case RISCV::NDS_BBC:
1082 case RISCV::NDS_BEQC:
1083 return RISCVCC::COND_EQ;
1084 case RISCV::BNE:
1085 case RISCV::BNEI:
1086 case RISCV::QC_BNEI:
1087 case RISCV::QC_E_BNEI:
1088 case RISCV::CV_BNEIMM:
1089 case RISCV::NDS_BBS:
1090 case RISCV::NDS_BNEC:
1091 return RISCVCC::COND_NE;
1092 case RISCV::BLT:
1093 case RISCV::QC_BLTI:
1094 case RISCV::QC_E_BLTI:
1095 return RISCVCC::COND_LT;
1096 case RISCV::BGE:
1097 case RISCV::QC_BGEI:
1098 case RISCV::QC_E_BGEI:
1099 return RISCVCC::COND_GE;
1100 case RISCV::BLTU:
1101 case RISCV::QC_BLTUI:
1102 case RISCV::QC_E_BLTUI:
1103 return RISCVCC::COND_LTU;
1104 case RISCV::BGEU:
1105 case RISCV::QC_BGEUI:
1106 case RISCV::QC_E_BGEUI:
1107 return RISCVCC::COND_GEU;
1108 }
1109}
1110
1112 int64_t C1) {
1113 switch (CC) {
1114 default:
1115 llvm_unreachable("Unexpected CC");
1116 case RISCVCC::COND_EQ:
1117 return C0 == C1;
1118 case RISCVCC::COND_NE:
1119 return C0 != C1;
1120 case RISCVCC::COND_LT:
1121 return C0 < C1;
1122 case RISCVCC::COND_GE:
1123 return C0 >= C1;
1124 case RISCVCC::COND_LTU:
1125 return (uint64_t)C0 < (uint64_t)C1;
1126 case RISCVCC::COND_GEU:
1127 return (uint64_t)C0 >= (uint64_t)C1;
1128 }
1129}
1130
1131// The contents of values added to Cond are not examined outside of
1132// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
1133// push BranchOpcode, Reg1, Reg2.
1136 // Block ends with fall-through condbranch.
1137 assert(LastInst.getDesc().isConditionalBranch() &&
1138 "Unknown conditional branch");
1139 Target = LastInst.getOperand(2).getMBB();
1140 Cond.push_back(MachineOperand::CreateImm(LastInst.getOpcode()));
1141 Cond.push_back(LastInst.getOperand(0));
1142 Cond.push_back(LastInst.getOperand(1));
1143}
1144
1145static unsigned getInverseXqcicmOpcode(unsigned Opcode) {
1146 switch (Opcode) {
1147 default:
1148 llvm_unreachable("Unexpected Opcode");
1149 case RISCV::QC_MVEQ:
1150 return RISCV::QC_MVNE;
1151 case RISCV::QC_MVNE:
1152 return RISCV::QC_MVEQ;
1153 case RISCV::QC_MVLT:
1154 return RISCV::QC_MVGE;
1155 case RISCV::QC_MVGE:
1156 return RISCV::QC_MVLT;
1157 case RISCV::QC_MVLTU:
1158 return RISCV::QC_MVGEU;
1159 case RISCV::QC_MVGEU:
1160 return RISCV::QC_MVLTU;
1161 case RISCV::QC_MVEQI:
1162 return RISCV::QC_MVNEI;
1163 case RISCV::QC_MVNEI:
1164 return RISCV::QC_MVEQI;
1165 case RISCV::QC_MVLTI:
1166 return RISCV::QC_MVGEI;
1167 case RISCV::QC_MVGEI:
1168 return RISCV::QC_MVLTI;
1169 case RISCV::QC_MVLTUI:
1170 return RISCV::QC_MVGEUI;
1171 case RISCV::QC_MVGEUI:
1172 return RISCV::QC_MVLTUI;
1173 }
1174}
1175
1176unsigned RISCVCC::getBrCond(RISCVCC::CondCode CC, unsigned SelectOpc) {
1177 switch (SelectOpc) {
1178 default:
1179 switch (CC) {
1180 default:
1181 llvm_unreachable("Unexpected condition code!");
1182 case RISCVCC::COND_EQ:
1183 return RISCV::BEQ;
1184 case RISCVCC::COND_NE:
1185 return RISCV::BNE;
1186 case RISCVCC::COND_LT:
1187 return RISCV::BLT;
1188 case RISCVCC::COND_GE:
1189 return RISCV::BGE;
1190 case RISCVCC::COND_LTU:
1191 return RISCV::BLTU;
1192 case RISCVCC::COND_GEU:
1193 return RISCV::BGEU;
1194 }
1195 break;
1196 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
1197 switch (CC) {
1198 default:
1199 llvm_unreachable("Unexpected condition code!");
1200 case RISCVCC::COND_EQ:
1201 return RISCV::BEQI;
1202 case RISCVCC::COND_NE:
1203 return RISCV::BNEI;
1204 }
1205 break;
1206 case RISCV::Select_GPR_Using_CC_SImm5_CV:
1207 switch (CC) {
1208 default:
1209 llvm_unreachable("Unexpected condition code!");
1210 case RISCVCC::COND_EQ:
1211 return RISCV::CV_BEQIMM;
1212 case RISCVCC::COND_NE:
1213 return RISCV::CV_BNEIMM;
1214 }
1215 break;
1216 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
1217 switch (CC) {
1218 default:
1219 llvm_unreachable("Unexpected condition code!");
1220 case RISCVCC::COND_EQ:
1221 return RISCV::QC_BEQI;
1222 case RISCVCC::COND_NE:
1223 return RISCV::QC_BNEI;
1224 case RISCVCC::COND_LT:
1225 return RISCV::QC_BLTI;
1226 case RISCVCC::COND_GE:
1227 return RISCV::QC_BGEI;
1228 }
1229 break;
1230 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
1231 switch (CC) {
1232 default:
1233 llvm_unreachable("Unexpected condition code!");
1234 case RISCVCC::COND_LTU:
1235 return RISCV::QC_BLTUI;
1236 case RISCVCC::COND_GEU:
1237 return RISCV::QC_BGEUI;
1238 }
1239 break;
1240 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
1241 switch (CC) {
1242 default:
1243 llvm_unreachable("Unexpected condition code!");
1244 case RISCVCC::COND_EQ:
1245 return RISCV::QC_E_BEQI;
1246 case RISCVCC::COND_NE:
1247 return RISCV::QC_E_BNEI;
1248 case RISCVCC::COND_LT:
1249 return RISCV::QC_E_BLTI;
1250 case RISCVCC::COND_GE:
1251 return RISCV::QC_E_BGEI;
1252 }
1253 break;
1254 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
1255 switch (CC) {
1256 default:
1257 llvm_unreachable("Unexpected condition code!");
1258 case RISCVCC::COND_LTU:
1259 return RISCV::QC_E_BLTUI;
1260 case RISCVCC::COND_GEU:
1261 return RISCV::QC_E_BGEUI;
1262 }
1263 break;
1264 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
1265 switch (CC) {
1266 default:
1267 llvm_unreachable("Unexpected condition code!");
1268 case RISCVCC::COND_EQ:
1269 return RISCV::NDS_BBC;
1270 case RISCVCC::COND_NE:
1271 return RISCV::NDS_BBS;
1272 }
1273 break;
1274 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
1275 switch (CC) {
1276 default:
1277 llvm_unreachable("Unexpected condition code!");
1278 case RISCVCC::COND_EQ:
1279 return RISCV::NDS_BEQC;
1280 case RISCVCC::COND_NE:
1281 return RISCV::NDS_BNEC;
1282 }
1283 break;
1284 }
1285}
1286
1288 switch (CC) {
1289 default:
1290 llvm_unreachable("Unrecognized conditional branch");
1291 case RISCVCC::COND_EQ:
1292 return RISCVCC::COND_NE;
1293 case RISCVCC::COND_NE:
1294 return RISCVCC::COND_EQ;
1295 case RISCVCC::COND_LT:
1296 return RISCVCC::COND_GE;
1297 case RISCVCC::COND_GE:
1298 return RISCVCC::COND_LT;
1299 case RISCVCC::COND_LTU:
1300 return RISCVCC::COND_GEU;
1301 case RISCVCC::COND_GEU:
1302 return RISCVCC::COND_LTU;
1303 }
1304}
1305
1306// Return inverse branch
1307unsigned RISCVCC::getInverseBranchOpcode(unsigned BCC) {
1308 switch (BCC) {
1309 default:
1310 llvm_unreachable("Unexpected branch opcode!");
1311 case RISCV::BEQ:
1312 return RISCV::BNE;
1313 case RISCV::BEQI:
1314 return RISCV::BNEI;
1315 case RISCV::BNE:
1316 return RISCV::BEQ;
1317 case RISCV::BNEI:
1318 return RISCV::BEQI;
1319 case RISCV::BLT:
1320 return RISCV::BGE;
1321 case RISCV::BGE:
1322 return RISCV::BLT;
1323 case RISCV::BLTU:
1324 return RISCV::BGEU;
1325 case RISCV::BGEU:
1326 return RISCV::BLTU;
1327 case RISCV::CV_BEQIMM:
1328 return RISCV::CV_BNEIMM;
1329 case RISCV::CV_BNEIMM:
1330 return RISCV::CV_BEQIMM;
1331 case RISCV::QC_BEQI:
1332 return RISCV::QC_BNEI;
1333 case RISCV::QC_BNEI:
1334 return RISCV::QC_BEQI;
1335 case RISCV::QC_BLTI:
1336 return RISCV::QC_BGEI;
1337 case RISCV::QC_BGEI:
1338 return RISCV::QC_BLTI;
1339 case RISCV::QC_BLTUI:
1340 return RISCV::QC_BGEUI;
1341 case RISCV::QC_BGEUI:
1342 return RISCV::QC_BLTUI;
1343 case RISCV::QC_E_BEQI:
1344 return RISCV::QC_E_BNEI;
1345 case RISCV::QC_E_BNEI:
1346 return RISCV::QC_E_BEQI;
1347 case RISCV::QC_E_BLTI:
1348 return RISCV::QC_E_BGEI;
1349 case RISCV::QC_E_BGEI:
1350 return RISCV::QC_E_BLTI;
1351 case RISCV::QC_E_BLTUI:
1352 return RISCV::QC_E_BGEUI;
1353 case RISCV::QC_E_BGEUI:
1354 return RISCV::QC_E_BLTUI;
1355 case RISCV::NDS_BBC:
1356 return RISCV::NDS_BBS;
1357 case RISCV::NDS_BBS:
1358 return RISCV::NDS_BBC;
1359 case RISCV::NDS_BEQC:
1360 return RISCV::NDS_BNEC;
1361 case RISCV::NDS_BNEC:
1362 return RISCV::NDS_BEQC;
1363 }
1364}
1365
1368 MachineBasicBlock *&FBB,
1370 bool AllowModify) const {
1371 TBB = FBB = nullptr;
1372 Cond.clear();
1373
1374 // If the block has no terminators, it just falls into the block after it.
1375 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1376 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1377 return false;
1378
1379 // Count the number of terminators and find the first unconditional or
1380 // indirect branch.
1381 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
1382 int NumTerminators = 0;
1383 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
1384 J++) {
1385 NumTerminators++;
1386 if (J->getDesc().isUnconditionalBranch() ||
1387 J->getDesc().isIndirectBranch()) {
1388 FirstUncondOrIndirectBr = J.getReverse();
1389 }
1390 }
1391
1392 // If AllowModify is true, we can erase any terminators after
1393 // FirstUncondOrIndirectBR.
1394 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
1395 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
1396 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
1397 NumTerminators--;
1398 }
1399 I = FirstUncondOrIndirectBr;
1400 }
1401
1402 // We can't handle blocks that end in an indirect branch.
1403 if (I->getDesc().isIndirectBranch())
1404 return true;
1405
1406 // We can't handle Generic branch opcodes from Global ISel.
1407 if (I->isPreISelOpcode())
1408 return true;
1409
1410 // We can't handle blocks with more than 2 terminators.
1411 if (NumTerminators > 2)
1412 return true;
1413
1414 // Handle a single unconditional branch.
1415 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
1417 return false;
1418 }
1419
1420 // Handle a single conditional branch.
1421 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
1423 return false;
1424 }
1425
1426 // Handle a conditional branch followed by an unconditional branch.
1427 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
1428 I->getDesc().isUnconditionalBranch()) {
1429 parseCondBranch(*std::prev(I), TBB, Cond);
1430 FBB = getBranchDestBlock(*I);
1431 return false;
1432 }
1433
1434 // Otherwise, we can't handle this.
1435 return true;
1436}
1437
1439 int *BytesRemoved) const {
1440 if (BytesRemoved)
1441 *BytesRemoved = 0;
1442 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
1443 if (I == MBB.end())
1444 return 0;
1445
1446 if (!I->getDesc().isUnconditionalBranch() &&
1447 !I->getDesc().isConditionalBranch())
1448 return 0;
1449
1450 // Remove the branch.
1451 if (BytesRemoved)
1452 *BytesRemoved += getInstSizeInBytes(*I);
1453 I->eraseFromParent();
1454
1455 I = MBB.end();
1456
1457 if (I == MBB.begin())
1458 return 1;
1459 --I;
1460 if (!I->getDesc().isConditionalBranch())
1461 return 1;
1462
1463 // Remove the branch.
1464 if (BytesRemoved)
1465 *BytesRemoved += getInstSizeInBytes(*I);
1466 I->eraseFromParent();
1467 return 2;
1468}
1469
1470// Inserts a branch into the end of the specific MachineBasicBlock, returning
1471// the number of instructions inserted.
1474 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
1475 if (BytesAdded)
1476 *BytesAdded = 0;
1477
1478 // Shouldn't be a fall through.
1479 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1480 assert((Cond.size() == 3 || Cond.size() == 0) &&
1481 "RISC-V branch conditions have two components!");
1482
1483 // Unconditional branch.
1484 if (Cond.empty()) {
1485 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
1486 if (BytesAdded)
1487 *BytesAdded += getInstSizeInBytes(MI);
1488 return 1;
1489 }
1490
1491 // Either a one or two-way conditional branch.
1492 MachineInstr &CondMI = *BuildMI(&MBB, DL, get(Cond[0].getImm()))
1493 .add(Cond[1])
1494 .add(Cond[2])
1495 .addMBB(TBB);
1496 if (BytesAdded)
1497 *BytesAdded += getInstSizeInBytes(CondMI);
1498
1499 // One-way conditional branch.
1500 if (!FBB)
1501 return 1;
1502
1503 // Two-way conditional branch.
1504 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
1505 if (BytesAdded)
1506 *BytesAdded += getInstSizeInBytes(MI);
1507 return 2;
1508}
1509
1511 MachineBasicBlock &DestBB,
1512 MachineBasicBlock &RestoreBB,
1513 const DebugLoc &DL, int64_t BrOffset,
1514 RegScavenger *RS) const {
1515 assert(RS && "RegScavenger required for long branching");
1516 assert(MBB.empty() &&
1517 "new block should be inserted for expanding unconditional branch");
1518 assert(MBB.pred_size() == 1);
1519 assert(RestoreBB.empty() &&
1520 "restore block should be inserted for restoring clobbered registers");
1521
1522 MachineFunction *MF = MBB.getParent();
1526
1527 if (!isInt<32>(BrOffset))
1529 "Branch offsets outside of the signed 32-bit range not supported");
1530
1531 // FIXME: A virtual register must be used initially, as the register
1532 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
1533 // uses the same workaround).
1534 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRJALRRegClass);
1535 auto II = MBB.end();
1536 // We may also update the jump target to RestoreBB later.
1537 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
1538 .addDef(ScratchReg, RegState::Dead)
1539 .addMBB(&DestBB, RISCVII::MO_CALL);
1540
1541 RS->enterBasicBlockEnd(MBB);
1542 // When cf-protection-branch is enabled, we must use t2 (x7) for software
1543 // guarded branches to hold the landing pad label.
1544 bool HasCFBranch =
1545 MF->getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
1546 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1547 if (HasCFBranch)
1548 RC = &RISCV::GPRX7RegClass;
1549 Register TmpGPR =
1550 RS->scavengeRegisterBackwards(*RC, MI.getIterator(),
1551 /*RestoreAfter=*/false, /*SpAdj=*/0,
1552 /*AllowSpill=*/false);
1553 if (TmpGPR.isValid())
1554 RS->setRegUsed(TmpGPR);
1555 else {
1556 // The case when there is no scavenged register needs special handling.
1557
1558 // Pick s11(or s1 for rve) because it doesn't make a difference.
1559 TmpGPR = STI.hasStdExtE() ? RISCV::X9 : RISCV::X27;
1560 // Force t2 if cf-protection-branch is enabled
1561 if (HasCFBranch)
1562 TmpGPR = RISCV::X7;
1563
1564 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
1565 if (FrameIndex == -1)
1566 report_fatal_error("underestimated function size");
1567
1568 storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
1569 &RISCV::GPRRegClass, Register());
1570 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
1571 /*SpAdj=*/0, /*FIOperandNum=*/1);
1572
1573 MI.getOperand(1).setMBB(&RestoreBB);
1574
1575 loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
1576 &RISCV::GPRRegClass, Register());
1577 TRI->eliminateFrameIndex(RestoreBB.back(),
1578 /*SpAdj=*/0, /*FIOperandNum=*/1);
1579 }
1580
1581 MRI.replaceRegWith(ScratchReg, TmpGPR);
1582 MRI.clearVirtRegs();
1583}
1584
1587 assert((Cond.size() == 3) && "Invalid branch condition!");
1588
1590
1591 return false;
1592}
1593
1594// Return true if the instruction is a load immediate instruction (i.e.
1595// (ADDI x0, imm) or (BSETI x0, imm)).
1596static bool isLoadImm(const MachineInstr *MI, int64_t &Imm) {
1597 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
1598 MI->getOperand(1).getReg() == RISCV::X0) {
1599 Imm = MI->getOperand(2).getImm();
1600 return true;
1601 }
1602 // BSETI can be used to create power of 2 constants. Only 2048 is currently
1603 // interesting because it is 1 more than the maximum ADDI constant.
1604 if (MI->getOpcode() == RISCV::BSETI && MI->getOperand(1).isReg() &&
1605 MI->getOperand(1).getReg() == RISCV::X0 &&
1606 MI->getOperand(2).getImm() == 11) {
1607 Imm = 2048;
1608 return true;
1609 }
1610 return false;
1611}
1612
1614 const MachineOperand &Op, int64_t &Imm) {
1615 // Either a load from immediate instruction or X0.
1616 if (!Op.isReg())
1617 return false;
1618
1619 Register Reg = Op.getReg();
1620 if (Reg == RISCV::X0) {
1621 Imm = 0;
1622 return true;
1623 }
1624
1625 if (!Reg.isVirtual())
1626 return false;
1627
1628 const MachineInstr *DefMI = MRI.getVRegDef(Reg);
1629 return DefMI && isLoadImm(DefMI, Imm);
1630}
1631
1633 bool IsSigned = false;
1634 bool IsEquality = false;
1635 switch (MI.getOpcode()) {
1636 default:
1637 return false;
1638 case RISCV::BEQ:
1639 case RISCV::BNE:
1640 IsEquality = true;
1641 break;
1642 case RISCV::BGE:
1643 case RISCV::BLT:
1644 IsSigned = true;
1645 break;
1646 case RISCV::BGEU:
1647 case RISCV::BLTU:
1648 break;
1649 }
1650
1651 MachineBasicBlock *MBB = MI.getParent();
1652 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
1653
1654 const MachineOperand &LHS = MI.getOperand(0);
1655 const MachineOperand &RHS = MI.getOperand(1);
1656 MachineBasicBlock *TBB = MI.getOperand(2).getMBB();
1657
1660
1661 // Canonicalize conditional branches which can be constant folded into
1662 // beqz or bnez. We can't modify the CFG here.
1663 int64_t C0, C1;
1664 if (isFromLoadImm(MRI, LHS, C0) && isFromLoadImm(MRI, RHS, C1)) {
1665 unsigned NewOpc = evaluateCondBranch(CC, C0, C1) ? RISCV::BEQ : RISCV::BNE;
1666 // Build the new branch and remove the old one.
1667 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1668 .addReg(RISCV::X0)
1669 .addReg(RISCV::X0)
1670 .addMBB(TBB);
1671 MI.eraseFromParent();
1672 return true;
1673 }
1674
1675 if (IsEquality)
1676 return false;
1677
1678 // For two constants C0 and C1 from
1679 // ```
1680 // li Y, C0
1681 // li Z, C1
1682 // ```
1683 // 1. if C1 = C0 + 1
1684 // we can turn:
1685 // (a) blt Y, X -> bge X, Z
1686 // (b) bge Y, X -> blt X, Z
1687 //
1688 // 2. if C1 = C0 - 1
1689 // we can turn:
1690 // (a) blt X, Y -> bge Z, X
1691 // (b) bge X, Y -> blt Z, X
1692 //
1693 // To make sure this optimization is really beneficial, we only
1694 // optimize for cases where Y had only one use (i.e. only used by the branch).
1695 // Try to find the register for constant Z; return
1696 // invalid register otherwise.
1697 auto searchConst = [&](int64_t C1) -> Register {
1699 auto DefC1 = std::find_if(++II, E, [&](const MachineInstr &I) -> bool {
1700 int64_t Imm;
1701 return isLoadImm(&I, Imm) && Imm == C1 &&
1702 I.getOperand(0).getReg().isVirtual();
1703 });
1704 if (DefC1 != E)
1705 return DefC1->getOperand(0).getReg();
1706
1707 return Register();
1708 };
1709
1710 unsigned NewOpc = RISCVCC::getBrCond(getInverseBranchCondition(CC));
1711
1712 // Might be case 1.
1713 // Don't change 0 to 1 since we can use x0.
1714 // For unsigned cases changing -1U to 0 would be incorrect.
1715 // The incorrect case for signed would be INT_MAX, but isFromLoadImm can't
1716 // return that.
1717 if (isFromLoadImm(MRI, LHS, C0) && C0 != 0 && LHS.getReg().isVirtual() &&
1718 MRI.hasOneUse(LHS.getReg()) && (IsSigned || C0 != -1)) {
1719 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1720 if (Register RegZ = searchConst(C0 + 1)) {
1721 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1722 .add(RHS)
1723 .addReg(RegZ)
1724 .addMBB(TBB);
1725 MI.eraseFromParent();
1726 return true;
1727 }
1728 }
1729
1730 // Might be case 2.
1731 // For signed cases we don't want to change 0 since we can use x0.
1732 // For unsigned cases changing 0 to -1U would be incorrect.
1733 // The incorrect case for signed would be INT_MIN, but isFromLoadImm can't
1734 // return that.
1735 if (isFromLoadImm(MRI, RHS, C0) && C0 != 0 && RHS.getReg().isVirtual() &&
1736 MRI.hasOneUse(RHS.getReg())) {
1737 assert((isInt<12>(C0) || C0 == 2048) && "Unexpected immediate");
1738 if (Register RegZ = searchConst(C0 - 1)) {
1739 BuildMI(*MBB, MI, MI.getDebugLoc(), get(NewOpc))
1740 .addReg(RegZ)
1741 .add(LHS)
1742 .addMBB(TBB);
1743 MI.eraseFromParent();
1744 return true;
1745 }
1746 }
1747
1748 return false;
1749}
1750
1753 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1754 // The branch target is always the last operand.
1755 int NumOp = MI.getNumExplicitOperands();
1756 return MI.getOperand(NumOp - 1).getMBB();
1757}
1758
1760 int64_t BrOffset) const {
1761 unsigned XLen = STI.getXLen();
1762 // Ideally we could determine the supported branch offset from the
1763 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1764 // PseudoBR.
1765 switch (BranchOp) {
1766 default:
1767 llvm_unreachable("Unexpected opcode!");
1768 case RISCV::NDS_BBC:
1769 case RISCV::NDS_BBS:
1770 case RISCV::NDS_BEQC:
1771 case RISCV::NDS_BNEC:
1772 return isInt<11>(BrOffset);
1773 case RISCV::BEQ:
1774 case RISCV::BNE:
1775 case RISCV::BLT:
1776 case RISCV::BGE:
1777 case RISCV::BLTU:
1778 case RISCV::BGEU:
1779 case RISCV::BEQI:
1780 case RISCV::BNEI:
1781 case RISCV::CV_BEQIMM:
1782 case RISCV::CV_BNEIMM:
1783 case RISCV::QC_BEQI:
1784 case RISCV::QC_BNEI:
1785 case RISCV::QC_BGEI:
1786 case RISCV::QC_BLTI:
1787 case RISCV::QC_BLTUI:
1788 case RISCV::QC_BGEUI:
1789 case RISCV::QC_E_BEQI:
1790 case RISCV::QC_E_BNEI:
1791 case RISCV::QC_E_BGEI:
1792 case RISCV::QC_E_BLTI:
1793 case RISCV::QC_E_BLTUI:
1794 case RISCV::QC_E_BGEUI:
1795 return isInt<13>(BrOffset);
1796 case RISCV::JAL:
1797 case RISCV::PseudoBR:
1798 return isInt<21>(BrOffset);
1799 case RISCV::PseudoJump:
1800 return isInt<32>(SignExtend64(BrOffset + 0x800, XLen));
1801 }
1802}
1803
1804static bool isJumpTableLoad(const MachineInstr &MI) {
1805 return any_of(MI.memoperands(), [](const MachineMemOperand *MMO) {
1806 const PseudoSourceValue *PSV = MMO->getPseudoValue();
1807 return PSV && PSV->isJumpTable();
1808 });
1809}
1810
1811// We want this instruction to be loading the base address of a jump table into
1812// a register. This can be PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI.
1814 Register Reg) {
1815 if (!Reg.isVirtual())
1816 return -1;
1817 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1818 if (!MI)
1819 return -1;
1820
1821 for (const MachineOperand &MO : MI->operands())
1822 if (MO.isJTI())
1823 return MO.getIndex();
1824
1825 return -1;
1826}
1827
1828// This instruction is used as the base address of a jump table load. We expect
1829// it to be adding the jump table base address to an index that may be scaled.
1831 Register Reg) {
1832 if (!Reg.isVirtual())
1833 return -1;
1834 const MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
1835 if (!MI)
1836 return -1;
1837
1838 int JTI;
1839 switch (MI->getOpcode()) {
1840 case RISCV::SH1ADD:
1841 case RISCV::SH2ADD:
1842 case RISCV::SH3ADD:
1843 // Only the index should be scaled so we just check the unscaled operand for
1844 // the base address.
1845 // TODO: Can the address be SHXADD_UW?
1846 JTI = getJumpTableIndexFromBase(MRI, MI->getOperand(2).getReg());
1847 if (JTI >= 0)
1848 return JTI;
1849 break;
1850 case RISCV::ADD:
1851 JTI = getJumpTableIndexFromBase(MRI, MI->getOperand(1).getReg());
1852 if (JTI >= 0)
1853 return JTI;
1854 JTI = getJumpTableIndexFromBase(MRI, MI->getOperand(2).getReg());
1855 if (JTI >= 0)
1856 return JTI;
1857 break;
1858 }
1859
1860 return -1;
1861}
1862
1863// Recursively search for %jump-table.N starting from PseudoBRIND,
1864// and return the index of %jump-table.N.
1865//
1866// One common jump table:
1867//
1868// %base = PseudoMovAddr/PseudoLLA/LUI(+ADDI)/QC_E_LI %jump-table.N
1869// %addr = SH2ADD %index, %base
1870// %entry = LW %addr, 0 :: (load from jump-table)
1871// %target = ADD %entry, %base
1872// PseudoBRIND %target, 0
1873//
1875 if (MI.getOpcode() != RISCV::PseudoBRIND &&
1876 MI.getOpcode() != RISCV::PseudoBRINDX7)
1877 return -1;
1878
1879 Register Reg = MI.getOperand(0).getReg();
1880 if (!Reg.isVirtual())
1881 return -1;
1882
1883 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1884 MachineInstr *Def = MRI.getUniqueVRegDef(Reg);
1885 if (!Def)
1886 return -1;
1887
1888 // The target may come directly from a load or the jump table may store
1889 // relative offset that needs the table base added to it.
1890 int JTI;
1891 switch (Def->getOpcode()) {
1892 case RISCV::LW:
1893 case RISCV::LWU:
1894 case RISCV::LD:
1895 if (!isJumpTableLoad(*Def))
1896 return -1;
1897
1898 JTI = getJumpTableIndexFromLoadAddr(MRI, Def->getOperand(1).getReg());
1899 if (JTI >= 0)
1900 return JTI;
1901 break;
1902 case RISCV::LXSW:
1903 case RISCV::LXWU:
1904 case RISCV::LXSD:
1905 case RISCV::QC_LRW:
1906 if (!isJumpTableLoad(*Def))
1907 return -1;
1908
1909 JTI = getJumpTableIndexFromBase(MRI, Def->getOperand(1).getReg());
1910 if (JTI >= 0)
1911 return JTI;
1912 break;
1913 case RISCV::ADD:
1914 JTI = getJumpTableIndexFromBase(MRI, Def->getOperand(1).getReg());
1915 if (JTI >= 0)
1916 return JTI;
1917 JTI = getJumpTableIndexFromBase(MRI, Def->getOperand(2).getReg());
1918 if (JTI >= 0)
1919 return JTI;
1920 break;
1921 }
1922
1923 return -1;
1924}
1925
1926// If the operation has a predicated pseudo instruction, return the pseudo
1927// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1928// TODO: Support more operations.
1929unsigned getPredicatedOpcode(unsigned Opcode) {
1930 // clang-format off
1931 switch (Opcode) {
1932 case RISCV::ADD: return RISCV::PseudoCCADD;
1933 case RISCV::SUB: return RISCV::PseudoCCSUB;
1934 case RISCV::SLL: return RISCV::PseudoCCSLL;
1935 case RISCV::SRL: return RISCV::PseudoCCSRL;
1936 case RISCV::SRA: return RISCV::PseudoCCSRA;
1937 case RISCV::AND: return RISCV::PseudoCCAND;
1938 case RISCV::OR: return RISCV::PseudoCCOR;
1939 case RISCV::XOR: return RISCV::PseudoCCXOR;
1940 case RISCV::MAX: return RISCV::PseudoCCMAX;
1941 case RISCV::MAXU: return RISCV::PseudoCCMAXU;
1942 case RISCV::MIN: return RISCV::PseudoCCMIN;
1943 case RISCV::MINU: return RISCV::PseudoCCMINU;
1944 case RISCV::MUL: return RISCV::PseudoCCMUL;
1945 case RISCV::LUI: return RISCV::PseudoCCLUI;
1946 case RISCV::QC_LI: return RISCV::PseudoCCQC_LI;
1947 case RISCV::QC_E_LI: return RISCV::PseudoCCQC_E_LI;
1948
1949 case RISCV::ADDI: return RISCV::PseudoCCADDI;
1950 case RISCV::SLLI: return RISCV::PseudoCCSLLI;
1951 case RISCV::SRLI: return RISCV::PseudoCCSRLI;
1952 case RISCV::SRAI: return RISCV::PseudoCCSRAI;
1953 case RISCV::ANDI: return RISCV::PseudoCCANDI;
1954 case RISCV::ORI: return RISCV::PseudoCCORI;
1955 case RISCV::XORI: return RISCV::PseudoCCXORI;
1956
1957 case RISCV::ADDW: return RISCV::PseudoCCADDW;
1958 case RISCV::SUBW: return RISCV::PseudoCCSUBW;
1959 case RISCV::SLLW: return RISCV::PseudoCCSLLW;
1960 case RISCV::SRLW: return RISCV::PseudoCCSRLW;
1961 case RISCV::SRAW: return RISCV::PseudoCCSRAW;
1962
1963 case RISCV::ADDIW: return RISCV::PseudoCCADDIW;
1964 case RISCV::SLLIW: return RISCV::PseudoCCSLLIW;
1965 case RISCV::SRLIW: return RISCV::PseudoCCSRLIW;
1966 case RISCV::SRAIW: return RISCV::PseudoCCSRAIW;
1967
1968 case RISCV::ANDN: return RISCV::PseudoCCANDN;
1969 case RISCV::ORN: return RISCV::PseudoCCORN;
1970 case RISCV::XNOR: return RISCV::PseudoCCXNOR;
1971
1972 case RISCV::NDS_BFOS: return RISCV::PseudoCCNDS_BFOS;
1973 case RISCV::NDS_BFOZ: return RISCV::PseudoCCNDS_BFOZ;
1974 }
1975 // clang-format on
1976
1977 return RISCV::INSTRUCTION_LIST_END;
1978}
1979
1980/// Identify instructions that can be folded into a CCMOV instruction, and
1981/// return the defining instruction.
1983 const MachineRegisterInfo &MRI,
1984 const TargetInstrInfo *TII,
1985 const RISCVSubtarget &STI) {
1986 if (!Reg.isVirtual())
1987 return nullptr;
1988 if (!MRI.hasOneNonDBGUse(Reg))
1989 return nullptr;
1990 MachineInstr *MI = MRI.getVRegDef(Reg);
1991 if (!MI)
1992 return nullptr;
1993
1994 if (!STI.hasShortForwardBranchIMinMax() &&
1995 (MI->getOpcode() == RISCV::MAX || MI->getOpcode() == RISCV::MIN ||
1996 MI->getOpcode() == RISCV::MINU || MI->getOpcode() == RISCV::MAXU))
1997 return nullptr;
1998
1999 if (!STI.hasShortForwardBranchIMul() && MI->getOpcode() == RISCV::MUL)
2000 return nullptr;
2001
2002 // Check if MI can be predicated and folded into the CCMOV.
2003 if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
2004 return nullptr;
2005 // Don't predicate li idiom.
2006 if (MI->getOpcode() == RISCV::ADDI && MI->getOperand(1).isReg() &&
2007 MI->getOperand(1).getReg() == RISCV::X0)
2008 return nullptr;
2009 // Check if MI has any other defs or physreg uses.
2010 for (const MachineOperand &MO : llvm::drop_begin(MI->operands())) {
2011 // Reject frame index operands, PEI can't handle the predicated pseudos.
2012 if (MO.isFI() || MO.isCPI() || MO.isJTI())
2013 return nullptr;
2014 if (!MO.isReg())
2015 continue;
2016 // MI can't have any tied operands, that would conflict with predication.
2017 if (MO.isTied())
2018 return nullptr;
2019 if (MO.isDef())
2020 return nullptr;
2021 // Allow constant physregs.
2022 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))
2023 return nullptr;
2024 }
2025 bool DontMoveAcrossStores = true;
2026 if (!MI->isSafeToMove(DontMoveAcrossStores))
2027 return nullptr;
2028 return MI;
2029}
2030
2034 bool PreferFalse) const {
2035 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
2036 "Unknown select instruction");
2037 if (!STI.hasShortForwardBranchIALU())
2038 return nullptr;
2039
2040 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
2042 canFoldAsPredicatedOp(MI.getOperand(2).getReg(), MRI, this, STI);
2043 bool Invert = !DefMI;
2044 if (!DefMI)
2045 DefMI = canFoldAsPredicatedOp(MI.getOperand(1).getReg(), MRI, this, STI);
2046 if (!DefMI)
2047 return nullptr;
2048
2049 // Find new register class to use.
2050 MachineOperand FalseReg = MI.getOperand(Invert ? 2 : 1);
2051 Register DestReg = MI.getOperand(0).getReg();
2052 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
2053 if (!MRI.constrainRegClass(DestReg, PreviousClass))
2054 return nullptr;
2055
2056 unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());
2057 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
2058
2059 // Create a new predicated version of DefMI.
2060 MachineInstrBuilder NewMI =
2061 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
2062
2063 // Copy the false register.
2064 NewMI.add(FalseReg);
2065
2066 // Copy all the DefMI operands.
2067 const MCInstrDesc &DefDesc = DefMI->getDesc();
2068 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
2069 NewMI.add(DefMI->getOperand(i));
2070
2071 // Add branch opcode, inverting if necessary.
2072 unsigned BCCOpcode = MI.getOperand(MI.getNumExplicitOperands() - 3).getImm();
2073 if (Invert)
2074 BCCOpcode = RISCVCC::getInverseBranchOpcode(BCCOpcode);
2075 NewMI.addImm(BCCOpcode);
2076
2077 // Copy the condition portion.
2078 NewMI.add(MI.getOperand(MI.getNumExplicitOperands() - 2));
2079 NewMI.add(MI.getOperand(MI.getNumExplicitOperands() - 1));
2080
2081 // Update SeenMIs set: register newly created MI and erase removed DefMI.
2082 SeenMIs.insert(NewMI);
2083 SeenMIs.erase(DefMI);
2084
2085 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
2086 // DefMI would be invalid when transferred inside the loop. Checking for a
2087 // loop is expensive, but at least remove kill flags if they are in different
2088 // BBs.
2089 if (DefMI->getParent() != MI.getParent())
2090 NewMI->clearKillInfo();
2091
2092 // The caller will erase MI, but not DefMI.
2093 DefMI->eraseFromParent();
2094 return NewMI;
2095}
2096
2098 if (MI.isMetaInstruction())
2099 return 0;
2100
2101 unsigned Opcode = MI.getOpcode();
2102
2103 if (Opcode == TargetOpcode::INLINEASM ||
2104 Opcode == TargetOpcode::INLINEASM_BR) {
2105 const MachineFunction &MF = *MI.getParent()->getParent();
2106 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
2107 MF.getTarget().getMCAsmInfo());
2108 }
2109
2110 if (requiresNTLHint(MI)) {
2111 if (STI.hasStdExtZca()) {
2112 if (unsigned Size = getCompressedSize(MI, STI))
2113 return 2 + Size; // c.ntl.all + c.load/c.store
2114 return 6; // c.ntl.all + load/store
2115 }
2116 return 8; // ntl.all + load/store
2117 }
2118
2119 if (Opcode == TargetOpcode::BUNDLE)
2120 return getInstBundleSize(MI);
2121
2122 if (MI.getParent() && MI.getParent()->getParent()) {
2123 if (unsigned Size = getCompressedSize(MI, STI))
2124 return Size;
2125 }
2126
2127 switch (Opcode) {
2128 case RISCV::PseudoMV_FPR16INX:
2129 case RISCV::PseudoMV_FPR32INX:
2130 case RISCV::PseudoClearGPR:
2131 // MV is always compressible to either c.mv or c.li rd, 0.
2132 return STI.hasStdExtZca() ? 2 : 4;
2133 // Below cases are for short forward branch pseudos
2134 case RISCV::PseudoCCMOVGPRNoX0:
2135 return get(MI.getOperand(MI.getNumExplicitOperands() - 3).getImm())
2136 .getSize() +
2137 2;
2138 case RISCV::PseudoCCMOVGPR:
2139 case RISCV::PseudoCCADD:
2140 case RISCV::PseudoCCSUB:
2141 case RISCV::PseudoCCSLL:
2142 case RISCV::PseudoCCSRL:
2143 case RISCV::PseudoCCSRA:
2144 case RISCV::PseudoCCAND:
2145 case RISCV::PseudoCCOR:
2146 case RISCV::PseudoCCXOR:
2147 case RISCV::PseudoCCADDI:
2148 case RISCV::PseudoCCANDI:
2149 case RISCV::PseudoCCORI:
2150 case RISCV::PseudoCCXORI:
2151 case RISCV::PseudoCCLUI:
2152 case RISCV::PseudoCCSLLI:
2153 case RISCV::PseudoCCSRLI:
2154 case RISCV::PseudoCCSRAI:
2155 case RISCV::PseudoCCADDW:
2156 case RISCV::PseudoCCSUBW:
2157 case RISCV::PseudoCCSLLW:
2158 case RISCV::PseudoCCSRLW:
2159 case RISCV::PseudoCCSRAW:
2160 case RISCV::PseudoCCADDIW:
2161 case RISCV::PseudoCCSLLIW:
2162 case RISCV::PseudoCCSRLIW:
2163 case RISCV::PseudoCCSRAIW:
2164 case RISCV::PseudoCCANDN:
2165 case RISCV::PseudoCCORN:
2166 case RISCV::PseudoCCXNOR:
2167 case RISCV::PseudoCCMAX:
2168 case RISCV::PseudoCCMIN:
2169 case RISCV::PseudoCCMAXU:
2170 case RISCV::PseudoCCMINU:
2171 case RISCV::PseudoCCMUL:
2172 case RISCV::PseudoCCLB:
2173 case RISCV::PseudoCCLH:
2174 case RISCV::PseudoCCLW:
2175 case RISCV::PseudoCCLHU:
2176 case RISCV::PseudoCCLBU:
2177 case RISCV::PseudoCCLWU:
2178 case RISCV::PseudoCCLD:
2179 case RISCV::PseudoCCQC_LI:
2180 return get(MI.getOperand(MI.getNumExplicitOperands() - 3).getImm())
2181 .getSize() +
2182 4;
2183 case RISCV::PseudoCCQC_E_LI:
2184 case RISCV::PseudoCCQC_E_LB:
2185 case RISCV::PseudoCCQC_E_LH:
2186 case RISCV::PseudoCCQC_E_LW:
2187 case RISCV::PseudoCCQC_E_LHU:
2188 case RISCV::PseudoCCQC_E_LBU:
2189 return get(MI.getOperand(MI.getNumExplicitOperands() - 3).getImm())
2190 .getSize() +
2191 6;
2192 case TargetOpcode::STACKMAP:
2193 // The upper bound for a stackmap intrinsic is the full length of its shadow
2195 case TargetOpcode::PATCHPOINT:
2196 // The size of the patchpoint intrinsic is the number of bytes requested
2198 case TargetOpcode::STATEPOINT: {
2199 // The size of the statepoint intrinsic is the number of bytes requested
2200 unsigned NumBytes = StatepointOpers(&MI).getNumPatchBytes();
2201 // No patch bytes means at most a PseudoCall is emitted
2202 return std::max(NumBytes, 8U);
2203 }
2204 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2205 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
2206 case TargetOpcode::PATCHABLE_TAIL_CALL: {
2207 const MachineFunction &MF = *MI.getParent()->getParent();
2208 const Function &F = MF.getFunction();
2209 if (Opcode == TargetOpcode::PATCHABLE_FUNCTION_ENTER &&
2210 F.hasFnAttribute("patchable-function-entry")) {
2211 unsigned Num =
2212 F.getFnAttributeAsParsedInteger("patchable-function-entry");
2213 // Number of C.NOP or NOP
2214 return (STI.hasStdExtZca() ? 2 : 4) * Num;
2215 }
2216 // XRay uses C.JAL + 21 or 33 C.NOP for each sled in RV32 and RV64,
2217 // respectively.
2218 return STI.is64Bit() ? 68 : 44;
2219 }
2220 default:
2221 return get(Opcode).getSize();
2222 }
2223}
2224
2226 const unsigned Opcode = MI.getOpcode();
2227 switch (Opcode) {
2228 default:
2229 break;
2230 case RISCV::FSGNJ_D:
2231 case RISCV::FSGNJ_S:
2232 case RISCV::FSGNJ_H:
2233 case RISCV::FSGNJ_D_INX:
2234 case RISCV::FSGNJ_D_IN32X:
2235 case RISCV::FSGNJ_S_INX:
2236 case RISCV::FSGNJ_H_INX:
2237 // The canonical floating-point move is fsgnj rd, rs, rs.
2238 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
2239 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
2240 case RISCV::ADDI:
2241 case RISCV::ORI:
2242 case RISCV::XORI:
2243 return (MI.getOperand(1).isReg() &&
2244 MI.getOperand(1).getReg() == RISCV::X0) ||
2245 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
2246 }
2247 return MI.isAsCheapAsAMove();
2248}
2249
2250std::optional<DestSourcePair>
2252 if (MI.isMoveReg())
2253 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2254 switch (MI.getOpcode()) {
2255 default:
2256 break;
2257 case RISCV::ADD:
2258 case RISCV::OR:
2259 case RISCV::XOR:
2260 if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&
2261 MI.getOperand(2).isReg())
2262 return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
2263 if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&
2264 MI.getOperand(1).isReg())
2265 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2266 break;
2267 case RISCV::ADDI:
2268 // Operand 1 can be a frameindex but callers expect registers
2269 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
2270 MI.getOperand(2).getImm() == 0)
2271 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2272 break;
2273 case RISCV::SUB:
2274 if (MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0 &&
2275 MI.getOperand(1).isReg())
2276 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2277 break;
2278 case RISCV::SH1ADD:
2279 case RISCV::SH1ADD_UW:
2280 case RISCV::SH2ADD:
2281 case RISCV::SH2ADD_UW:
2282 case RISCV::SH3ADD:
2283 case RISCV::SH3ADD_UW:
2284 if (MI.getOperand(1).isReg() && MI.getOperand(1).getReg() == RISCV::X0 &&
2285 MI.getOperand(2).isReg())
2286 return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
2287 break;
2288 case RISCV::FSGNJ_D:
2289 case RISCV::FSGNJ_S:
2290 case RISCV::FSGNJ_H:
2291 case RISCV::FSGNJ_D_INX:
2292 case RISCV::FSGNJ_D_IN32X:
2293 case RISCV::FSGNJ_S_INX:
2294 case RISCV::FSGNJ_H_INX:
2295 // The canonical floating-point move is fsgnj rd, rs, rs.
2296 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
2297 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
2298 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
2299 break;
2300 }
2301 return std::nullopt;
2302}
2303
2305 std::optional<MachineTraceStrategy> Forced =
2306 STI.getCLOpts().force_machine_combiner_strategy;
2307 if (!Forced) {
2308 // The option is unused. Choose Local strategy only for in-order cores. When
2309 // scheduling model is unspecified, use MinInstrCount strategy as more
2310 // generic one.
2311 const auto &SchedModel = STI.getSchedModel();
2312 return (!SchedModel.hasInstrSchedModel() || SchedModel.isOutOfOrder())
2315 }
2316 // The strategy was forced by the option.
2317 return *Forced;
2318}
2319
2321 MachineInstr &Root, unsigned &Pattern,
2322 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
2323 int16_t FrmOpIdx =
2324 RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
2325 if (FrmOpIdx < 0) {
2326 assert(all_of(InsInstrs,
2327 [](MachineInstr *MI) {
2328 return RISCV::getNamedOperandIdx(MI->getOpcode(),
2329 RISCV::OpName::frm) < 0;
2330 }) &&
2331 "New instructions require FRM whereas the old one does not have it");
2332 return;
2333 }
2334
2335 const MachineOperand &FRM = Root.getOperand(FrmOpIdx);
2336 MachineFunction &MF = *Root.getMF();
2337
2338 for (auto *NewMI : InsInstrs) {
2339 // We'd already added the FRM operand.
2340 if (static_cast<unsigned>(RISCV::getNamedOperandIdx(
2341 NewMI->getOpcode(), RISCV::OpName::frm)) != NewMI->getNumOperands())
2342 continue;
2343 MachineInstrBuilder MIB(MF, NewMI);
2344 MIB.add(FRM);
2345 if (FRM.getImm() == RISCVFPRndMode::DYN)
2346 MIB.addUse(RISCV::FRM, RegState::Implicit);
2347 }
2348}
2349
2350static bool isFADD(unsigned Opc) {
2351 switch (Opc) {
2352 default:
2353 return false;
2354 case RISCV::FADD_H:
2355 case RISCV::FADD_S:
2356 case RISCV::FADD_D:
2357 return true;
2358 }
2359}
2360
2361static bool isFSUB(unsigned Opc) {
2362 switch (Opc) {
2363 default:
2364 return false;
2365 case RISCV::FSUB_H:
2366 case RISCV::FSUB_S:
2367 case RISCV::FSUB_D:
2368 return true;
2369 }
2370}
2371
2372static bool isFMUL(unsigned Opc) {
2373 switch (Opc) {
2374 default:
2375 return false;
2376 case RISCV::FMUL_H:
2377 case RISCV::FMUL_S:
2378 case RISCV::FMUL_D:
2379 return true;
2380 }
2381}
2382
2383bool RISCVInstrInfo::isVectorAssociativeAndCommutative(const MachineInstr &Inst,
2384 bool Invert) const {
2385#define OPCODE_LMUL_CASE(OPC) \
2386 case RISCV::OPC##_M1: \
2387 case RISCV::OPC##_M2: \
2388 case RISCV::OPC##_M4: \
2389 case RISCV::OPC##_M8: \
2390 case RISCV::OPC##_MF2: \
2391 case RISCV::OPC##_MF4: \
2392 case RISCV::OPC##_MF8
2393
2394#define OPCODE_LMUL_MASK_CASE(OPC) \
2395 case RISCV::OPC##_M1_MASK: \
2396 case RISCV::OPC##_M2_MASK: \
2397 case RISCV::OPC##_M4_MASK: \
2398 case RISCV::OPC##_M8_MASK: \
2399 case RISCV::OPC##_MF2_MASK: \
2400 case RISCV::OPC##_MF4_MASK: \
2401 case RISCV::OPC##_MF8_MASK
2402
2403 unsigned Opcode = Inst.getOpcode();
2404 if (Invert) {
2405 if (auto InvOpcode = getInverseOpcode(Opcode))
2406 Opcode = *InvOpcode;
2407 else
2408 return false;
2409 }
2410
2411 // clang-format off
2412 switch (Opcode) {
2413 default:
2414 return false;
2415 OPCODE_LMUL_CASE(PseudoVADD_VV):
2416 OPCODE_LMUL_MASK_CASE(PseudoVADD_VV):
2417 OPCODE_LMUL_CASE(PseudoVMUL_VV):
2418 OPCODE_LMUL_MASK_CASE(PseudoVMUL_VV):
2419 return true;
2420 }
2421 // clang-format on
2422
2423#undef OPCODE_LMUL_MASK_CASE
2424#undef OPCODE_LMUL_CASE
2425}
2426
2427bool RISCVInstrInfo::areRVVInstsReassociable(const MachineInstr &Root,
2428 const MachineInstr &Prev) const {
2429 if (!areOpcodesEqualOrInverse(Root.getOpcode(), Prev.getOpcode()))
2430 return false;
2431
2432 assert(Root.getMF() == Prev.getMF());
2433 const MachineRegisterInfo *MRI = &Root.getMF()->getRegInfo();
2434 const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
2435
2436 // Make sure vtype operands are also the same.
2437 const MCInstrDesc &Desc = get(Root.getOpcode());
2438 const uint64_t TSFlags = Desc.TSFlags;
2439
2440 auto checkImmOperand = [&](unsigned OpIdx) {
2441 return Root.getOperand(OpIdx).getImm() == Prev.getOperand(OpIdx).getImm();
2442 };
2443
2444 auto checkRegOperand = [&](unsigned OpIdx) {
2445 return Root.getOperand(OpIdx).getReg() == Prev.getOperand(OpIdx).getReg();
2446 };
2447
2448 // PassThru
2449 // TODO: Potentially we can loosen the condition to consider Root to be
2450 // associable with Prev if Root has NoReg as passthru. In which case we
2451 // also need to loosen the condition on vector policies between these.
2452 if (!checkRegOperand(1))
2453 return false;
2454
2455 // SEW
2456 if (RISCVII::hasSEWOp(TSFlags) &&
2457 !checkImmOperand(RISCVII::getSEWOpNum(Desc)))
2458 return false;
2459
2460 // Mask
2461 if (RISCVII::usesMaskPolicy(TSFlags)) {
2462 const MachineBasicBlock *MBB = Root.getParent();
2465 Register MI1VReg;
2466
2467 bool SeenMI2 = false;
2468 for (auto End = MBB->rend(), It = It1; It != End; ++It) {
2469 if (It == It2) {
2470 SeenMI2 = true;
2471 if (!MI1VReg.isValid())
2472 // There is no V0 def between Root and Prev; they're sharing the
2473 // same V0.
2474 break;
2475 }
2476
2477 if (It->modifiesRegister(RISCV::V0, TRI)) {
2478 Register SrcReg = It->getOperand(1).getReg();
2479 // If it's not VReg it'll be more difficult to track its defs, so
2480 // bailing out here just to be safe.
2481 if (!SrcReg.isVirtual())
2482 return false;
2483
2484 if (!MI1VReg.isValid()) {
2485 // This is the V0 def for Root.
2486 MI1VReg = SrcReg;
2487 continue;
2488 }
2489
2490 // Some random mask updates.
2491 if (!SeenMI2)
2492 continue;
2493
2494 // This is the V0 def for Prev; check if it's the same as that of
2495 // Root.
2496 if (MI1VReg != SrcReg)
2497 return false;
2498 else
2499 break;
2500 }
2501 }
2502
2503 // If we haven't encountered Prev, it's likely that this function was
2504 // called in a wrong way (e.g. Root is before Prev).
2505 assert(SeenMI2 && "Prev is expected to appear before Root");
2506 }
2507
2508 // Tail / Mask policies
2509 if (RISCVII::hasVecPolicyOp(TSFlags) &&
2510 !checkImmOperand(RISCVII::getVecPolicyOpNum(Desc)))
2511 return false;
2512
2513 // VL
2514 if (RISCVII::hasVLOp(TSFlags)) {
2515 unsigned OpIdx = RISCVII::getVLOpNum(Desc);
2516 const MachineOperand &Op1 = Root.getOperand(OpIdx);
2517 const MachineOperand &Op2 = Prev.getOperand(OpIdx);
2518 if (Op1.getType() != Op2.getType())
2519 return false;
2520 switch (Op1.getType()) {
2522 if (Op1.getReg() != Op2.getReg())
2523 return false;
2524 break;
2526 if (Op1.getImm() != Op2.getImm())
2527 return false;
2528 break;
2529 default:
2530 llvm_unreachable("Unrecognized VL operand type");
2531 }
2532 }
2533
2534 // Rounding modes
2535 if (int Idx = RISCVII::getFRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2536 return false;
2537 if (int Idx = RISCVII::getVXRMOpNum(Desc); Idx >= 0 && !checkImmOperand(Idx))
2538 return false;
2539
2540 return true;
2541}
2542
2543// Most of our RVV pseudos have passthru operand, so the real operands
2544// start from index = 2.
2545bool RISCVInstrInfo::hasReassociableVectorSibling(const MachineInstr &Inst,
2546 bool &Commuted) const {
2547 const MachineBasicBlock *MBB = Inst.getParent();
2548 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2550 "Expect the present of passthrough operand.");
2551 MachineInstr *MI1 = MRI.getUniqueVRegDef(Inst.getOperand(2).getReg());
2552 MachineInstr *MI2 = MRI.getUniqueVRegDef(Inst.getOperand(3).getReg());
2553
2554 // If only one operand has the same or inverse opcode and it's the second
2555 // source operand, the operands must be commuted.
2556 Commuted = !areRVVInstsReassociable(Inst, *MI1) &&
2557 areRVVInstsReassociable(Inst, *MI2);
2558 if (Commuted)
2559 std::swap(MI1, MI2);
2560
2561 return areRVVInstsReassociable(Inst, *MI1) &&
2562 (isVectorAssociativeAndCommutative(*MI1) ||
2563 isVectorAssociativeAndCommutative(*MI1, /* Invert */ true)) &&
2566}
2567
2569 const MachineInstr &Inst, const MachineBasicBlock *MBB) const {
2570 if (!isVectorAssociativeAndCommutative(Inst) &&
2571 !isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2573
2574 const MachineOperand &Op1 = Inst.getOperand(2);
2575 const MachineOperand &Op2 = Inst.getOperand(3);
2576 const MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
2577
2578 // We need virtual register definitions for the operands that we will
2579 // reassociate.
2580 MachineInstr *MI1 = nullptr;
2581 MachineInstr *MI2 = nullptr;
2582 if (Op1.isReg() && Op1.getReg().isVirtual())
2583 MI1 = MRI.getUniqueVRegDef(Op1.getReg());
2584 if (Op2.isReg() && Op2.getReg().isVirtual())
2585 MI2 = MRI.getUniqueVRegDef(Op2.getReg());
2586
2587 // And at least one operand must be defined in MBB.
2588 return MI1 && MI2 && (MI1->getParent() == MBB || MI2->getParent() == MBB);
2589}
2590
2592 const MachineInstr &Root, unsigned Pattern,
2593 std::array<unsigned, 5> &OperandIndices) const {
2595 if (RISCV::getRVVMCOpcode(Root.getOpcode())) {
2596 // Skip the passthrough operand, so increment all indices by one.
2597 for (unsigned I = 0; I < 5; ++I)
2598 ++OperandIndices[I];
2599 }
2600}
2601
2603 bool &Commuted) const {
2604 if (isVectorAssociativeAndCommutative(Inst) ||
2605 isVectorAssociativeAndCommutative(Inst, /*Invert=*/true))
2606 return hasReassociableVectorSibling(Inst, Commuted);
2607
2608 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
2609 return false;
2610
2611 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
2612 unsigned OperandIdx = Commuted ? 2 : 1;
2613 const MachineInstr &Sibling =
2614 *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());
2615
2616 int16_t InstFrmOpIdx =
2617 RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
2618 int16_t SiblingFrmOpIdx =
2619 RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
2620
2621 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
2622 RISCV::hasEqualFRM(Inst, Sibling);
2623}
2624
2626 bool Invert) const {
2627 if (isVectorAssociativeAndCommutative(Inst, Invert))
2628 return true;
2629
2630 unsigned Opc = Inst.getOpcode();
2631 if (Invert) {
2632 auto InverseOpcode = getInverseOpcode(Opc);
2633 if (!InverseOpcode)
2634 return false;
2635 Opc = *InverseOpcode;
2636 }
2637
2638 if (isFADD(Opc) || isFMUL(Opc))
2641
2642 switch (Opc) {
2643 default:
2644 return false;
2645 case RISCV::ADD:
2646 case RISCV::ADDW:
2647 case RISCV::AND:
2648 case RISCV::OR:
2649 case RISCV::XOR:
2650 // From RISC-V ISA spec, if both the high and low bits of the same product
2651 // are required, then the recommended code sequence is:
2652 //
2653 // MULH[[S]U] rdh, rs1, rs2
2654 // MUL rdl, rs1, rs2
2655 // (source register specifiers must be in same order and rdh cannot be the
2656 // same as rs1 or rs2)
2657 //
2658 // Microarchitectures can then fuse these into a single multiply operation
2659 // instead of performing two separate multiplies.
2660 // MachineCombiner may reassociate MUL operands and lose the fusion
2661 // opportunity.
2662 case RISCV::MUL:
2663 case RISCV::MULW:
2664 case RISCV::MIN:
2665 case RISCV::MINU:
2666 case RISCV::MAX:
2667 case RISCV::MAXU:
2668 case RISCV::FMIN_H:
2669 case RISCV::FMIN_S:
2670 case RISCV::FMIN_D:
2671 case RISCV::FMAX_H:
2672 case RISCV::FMAX_S:
2673 case RISCV::FMAX_D:
2674 return true;
2675 }
2676
2677 return false;
2678}
2679
2680std::optional<unsigned>
2681RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
2682#define RVV_OPC_LMUL_CASE(OPC, INV) \
2683 case RISCV::OPC##_M1: \
2684 return RISCV::INV##_M1; \
2685 case RISCV::OPC##_M2: \
2686 return RISCV::INV##_M2; \
2687 case RISCV::OPC##_M4: \
2688 return RISCV::INV##_M4; \
2689 case RISCV::OPC##_M8: \
2690 return RISCV::INV##_M8; \
2691 case RISCV::OPC##_MF2: \
2692 return RISCV::INV##_MF2; \
2693 case RISCV::OPC##_MF4: \
2694 return RISCV::INV##_MF4; \
2695 case RISCV::OPC##_MF8: \
2696 return RISCV::INV##_MF8
2697
2698#define RVV_OPC_LMUL_MASK_CASE(OPC, INV) \
2699 case RISCV::OPC##_M1_MASK: \
2700 return RISCV::INV##_M1_MASK; \
2701 case RISCV::OPC##_M2_MASK: \
2702 return RISCV::INV##_M2_MASK; \
2703 case RISCV::OPC##_M4_MASK: \
2704 return RISCV::INV##_M4_MASK; \
2705 case RISCV::OPC##_M8_MASK: \
2706 return RISCV::INV##_M8_MASK; \
2707 case RISCV::OPC##_MF2_MASK: \
2708 return RISCV::INV##_MF2_MASK; \
2709 case RISCV::OPC##_MF4_MASK: \
2710 return RISCV::INV##_MF4_MASK; \
2711 case RISCV::OPC##_MF8_MASK: \
2712 return RISCV::INV##_MF8_MASK
2713
2714 switch (Opcode) {
2715 default:
2716 return std::nullopt;
2717 case RISCV::FADD_H:
2718 return RISCV::FSUB_H;
2719 case RISCV::FADD_S:
2720 return RISCV::FSUB_S;
2721 case RISCV::FADD_D:
2722 return RISCV::FSUB_D;
2723 case RISCV::FSUB_H:
2724 return RISCV::FADD_H;
2725 case RISCV::FSUB_S:
2726 return RISCV::FADD_S;
2727 case RISCV::FSUB_D:
2728 return RISCV::FADD_D;
2729 case RISCV::ADD:
2730 return RISCV::SUB;
2731 case RISCV::SUB:
2732 return RISCV::ADD;
2733 case RISCV::ADDW:
2734 return RISCV::SUBW;
2735 case RISCV::SUBW:
2736 return RISCV::ADDW;
2737 // clang-format off
2738 RVV_OPC_LMUL_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2739 RVV_OPC_LMUL_MASK_CASE(PseudoVADD_VV, PseudoVSUB_VV);
2740 RVV_OPC_LMUL_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2741 RVV_OPC_LMUL_MASK_CASE(PseudoVSUB_VV, PseudoVADD_VV);
2742 // clang-format on
2743 }
2744
2745#undef RVV_OPC_LMUL_MASK_CASE
2746#undef RVV_OPC_LMUL_CASE
2747}
2748
2750 const MachineOperand &MO,
2751 bool DoRegPressureReduce) {
2752 if (!MO.isReg() || !MO.getReg().isVirtual())
2753 return false;
2754 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
2755 MachineInstr *MI = MRI.getVRegDef(MO.getReg());
2756 if (!MI || !isFMUL(MI->getOpcode()))
2757 return false;
2758
2761 return false;
2762
2763 // Try combining even if fmul has more than one use as it eliminates
2764 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
2765 // for fmul operands, so reject the transformation in register pressure
2766 // reduction mode.
2767 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2768 return false;
2769
2770 // Do not combine instructions from different basic blocks.
2771 if (Root.getParent() != MI->getParent())
2772 return false;
2773 return RISCV::hasEqualFRM(Root, *MI);
2774}
2775
2777 SmallVectorImpl<unsigned> &Patterns,
2778 bool DoRegPressureReduce) {
2779 unsigned Opc = Root.getOpcode();
2780 bool IsFAdd = isFADD(Opc);
2781 if (!IsFAdd && !isFSUB(Opc))
2782 return false;
2783 bool Added = false;
2784 if (canCombineFPFusedMultiply(Root, Root.getOperand(1),
2785 DoRegPressureReduce)) {
2788 Added = true;
2789 }
2790 if (canCombineFPFusedMultiply(Root, Root.getOperand(2),
2791 DoRegPressureReduce)) {
2794 Added = true;
2795 }
2796 return Added;
2797}
2798
2799static bool getFPPatterns(MachineInstr &Root,
2800 SmallVectorImpl<unsigned> &Patterns,
2801 bool DoRegPressureReduce) {
2802 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
2803}
2804
2805/// Utility routine that checks if \param MO is defined by an
2806/// \param CombineOpc instruction in the basic block \param MBB
2808 const MachineOperand &MO,
2809 unsigned CombineOpc) {
2810 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
2811 const MachineInstr *MI = nullptr;
2812
2813 if (MO.isReg() && MO.getReg().isVirtual())
2814 MI = MRI.getUniqueVRegDef(MO.getReg());
2815 // And it needs to be in the trace (otherwise, it won't have a depth).
2816 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
2817 return nullptr;
2818 // Must only used by the user we combine with.
2819 if (!MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
2820 return nullptr;
2821
2822 return MI;
2823}
2824
2825/// Utility routine that checks if \param MO is defined by a SLLI in \param
2826/// MBB that can be combined by splitting across 2 SHXADD instructions. The
2827/// first SHXADD shift amount is given by \param OuterShiftAmt.
2829 const MachineOperand &MO,
2830 unsigned OuterShiftAmt) {
2831 const MachineInstr *ShiftMI = canCombine(MBB, MO, RISCV::SLLI);
2832 if (!ShiftMI)
2833 return false;
2834
2835 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
2836 if (InnerShiftAmt < OuterShiftAmt || (InnerShiftAmt - OuterShiftAmt) > 3)
2837 return false;
2838
2839 return true;
2840}
2841
2842// Returns the shift amount from a SHXADD instruction. Returns 0 if the
2843// instruction is not a SHXADD.
2844static unsigned getSHXADDShiftAmount(unsigned Opc) {
2845 switch (Opc) {
2846 default:
2847 return 0;
2848 case RISCV::SH1ADD:
2849 return 1;
2850 case RISCV::SH2ADD:
2851 return 2;
2852 case RISCV::SH3ADD:
2853 return 3;
2854 }
2855}
2856
2857// Returns the shift amount from a SHXADD.UW instruction. Returns 0 if the
2858// instruction is not a SHXADD.UW.
2859static unsigned getSHXADDUWShiftAmount(unsigned Opc) {
2860 switch (Opc) {
2861 default:
2862 return 0;
2863 case RISCV::SH1ADD_UW:
2864 return 1;
2865 case RISCV::SH2ADD_UW:
2866 return 2;
2867 case RISCV::SH3ADD_UW:
2868 return 3;
2869 }
2870}
2871
2872// Look for opportunities to combine (sh3add Z, (add X, (slli Y, 5))) into
2873// (sh3add (sh2add Y, Z), X).
2874static bool getSHXADDPatterns(const MachineInstr &Root,
2875 SmallVectorImpl<unsigned> &Patterns) {
2876 unsigned ShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
2877 if (!ShiftAmt)
2878 return false;
2879
2880 const MachineBasicBlock &MBB = *Root.getParent();
2881
2882 const MachineInstr *AddMI = canCombine(MBB, Root.getOperand(2), RISCV::ADD);
2883 if (!AddMI)
2884 return false;
2885
2886 bool Found = false;
2887 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(1), ShiftAmt)) {
2889 Found = true;
2890 }
2891 if (canCombineShiftIntoShXAdd(MBB, AddMI->getOperand(2), ShiftAmt)) {
2893 Found = true;
2894 }
2895
2896 return Found;
2897}
2898
2910
2912 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
2913 bool DoRegPressureReduce) const {
2914
2915 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
2916 return true;
2917
2918 if (getSHXADDPatterns(Root, Patterns))
2919 return true;
2920
2921 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
2922 DoRegPressureReduce);
2923}
2924
2925static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern) {
2926 switch (RootOpc) {
2927 default:
2928 llvm_unreachable("Unexpected opcode");
2929 case RISCV::FADD_H:
2930 return RISCV::FMADD_H;
2931 case RISCV::FADD_S:
2932 return RISCV::FMADD_S;
2933 case RISCV::FADD_D:
2934 return RISCV::FMADD_D;
2935 case RISCV::FSUB_H:
2936 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
2937 : RISCV::FNMSUB_H;
2938 case RISCV::FSUB_S:
2939 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
2940 : RISCV::FNMSUB_S;
2941 case RISCV::FSUB_D:
2942 return Pattern == RISCVMachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
2943 : RISCV::FNMSUB_D;
2944 }
2945}
2946
2947static unsigned getAddendOperandIdx(unsigned Pattern) {
2948 switch (Pattern) {
2949 default:
2950 llvm_unreachable("Unexpected pattern");
2953 return 2;
2956 return 1;
2957 }
2958}
2959
2961 unsigned Pattern,
2964 MachineFunction *MF = Root.getMF();
2967
2968 MachineOperand &Mul1 = Prev.getOperand(1);
2969 MachineOperand &Mul2 = Prev.getOperand(2);
2970 MachineOperand &Dst = Root.getOperand(0);
2972
2973 Register DstReg = Dst.getReg();
2974 unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
2975 uint32_t IntersectedFlags = Root.getFlags() & Prev.getFlags();
2976 DebugLoc MergedLoc =
2978
2979 MachineInstrBuilder MIB = BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
2980 .addReg(Mul1.getReg())
2981 .addReg(Mul2.getReg())
2982 .addReg(Addend.getReg())
2983 .setMIFlags(IntersectedFlags);
2984
2985 InsInstrs.push_back(MIB);
2986 if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))
2987 DelInstrs.push_back(&Prev);
2988 DelInstrs.push_back(&Root);
2989}
2990
2991// Combine patterns like (sh3add Z, (add X, (slli Y, 5))) to
2992// (sh3add (sh2add Y, Z), X) if the shift amount can be split across two
2993// shXadd instructions. The outer shXadd keeps its original opcode.
2994static void
2995genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx,
2998 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
2999 MachineFunction *MF = Root.getMF();
3002
3003 unsigned OuterShiftAmt = getSHXADDShiftAmount(Root.getOpcode());
3004 assert(OuterShiftAmt != 0 && "Unexpected opcode");
3005
3006 MachineInstr *AddMI = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
3007 MachineInstr *ShiftMI =
3008 MRI.getUniqueVRegDef(AddMI->getOperand(AddOpIdx).getReg());
3009
3010 unsigned InnerShiftAmt = ShiftMI->getOperand(2).getImm();
3011 assert(InnerShiftAmt >= OuterShiftAmt && "Unexpected shift amount");
3012
3013 unsigned InnerOpc;
3014 switch (InnerShiftAmt - OuterShiftAmt) {
3015 default:
3016 llvm_unreachable("Unexpected shift amount");
3017 case 0:
3018 InnerOpc = RISCV::ADD;
3019 break;
3020 case 1:
3021 InnerOpc = RISCV::SH1ADD;
3022 break;
3023 case 2:
3024 InnerOpc = RISCV::SH2ADD;
3025 break;
3026 case 3:
3027 InnerOpc = RISCV::SH3ADD;
3028 break;
3029 }
3030
3031 const MachineOperand &X = AddMI->getOperand(3 - AddOpIdx);
3032 const MachineOperand &Y = ShiftMI->getOperand(1);
3033 const MachineOperand &Z = Root.getOperand(1);
3034
3035 Register NewVR = MRI.createVirtualRegister(&RISCV::GPRRegClass);
3036
3037 auto MIB1 = BuildMI(*MF, MIMetadata(Root), TII->get(InnerOpc), NewVR)
3038 .addReg(Y.getReg())
3039 .addReg(Z.getReg());
3040 auto MIB2 = BuildMI(*MF, MIMetadata(Root), TII->get(Root.getOpcode()),
3041 Root.getOperand(0).getReg())
3042 .addReg(NewVR)
3043 .addReg(X.getReg());
3044
3045 InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
3046 InsInstrs.push_back(MIB1);
3047 InsInstrs.push_back(MIB2);
3048 DelInstrs.push_back(ShiftMI);
3049 DelInstrs.push_back(AddMI);
3050 DelInstrs.push_back(&Root);
3051}
3052
3054 MachineInstr &Root, unsigned Pattern,
3057 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
3059 switch (Pattern) {
3060 default:
3062 DelInstrs, InstrIdxForVirtReg);
3063 return;
3066 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());
3067 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3068 return;
3069 }
3072 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());
3073 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
3074 return;
3075 }
3077 genShXAddAddShift(Root, 1, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3078 return;
3080 genShXAddAddShift(Root, 2, InsInstrs, DelInstrs, InstrIdxForVirtReg);
3081 return;
3082 }
3083}
3084
3086 StringRef &ErrInfo) const {
3087 MCInstrDesc const &Desc = MI.getDesc();
3088
3089 for (const auto &[Index, Operand] : enumerate(Desc.operands())) {
3090 const MachineOperand &MO = MI.getOperand(Index);
3091 unsigned OpType = Operand.OperandType;
3092 switch (OpType) {
3093 default:
3094 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
3096 if (!MO.isImm()) {
3097 ErrInfo = "Expected an immediate operand.";
3098 return false;
3099 }
3100 int64_t Imm = MO.getImm();
3101 bool Ok;
3102 switch (OpType) {
3103 default:
3104 llvm_unreachable("Unexpected operand type");
3105
3106#define CASE_OPERAND_UIMM(NUM) \
3107 case RISCVOp::OPERAND_UIMM##NUM: \
3108 Ok = isUInt<NUM>(Imm); \
3109 break;
3110#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX) \
3111 case RISCVOp::OPERAND_UIMM##BITS##_LSB##SUFFIX: { \
3112 constexpr size_t NumZeros = sizeof(#SUFFIX) - 1; \
3113 Ok = isShiftedUInt<BITS - NumZeros, NumZeros>(Imm); \
3114 break; \
3115 }
3116#define CASE_OPERAND_SIMM(NUM) \
3117 case RISCVOp::OPERAND_SIMM##NUM: \
3118 Ok = isInt<NUM>(Imm); \
3119 break;
3120 // clang-format off
3145 // clang-format on
3147 Ok = Imm >= 1 && Imm <= 16;
3148 break;
3150 Ok = isUInt<5>(Imm) && (Imm != 0);
3151 break;
3153 Ok = isUInt<5>(Imm) && (Imm > 3);
3154 break;
3156 Ok = Imm >= 1 && Imm <= 32;
3157 break;
3159 Ok = Imm >= 1 && Imm <= 64;
3160 break;
3162 Ok = isUInt<8>(Imm) && Imm >= 32;
3163 break;
3166 break;
3168 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
3169 break;
3171 Ok = isShiftedUInt<8, 2>(Imm) && (Imm != 0);
3172 break;
3174 Ok = isUInt<16>(Imm) && (Imm != 0);
3175 break;
3177 Ok = Imm == 3;
3178 break;
3180 Ok = Imm == 4;
3181 break;
3183 Ok = (isUInt<5>(Imm) && Imm != 0) || Imm == -1;
3184 break;
3185 // clang-format off
3193 // clang-format on
3195 Ok = Imm >= -15 && Imm <= 16;
3196 break;
3198 Ok = isInt<5>(Imm) && (Imm != 0);
3199 break;
3201 Ok = Imm != 0 && isInt<6>(Imm);
3202 break;
3205 break;
3208 break;
3210 Ok = isShiftedInt<7, 5>(Imm);
3211 break;
3213 Ok = isInt<16>(Imm) && (Imm != 0);
3214 break;
3216 Ok = isInt<20>(Imm);
3217 break;
3219 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
3220 break;
3222 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
3223 Ok = Ok && Imm != 0;
3224 break;
3226 Ok = (isUInt<5>(Imm) && Imm != 0) || (Imm >= 0xfffe0 && Imm <= 0xfffff);
3227 break;
3229 Ok = Imm >= 0 && Imm <= 10;
3230 break;
3232 Ok = Imm >= 0 && Imm <= 7;
3233 break;
3235 Ok = Imm >= 1 && Imm <= 10;
3236 break;
3238 Ok = Imm >= 2 && Imm <= 14;
3239 break;
3241 Ok = Imm >= RISCVZC::RA && Imm <= RISCVZC::RA_S0_S11;
3242 break;
3245 break;
3247 Ok = Imm >= 0 && Imm <= 48 && Imm % 16 == 0;
3248 break;
3251 break;
3253 Ok = Imm == RISCVFPRndMode::RTZ;
3254 break;
3257 break;
3259 Ok = Imm == XSMTVTypeMode::SMT_I8;
3260 break;
3262 Ok = Imm >= 0 && Imm < RISCVCC::COND_INVALID;
3263 break;
3266 break;
3269 Imm;
3270 break;
3272 Ok = (isUInt<5>(Imm) && RISCVVType::isValidSEW(1 << Imm));
3273 break;
3275 Ok = Imm == 0;
3276 break;
3279 if (RISCVII::usesVXRM(Desc.TSFlags))
3280 Ok = isUInt<2>(Imm);
3281 else
3283 break;
3286 break;
3288 Ok = Imm == 1 || Imm == 2 || Imm == 4;
3289 break;
3290 }
3291 if (!Ok) {
3292 ErrInfo = "Invalid immediate";
3293 return false;
3294 }
3295 }
3296 break;
3298 // TODO: We could be stricter about what non-register operands are
3299 // allowed.
3300 if (MO.isReg()) {
3301 ErrInfo = "Expected a non-register operand.";
3302 return false;
3303 }
3304 if (MO.isImm() && !isInt<12>(MO.getImm())) {
3305 ErrInfo = "Invalid immediate";
3306 return false;
3307 }
3308 break;
3311 // TODO: We could be stricter about what non-register operands are
3312 // allowed.
3313 if (MO.isReg()) {
3314 ErrInfo = "Expected a non-register operand.";
3315 return false;
3316 }
3317 if (MO.isImm() && !isUInt<20>(MO.getImm())) {
3318 ErrInfo = "Invalid immediate";
3319 return false;
3320 }
3321 break;
3323 // TODO: We could be stricter about what non-register operands are
3324 // allowed.
3325 if (MO.isReg()) {
3326 ErrInfo = "Expected a non-register operand.";
3327 return false;
3328 }
3329 if (MO.isImm() && !isInt<32>(MO.getImm())) {
3330 ErrInfo = "Invalid immediate";
3331 return false;
3332 }
3333 break;
3335 if (MO.isImm()) {
3336 int64_t Imm = MO.getImm();
3337 // VLMAX is represented as -1.
3338 if (!isUInt<5>(Imm) && Imm != -1) {
3339 ErrInfo = "Invalid immediate";
3340 return false;
3341 }
3342 } else if (!MO.isReg()) {
3343 ErrInfo = "Expected a register or immediate operand.";
3344 return false;
3345 }
3346 break;
3348 if (!MO.isReg() && !MO.isImm()) {
3349 ErrInfo = "Expected a register or immediate operand.";
3350 return false;
3351 }
3352 break;
3353 }
3354 }
3355
3356 const uint64_t TSFlags = Desc.TSFlags;
3357 if (RISCVII::hasVLOp(TSFlags)) {
3358 const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));
3359 if (!Op.isImm() && !Op.isReg()) {
3360 ErrInfo = "Invalid operand type for VL operand";
3361 return false;
3362 }
3363 if (Op.isReg() && Op.getReg().isValid()) {
3364 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3365 auto *RC = MRI.getRegClass(Op.getReg());
3366 if (!RISCV::GPRNoX0RegClass.hasSubClassEq(RC)) {
3367 ErrInfo = "Invalid register class for VL operand";
3368 return false;
3369 }
3370 }
3371 if (!RISCVII::hasSEWOp(TSFlags)) {
3372 ErrInfo = "VL operand w/o SEW operand?";
3373 return false;
3374 }
3375 }
3376 if (RISCVII::hasSEWOp(TSFlags)) {
3377 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
3378 if (!MI.getOperand(OpIdx).isImm()) {
3379 ErrInfo = "SEW value expected to be an immediate";
3380 return false;
3381 }
3382 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
3383 if (Log2SEW > 31) {
3384 ErrInfo = "Unexpected SEW value";
3385 return false;
3386 }
3387 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
3388 if (!RISCVVType::isValidSEW(SEW)) {
3389 ErrInfo = "Unexpected SEW value";
3390 return false;
3391 }
3392 }
3393 if (RISCVII::hasVecPolicyOp(TSFlags)) {
3394 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
3395 if (!MI.getOperand(OpIdx).isImm()) {
3396 ErrInfo = "Policy operand expected to be an immediate";
3397 return false;
3398 }
3399 uint64_t Policy = MI.getOperand(OpIdx).getImm();
3401 ErrInfo = "Invalid Policy Value";
3402 return false;
3403 }
3404 if (!RISCVII::hasVLOp(TSFlags)) {
3405 ErrInfo = "policy operand w/o VL operand?";
3406 return false;
3407 }
3408
3409 // VecPolicy operands can only exist on instructions with passthru/merge
3410 // arguments. Note that not all arguments with passthru have vec policy
3411 // operands- some instructions have implicit policies.
3412 unsigned UseOpIdx;
3413 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
3414 ErrInfo = "policy operand w/o tied operand?";
3415 return false;
3416 }
3417 }
3418
3419 if (int Idx = RISCVII::getFRMOpNum(Desc);
3420 Idx >= 0 && MI.getOperand(Idx).getImm() == RISCVFPRndMode::DYN &&
3421 !MI.readsRegister(RISCV::FRM, /*TRI=*/nullptr)) {
3422 ErrInfo = "dynamic rounding mode should read FRM";
3423 return false;
3424 }
3425
3426 return true;
3427}
3428
3430 const MachineInstr &AddrI,
3431 ExtAddrMode &AM) const {
3432 switch (MemI.getOpcode()) {
3433 default:
3434 return false;
3435 case RISCV::LB:
3436 case RISCV::LBU:
3437 case RISCV::LH:
3438 case RISCV::LH_INX:
3439 case RISCV::LHU:
3440 case RISCV::LW:
3441 case RISCV::LW_INX:
3442 case RISCV::LWU:
3443 case RISCV::LD:
3444 case RISCV::LD_RV32:
3445 case RISCV::FLH:
3446 case RISCV::FLW:
3447 case RISCV::FLD:
3448 case RISCV::SB:
3449 case RISCV::SH:
3450 case RISCV::SH_INX:
3451 case RISCV::SW:
3452 case RISCV::SW_INX:
3453 case RISCV::SD:
3454 case RISCV::SD_RV32:
3455 case RISCV::FSH:
3456 case RISCV::FSW:
3457 case RISCV::FSD:
3458 break;
3459 }
3460
3461 if (MemI.getOperand(0).getReg() == Reg)
3462 return false;
3463
3464 if (AddrI.getOpcode() != RISCV::ADDI || !AddrI.getOperand(1).isReg() ||
3465 !AddrI.getOperand(2).isImm())
3466 return false;
3467
3468 int64_t OldOffset = MemI.getOperand(2).getImm();
3469 int64_t Disp = AddrI.getOperand(2).getImm();
3470 int64_t NewOffset = OldOffset + Disp;
3471 if (!STI.is64Bit())
3472 NewOffset = SignExtend64<32>(NewOffset);
3473
3474 if (!isInt<12>(NewOffset))
3475 return false;
3476
3477 AM.BaseReg = AddrI.getOperand(1).getReg();
3478 AM.ScaledReg = 0;
3479 AM.Scale = 0;
3480 AM.Displacement = NewOffset;
3482 return true;
3483}
3484
3486 const ExtAddrMode &AM) const {
3487
3488 const DebugLoc &DL = MemI.getDebugLoc();
3489 MachineBasicBlock &MBB = *MemI.getParent();
3490
3491 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
3492 "Addressing mode not supported for folding");
3493
3494 return BuildMI(MBB, MemI, DL, get(MemI.getOpcode()))
3495 .addReg(MemI.getOperand(0).getReg(), getDefRegState(MemI.mayLoad()))
3496 .addReg(AM.BaseReg)
3497 .addImm(AM.Displacement)
3498 .setMemRefs(MemI.memoperands())
3499 .setMIFlags(MemI.getFlags());
3500}
3501
3502// TODO: At the moment, MIPS introduced paring of instructions operating with
3503// word or double word. This should be extended with more instructions when more
3504// vendors support load/store pairing.
3506 switch (Opc) {
3507 default:
3508 return false;
3509 case RISCV::SW:
3510 case RISCV::SD:
3511 case RISCV::LD:
3512 case RISCV::LW:
3513 return true;
3514 }
3515}
3516
3518 const TargetRegisterInfo *TRI) {
3519 // If this is a volatile load/store, don't mess with it.
3520 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
3521 return false;
3522
3523 if (LdSt.getOperand(1).isFI())
3524 return true;
3525
3526 assert(LdSt.getOperand(1).isReg() && "Expected a reg operand.");
3527 // Can't cluster if the instruction modifies the base register
3528 // or it is update form. e.g. ld x5,8(x5)
3529 if (LdSt.modifiesRegister(LdSt.getOperand(1).getReg(), TRI))
3530 return false;
3531
3532 if (!LdSt.getOperand(2).isImm())
3533 return false;
3534
3535 return true;
3536}
3537
3540 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3541 if (!LdSt.mayLoadOrStore())
3542 return false;
3543
3544 // Conservatively, only handle scalar loads/stores for now.
3545 switch (LdSt.getOpcode()) {
3546 case RISCV::LB:
3547 case RISCV::LBU:
3548 case RISCV::SB:
3549 case RISCV::LH:
3550 case RISCV::LH_INX:
3551 case RISCV::LHU:
3552 case RISCV::FLH:
3553 case RISCV::SH:
3554 case RISCV::SH_INX:
3555 case RISCV::FSH:
3556 case RISCV::LW:
3557 case RISCV::LW_INX:
3558 case RISCV::LWU:
3559 case RISCV::FLW:
3560 case RISCV::SW:
3561 case RISCV::SW_INX:
3562 case RISCV::FSW:
3563 case RISCV::LD:
3564 case RISCV::LD_RV32:
3565 case RISCV::FLD:
3566 case RISCV::SD:
3567 case RISCV::SD_RV32:
3568 case RISCV::FSD:
3569 break;
3570 default:
3571 return false;
3572 }
3573 const MachineOperand *BaseOp;
3574 OffsetIsScalable = false;
3575 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width))
3576 return false;
3577 BaseOps.push_back(BaseOp);
3578 return true;
3579}
3580
3581// TODO: This was copied from SIInstrInfo. Could it be lifted to a common
3582// helper?
3585 const MachineInstr &MI2,
3587 // Only examine the first "base" operand of each instruction, on the
3588 // assumption that it represents the real base address of the memory access.
3589 // Other operands are typically offsets or indices from this base address.
3590 if (BaseOps1.front()->isIdenticalTo(*BaseOps2.front()))
3591 return true;
3592
3593 if (!MI1.hasOneMemOperand() || !MI2.hasOneMemOperand())
3594 return false;
3595
3596 auto MO1 = *MI1.memoperands_begin();
3597 auto MO2 = *MI2.memoperands_begin();
3598 if (MO1->getAddrSpace() != MO2->getAddrSpace())
3599 return false;
3600
3601 auto Base1 = MO1->getValue();
3602 auto Base2 = MO2->getValue();
3603 if (!Base1 || !Base2)
3604 return false;
3605 Base1 = getUnderlyingObject(Base1);
3606 Base2 = getUnderlyingObject(Base2);
3607
3608 if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2))
3609 return false;
3610
3611 return Base1 == Base2;
3612}
3613
3615 ArrayRef<const MachineOperand *> BaseOps1, int64_t Offset1,
3616 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
3617 int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize,
3618 unsigned NumBytes) const {
3619 // If the mem ops (to be clustered) do not have the same base ptr, then they
3620 // should not be clustered
3621 if (!BaseOps1.empty() && !BaseOps2.empty()) {
3622 const MachineInstr &FirstLdSt = *BaseOps1.front()->getParent();
3623 const MachineInstr &SecondLdSt = *BaseOps2.front()->getParent();
3624 if (!memOpsHaveSameBasePtr(FirstLdSt, BaseOps1, SecondLdSt, BaseOps2))
3625 return false;
3626 } else if (!BaseOps1.empty() || !BaseOps2.empty()) {
3627 // If only one base op is empty, they do not have the same base ptr
3628 return false;
3629 }
3630
3631 unsigned CacheLineSize =
3632 BaseOps1.front()->getParent()->getMF()->getSubtarget().getCacheLineSize();
3633 // Assume a cache line size of 64 bytes if no size is set in RISCVSubtarget.
3635 // Cluster if the memory operations are on the same or a neighbouring cache
3636 // line, but limit the maximum ClusterSize to avoid creating too much
3637 // additional register pressure.
3638 return ClusterSize <= 4 && std::abs(Offset1 - Offset2) < CacheLineSize;
3639}
3640
3641// Set BaseReg (the base register operand), Offset (the byte offset being
3642// accessed) and the access Width of the passed instruction that reads/writes
3643// memory. Returns false if the instruction does not read/write memory or the
3644// BaseReg/Offset/Width can't be determined. Is not guaranteed to always
3645// recognise base operands and offsets in all cases.
3646// TODO: Add an IsScalable bool ref argument (like the equivalent AArch64
3647// function) and set it as appropriate.
3649 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
3650 LocationSize &Width) const {
3651 if (!LdSt.mayLoadOrStore())
3652 return false;
3653
3654 // Here we assume the standard RISC-V ISA, which uses a base+offset
3655 // addressing mode. You'll need to relax these conditions to support custom
3656 // load/store instructions.
3657 if (LdSt.getNumExplicitOperands() != 3)
3658 return false;
3659 if ((!LdSt.getOperand(1).isReg() && !LdSt.getOperand(1).isFI()) ||
3660 !LdSt.getOperand(2).isImm())
3661 return false;
3662
3663 if (!LdSt.hasOneMemOperand())
3664 return false;
3665
3666 Width = (*LdSt.memoperands_begin())->getSize();
3667 BaseReg = &LdSt.getOperand(1);
3668 Offset = LdSt.getOperand(2).getImm();
3669 return true;
3670}
3671
3673 const MachineInstr &MIa, const MachineInstr &MIb) const {
3674 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
3675 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
3676
3679 return false;
3680
3681 // Retrieve the base register, offset from the base register and width. Width
3682 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
3683 // base registers are identical, and the offset of a lower memory access +
3684 // the width doesn't overlap the offset of a higher memory access,
3685 // then the memory accesses are different.
3686 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
3687 int64_t OffsetA = 0, OffsetB = 0;
3689 WidthB = LocationSize::precise(0);
3690 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA) &&
3691 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB)) {
3692 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
3693 int LowOffset = std::min(OffsetA, OffsetB);
3694 int HighOffset = std::max(OffsetA, OffsetB);
3695 LocationSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
3696 if (LowWidth.hasValue() &&
3697 LowOffset + (int)LowWidth.getValue() <= HighOffset)
3698 return true;
3699 }
3700 }
3701 return false;
3702}
3703
3704std::pair<unsigned, unsigned>
3706 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
3707 return std::make_pair(TF & Mask, TF & ~Mask);
3708}
3709
3712 using namespace RISCVII;
3713 static const std::pair<unsigned, const char *> TargetFlags[] = {
3714 {MO_CALL, "riscv-call"},
3715 {MO_LO, "riscv-lo"},
3716 {MO_HI, "riscv-hi"},
3717 {MO_PCREL_LO, "riscv-pcrel-lo"},
3718 {MO_PCREL_HI, "riscv-pcrel-hi"},
3719 {MO_GOT_HI, "riscv-got-hi"},
3720 {MO_TPREL_LO, "riscv-tprel-lo"},
3721 {MO_TPREL_HI, "riscv-tprel-hi"},
3722 {MO_TPREL_ADD, "riscv-tprel-add"},
3723 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
3724 {MO_TLS_GD_HI, "riscv-tls-gd-hi"},
3725 {MO_TLSDESC_HI, "riscv-tlsdesc-hi"},
3726 {MO_TLSDESC_LOAD_LO, "riscv-tlsdesc-load-lo"},
3727 {MO_TLSDESC_ADD_LO, "riscv-tlsdesc-add-lo"},
3728 {MO_TLSDESC_CALL, "riscv-tlsdesc-call"},
3729 {MO_QC_ACCESS, "riscv-qc-access"},
3730 };
3731 return ArrayRef(TargetFlags);
3732}
3733
3735 unsigned &Flags) const {
3736 // More accurate safety checking is done in getOutliningCandidateInfo.
3738}
3739
3740// Enum values indicating how an outlined call should be constructed.
3746
3751
3753 const MachineFunction *MF = MBB.getParent();
3754 const Function &F = MF->getFunction();
3755 return F.getFnAttribute("fentry-call").getValueAsBool() ||
3756 F.hasFnAttribute("patchable-function-entry");
3757}
3758
3760 MCRegister RegNo) {
3761 return MI.readsRegister(RegNo, TRI) ||
3762 MI.getDesc().hasImplicitUseOfPhysReg(RegNo);
3763}
3764
3766 const TargetRegisterInfo *TRI, MCRegister RegNo) {
3767 return MI.modifiesRegister(RegNo, TRI) ||
3768 MI.getDesc().hasImplicitDefOfPhysReg(RegNo);
3769}
3770
3772 if (!MBB.back().isReturn())
3773 return true;
3775 return true;
3776
3777 // If the candidate reads the pre-set register
3778 // that can be used for expanding PseudoTAIL instruction,
3779 // then we cannot insert tail call.
3780 const TargetSubtargetInfo &STI = MBB.getParent()->getSubtarget();
3781 const RISCVMachineFunctionInfo *RVFI =
3782 MBB.getParent()->getInfo<RISCVMachineFunctionInfo>();
3783 // When cf-protection-branch is active, the outliner will emit PseudoTAILX7
3784 // which always uses X7. Otherwise, PseudoTAIL is emitted and the register
3785 // is determined by Zicfilp at encode time.
3786 MCRegister TailExpandUseRegNo =
3787 RVFI->hasCFProtectionBranch()
3788 ? RISCV::X7
3790 for (const MachineInstr &MI : MBB) {
3791 if (isMIReadsReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))
3792 return true;
3793 if (isMIModifiesReg(MI, STI.getRegisterInfo(), TailExpandUseRegNo))
3794 break;
3795 }
3796 return false;
3797}
3798
3800 const TargetRegisterInfo &TRI) {
3801 // Candidate registers for saving X5: t1-t6
3802 static const MCPhysReg TempRegs[] = {
3803 RISCV::X6, // t1
3804 RISCV::X7, // t2
3805 RISCV::X28, // t3
3806 RISCV::X29, // t4
3807 RISCV::X30, // t5
3808 RISCV::X31 // t6
3809 };
3810
3811 const MachineFunction *MF = C.getMF();
3812 const MachineRegisterInfo &MRI = MF->getRegInfo();
3813
3814 for (MCPhysReg Reg : TempRegs) {
3815 if (MRI.isReserved(Reg))
3816 continue;
3817
3818 if (C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
3819 C.isAvailableInsideSeq(Reg, TRI)) {
3820 return Reg;
3821 }
3822 }
3823
3824 return Register();
3825}
3826
3828 // If the expansion register for tail calls is live across the candidate
3829 // outlined call site, we cannot outline that candidate as the expansion
3830 // would clobber the register.
3831 const RISCVMachineFunctionInfo *RVFI =
3832 C.getMF()->getInfo<RISCVMachineFunctionInfo>();
3833 MCRegister TailExpandUseReg =
3834 RVFI->hasCFProtectionBranch()
3835 ? RISCV::X7
3836 : RISCVII::getTailExpandUseRegNo(STI.getFeatureBits());
3837 if (C.back().isReturn() &&
3838 !C.isAvailableAcrossAndOutOfSeq(TailExpandUseReg, RegInfo)) {
3839 LLVM_DEBUG(dbgs() << "MBB:\n" << *C.getMBB());
3840 LLVM_DEBUG(dbgs() << "Cannot be outlined between: " << C.front() << "and "
3841 << C.back());
3842 LLVM_DEBUG(dbgs() << "Because the tail-call register is live across "
3843 "the proposed outlined function call\n");
3844 return true;
3845 }
3846
3847 // If last instruction is return then we can rely on
3848 // the verification already performed in the getOutliningTypeImpl.
3849 if (C.back().isReturn()) {
3850 assert(!cannotInsertTailCall(*C.getMBB()) &&
3851 "The candidate who uses return instruction must be outlined "
3852 "using tail call");
3853 return false;
3854 }
3855
3856 // Filter out candidates where the X5 register (t0) can't be used to setup
3857 // the function call.
3858 if (!C.isAvailableInsideSeq(RISCV::X5, RegInfo))
3859 return true;
3860
3861 // If X5 is available in the region, use X5 directly (MachineOutlinerDefault).
3862 if (C.isAvailableAcrossAndOutOfSeq(RISCV::X5, RegInfo))
3863 return false;
3864
3865 // Otherwise, try to save X5 into t1-t6 (MachineOutlinerRegSave).
3866 if (STI.getCLOpts().outliner_regsave && !C.getMF()->isOutlined() &&
3867 findRegisterToSaveX5To(C, RegInfo))
3868 return false;
3869
3870 return true;
3871}
3872
3873std::optional<std::unique_ptr<outliner::OutlinedFunction>>
3875 const MachineModuleInfo &MMI,
3876 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
3877 unsigned MinRepeats) const {
3878
3879 // Analyze each candidate and erase the ones that are not viable.
3880 llvm::erase_if(RepeatedSequenceLocs, [this](auto Candidate) {
3881 return analyzeCandidate(Candidate);
3882 });
3883
3884 // If the sequence doesn't have enough candidates left, then we're done.
3885 if (RepeatedSequenceLocs.size() < MinRepeats)
3886 return std::nullopt;
3887
3888 // Each RepeatedSequenceLoc is identical.
3889 outliner::Candidate &Candidate = RepeatedSequenceLocs[0];
3890 unsigned InstrSizeCExt =
3891 Candidate.getMF()->getSubtarget<RISCVSubtarget>().hasStdExtZca() ? 2 : 4;
3892 unsigned CallOverhead = 0, FrameOverhead = 0;
3893
3894 // Count the number of CFI instructions in the candidate, if present.
3895 unsigned CFICount = 0;
3896 for (auto &I : Candidate) {
3897 if (I.isCFIInstruction())
3898 CFICount++;
3899 }
3900
3901 // Ensure CFI coverage matches: comparing the number of CFIs in the candidate
3902 // with the total number of CFIs in the parent function for each candidate.
3903 // Outlining only a subset of a function’s CFIs would split the unwind state
3904 // across two code regions and lead to incorrect address offsets between the
3905 // outlined body and the remaining code. To preserve correct unwind info, we
3906 // only outline when all CFIs in the function can be outlined together.
3907 for (outliner::Candidate &C : RepeatedSequenceLocs) {
3908 std::vector<MCCFIInstruction> CFIInstructions =
3909 C.getMF()->getFrameInstructions();
3910
3911 if (CFICount > 0 && CFICount != CFIInstructions.size())
3912 return std::nullopt;
3913 }
3914
3916 if (Candidate.back().isReturn()) {
3918 // tail call = auipc + jalr in the worst case without linker relaxation.
3919 // FIXME: This code suggests the JALR can be compressed - how?
3920 CallOverhead = 4 + InstrSizeCExt;
3921 // Using tail call we move ret instruction from caller to callee.
3922 FrameOverhead = 0;
3923 } else {
3924 // call t0, function = 8 bytes.
3925 CallOverhead = 8;
3926 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
3927 FrameOverhead = InstrSizeCExt;
3928 }
3929
3930 // If we have CFI instructions, we can only outline if the outlined section
3931 // can be a tail call.
3932 if (MOCI != MachineOutlinerTailCall && CFICount > 0)
3933 return std::nullopt;
3934
3935 if (STI.getCLOpts().outliner_regsave && MOCI == MachineOutlinerDefault) {
3936 // Set per-candidate overhead based on X5 availability
3937 for (auto &C : RepeatedSequenceLocs) {
3938
3939 if (C.isAvailableAcrossAndOutOfSeq(RISCV::X5, RegInfo)) {
3940 // X5 is available, just need the call
3941 unsigned CandCallOverhead = 8;
3942 C.setCallInfo(MachineOutlinerDefault, CandCallOverhead);
3943 } else {
3944 // X5 unavailable, need save + call + restore
3945 // Save (2-4) + Call (8) + Restore (2-4)
3946 unsigned CandCallOverhead = InstrSizeCExt + 8 + InstrSizeCExt;
3947 C.setCallInfo(MachineOutlinerRegSave, CandCallOverhead);
3948 }
3949 }
3950 } else {
3951 for (auto &C : RepeatedSequenceLocs)
3952 C.setCallInfo(MOCI, CallOverhead);
3953 }
3954
3955 unsigned SequenceSize = 0;
3956 for (auto &MI : Candidate)
3957 SequenceSize += getInstSizeInBytes(MI);
3958
3959 return std::make_unique<outliner::OutlinedFunction>(
3960 RepeatedSequenceLocs, SequenceSize, FrameOverhead, MOCI);
3961}
3962
3966 unsigned Flags) const {
3967 MachineInstr &MI = *MBBI;
3968 MachineBasicBlock *MBB = MI.getParent();
3969 const TargetRegisterInfo *TRI =
3970 MBB->getParent()->getSubtarget().getRegisterInfo();
3971 const auto &F = MI.getMF()->getFunction();
3972
3973 // We can only outline CFI instructions if we will tail call the outlined
3974 // function, or fix up the CFI offsets. Currently, CFI instructions are
3975 // outlined only if in a tail call.
3976 if (MI.isCFIInstruction())
3978
3979 if (cannotInsertTailCall(*MBB) &&
3980 (MI.isReturn() || isMIModifiesReg(MI, TRI, RISCV::X5)))
3982
3983 // Make sure the operands don't reference something unsafe.
3984 for (const auto &MO : MI.operands()) {
3985
3986 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
3987 // if any possible.
3988 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
3989 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
3990 F.hasSection() || F.getSectionPrefix()))
3992 }
3993
3994 if (isLPAD(MI))
3996
3998}
3999
4002 const outliner::OutlinedFunction &OF) const {
4003
4004 if (OF.FrameConstructionID == MachineOutlinerTailCall)
4005 return;
4006
4007 MBB.addLiveIn(RISCV::X5);
4008
4009 // Add in a return instruction to the end of the outlined frame.
4010 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR), RISCV::X0)
4011 .addReg(RISCV::X5)
4012 .addImm(0));
4013}
4014
4018
4019 if (C.CallConstructionID == MachineOutlinerTailCall) {
4020 const RISCVMachineFunctionInfo *RVFI =
4022 unsigned TailOpc =
4023 RVFI->hasCFProtectionBranch() ? RISCV::PseudoTAILX7 : RISCV::PseudoTAIL;
4024 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(TailOpc))
4025 .addGlobalAddress(M.getNamedValue(MF.getName()),
4026 /*Offset=*/0, RISCVII::MO_CALL));
4027 return It;
4028 }
4029
4030 if (C.CallConstructionID == MachineOutlinerRegSave) {
4031 Register SaveReg = findRegisterToSaveX5To(C, RegInfo);
4032 assert(SaveReg && "Cannot find an available register to save/restore X5.");
4033
4034 // Save: ADDI SaveReg, X5, 0 (equivalent to MV SaveReg, X5)
4035 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(RISCV::ADDI), SaveReg)
4036 .addReg(RISCV::X5)
4037 .addImm(0));
4038 It++;
4039
4040 // Call: PseudoCALLReg X5
4041 It = MBB.insert(
4042 It, BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
4043 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
4045 MachineBasicBlock::iterator CallPt = It;
4046 It++;
4047
4048 // Restore: ADDI X5, SaveReg, 0 (equivalent to MV X5, SaveReg)
4049 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(RISCV::ADDI), RISCV::X5)
4050 .addReg(SaveReg)
4051 .addImm(0));
4052
4053 return CallPt;
4054 }
4055
4056 // Add in a call instruction to the outlined function at the given location.
4057 It = MBB.insert(It,
4058 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
4059 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
4061 return It;
4062}
4063
4066 DebugLoc &DL,
4067 bool AllowSideEffects) const {
4068
4069 const MachineFunction &MF = *MBB.getParent();
4070 const RISCVRegisterInfo &TRI = *STI.getRegisterInfo();
4071
4072 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
4073 BuildMI(MBB, Iter, DL, get(RISCV::PseudoClearGPR), Reg);
4074 } else if (RISCV::FPR32RegClass.contains(Reg)) {
4075 BuildMI(MBB, Iter, DL, get(RISCV::PseudoClearFPR32), Reg);
4076 } else if (RISCV::FPR64RegClass.contains(Reg)) {
4077 BuildMI(MBB, Iter, DL, get(RISCV::PseudoClearFPR64), Reg);
4078 } else if (RISCV::FPR128RegClass.contains(Reg)) {
4079 BuildMI(MBB, Iter, DL, get(RISCV::PseudoClearFPR128), Reg);
4080 } else if (RISCV::VRRegClass.contains(Reg)) {
4081 BuildMI(MBB, Iter, DL, get(RISCV::PseudoClearVR), Reg);
4082 } else {
4084 "buildClearRegister is not implemented for " + TRI.getRegAsmName(Reg));
4085 }
4086}
4087
4088std::optional<RegImmPair> RISCVInstrInfo::isAddImmediate(const MachineInstr &MI,
4089 Register Reg) const {
4090 // TODO: Handle cases where Reg is a super- or sub-register of the
4091 // destination register.
4092 const MachineOperand &Op0 = MI.getOperand(0);
4093 if (!Op0.isReg() || Reg != Op0.getReg())
4094 return std::nullopt;
4095
4096 // Don't consider ADDIW as a candidate because the caller may not be aware
4097 // of its sign extension behaviour.
4098 if (MI.getOpcode() == RISCV::ADDI && MI.getOperand(1).isReg() &&
4099 MI.getOperand(2).isImm())
4100 return RegImmPair{MI.getOperand(1).getReg(), MI.getOperand(2).getImm()};
4101
4102 return std::nullopt;
4103}
4104
4105// MIR printer helper function to annotate Operands with a comment.
4107 const MachineOperand &Op,
4108 unsigned OpIdx) const {
4109 // Print a generic comment for this operand if there is one.
4110 std::string GenericComment =
4112 if (!GenericComment.empty())
4113 return GenericComment;
4114
4115 const MCInstrDesc &Desc = MI.getDesc();
4116 if (OpIdx >= Desc.getNumOperands())
4117 return std::string();
4118
4119 std::string Comment;
4120 raw_string_ostream OS(Comment);
4121
4122 const MCOperandInfo &OpInfo = Desc.operands()[OpIdx];
4123
4124 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
4125 // operand of vector codegen pseudos.
4126 switch (OpInfo.OperandType) {
4129 unsigned Imm = Op.getImm();
4131 break;
4132 }
4134 unsigned Imm = Op.getImm();
4136 break;
4137 }
4139 unsigned Imm = Op.getImm();
4140 OS << "w" << Imm;
4141 break;
4142 }
4145 unsigned Log2SEW = Op.getImm();
4146 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
4147 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
4148 OS << "e" << SEW;
4149 break;
4150 }
4152 unsigned Policy = Op.getImm();
4154 "Invalid Policy Value");
4155 OS << (Policy & RISCVVType::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
4156 << (Policy & RISCVVType::MASK_AGNOSTIC ? "ma" : "mu");
4157 break;
4158 }
4160 if (Op.isImm() && Op.getImm() == -1)
4161 OS << "vl=VLMAX";
4162 else
4163 OS << "vl";
4164 break;
4166 if (RISCVII::usesVXRM(Desc.TSFlags)) {
4168 auto VXRM = static_cast<RISCVVXRndMode::RoundingMode>(Op.getImm());
4169 OS << "vxrm=" << RISCVVXRndMode::roundingModeToString(VXRM);
4170 } else {
4172 auto FRM = static_cast<RISCVFPRndMode::RoundingMode>(Op.getImm());
4173 OS << "frm=" << RISCVFPRndMode::roundingModeToString(FRM);
4174 }
4175 break;
4176 }
4177
4178 return Comment;
4179}
4180
4181// clang-format off
4182#define CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL) \
4183 RISCV::Pseudo##OP##_##LMUL
4184
4185#define CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL) \
4186 RISCV::Pseudo##OP##_##LMUL##_MASK
4187
4188#define CASE_RVV_OPCODE_LMUL(OP, LMUL) \
4189 CASE_RVV_OPCODE_UNMASK_LMUL(OP, LMUL): \
4190 case CASE_RVV_OPCODE_MASK_LMUL(OP, LMUL)
4191
4192#define CASE_RVV_OPCODE_UNMASK_WIDEN(OP) \
4193 CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF8): \
4194 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF4): \
4195 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, MF2): \
4196 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M1): \
4197 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M2): \
4198 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M4)
4199
4200#define CASE_RVV_OPCODE_UNMASK(OP) \
4201 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4202 case CASE_RVV_OPCODE_UNMASK_LMUL(OP, M8)
4203
4204#define CASE_RVV_OPCODE_MASK_WIDEN(OP) \
4205 CASE_RVV_OPCODE_MASK_LMUL(OP, MF8): \
4206 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF4): \
4207 case CASE_RVV_OPCODE_MASK_LMUL(OP, MF2): \
4208 case CASE_RVV_OPCODE_MASK_LMUL(OP, M1): \
4209 case CASE_RVV_OPCODE_MASK_LMUL(OP, M2): \
4210 case CASE_RVV_OPCODE_MASK_LMUL(OP, M4)
4211
4212#define CASE_RVV_OPCODE_MASK(OP) \
4213 CASE_RVV_OPCODE_MASK_WIDEN(OP): \
4214 case CASE_RVV_OPCODE_MASK_LMUL(OP, M8)
4215
4216#define CASE_RVV_OPCODE_WIDEN(OP) \
4217 CASE_RVV_OPCODE_UNMASK_WIDEN(OP): \
4218 case CASE_RVV_OPCODE_MASK_WIDEN(OP)
4219
4220#define CASE_RVV_OPCODE(OP) \
4221 CASE_RVV_OPCODE_UNMASK(OP): \
4222 case CASE_RVV_OPCODE_MASK(OP)
4223// clang-format on
4224
4225// clang-format off
4226#define CASE_VMA_OPCODE_COMMON(OP, TYPE, LMUL) \
4227 RISCV::PseudoV##OP##_##TYPE##_##LMUL
4228
4229#define CASE_VMA_OPCODE_LMULS(OP, TYPE) \
4230 CASE_VMA_OPCODE_COMMON(OP, TYPE, MF8): \
4231 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF4): \
4232 case CASE_VMA_OPCODE_COMMON(OP, TYPE, MF2): \
4233 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M1): \
4234 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M2): \
4235 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M4): \
4236 case CASE_VMA_OPCODE_COMMON(OP, TYPE, M8)
4237
4238// VFMA instructions are SEW specific.
4239#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL, SEW) \
4240 RISCV::PseudoV##OP##_##TYPE##_##LMUL##_##SEW
4241
4242#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW) \
4243 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1, SEW): \
4244 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2, SEW): \
4245 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4, SEW): \
4246 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8, SEW)
4247
4248#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW) \
4249 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2, SEW): \
4250 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE, SEW)
4251
4252#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE, SEW) \
4253 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4, SEW): \
4254 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE, SEW)
4255
4256#define CASE_VFMA_OPCODE_VV(OP) \
4257 CASE_VFMA_OPCODE_LMULS_MF4(OP, VV, E16): \
4258 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VV, E16): \
4259 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VV, E32): \
4260 case CASE_VFMA_OPCODE_LMULS_M1(OP, VV, E64)
4261
4262#define CASE_VFMA_SPLATS(OP) \
4263 CASE_VFMA_OPCODE_LMULS_MF4(OP, VFPR16, E16): \
4264 case CASE_VFMA_OPCODE_LMULS_MF4(OP##_ALT, VFPR16, E16): \
4265 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VFPR32, E32): \
4266 case CASE_VFMA_OPCODE_LMULS_M1(OP, VFPR64, E64)
4267// clang-format on
4268
4270 unsigned &SrcOpIdx1,
4271 unsigned &SrcOpIdx2) const {
4272 const MCInstrDesc &Desc = MI.getDesc();
4273 if (!Desc.isCommutable())
4274 return false;
4275
4276 switch (MI.getOpcode()) {
4277 case RISCV::TH_MVEQZ:
4278 case RISCV::TH_MVNEZ:
4279 // We can't commute operands if operand 2 (i.e., rs1 in
4280 // mveqz/mvnez rd,rs1,rs2) is the zero-register (as it is
4281 // not valid as the in/out-operand 1).
4282 if (MI.getOperand(2).getReg() == RISCV::X0)
4283 return false;
4284 // Operands 1 and 2 are commutable, if we switch the opcode.
4285 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
4286 case RISCV::QC_SELECTIEQ:
4287 case RISCV::QC_SELECTINE:
4288 case RISCV::QC_SELECTIIEQ:
4289 case RISCV::QC_SELECTIINE:
4290 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
4291 case RISCV::QC_MVEQ:
4292 case RISCV::QC_MVNE:
4293 case RISCV::QC_MVLT:
4294 case RISCV::QC_MVGE:
4295 case RISCV::QC_MVLTU:
4296 case RISCV::QC_MVGEU:
4297 case RISCV::QC_MVEQI:
4298 case RISCV::QC_MVNEI:
4299 case RISCV::QC_MVLTI:
4300 case RISCV::QC_MVGEI:
4301 case RISCV::QC_MVLTUI:
4302 case RISCV::QC_MVGEUI:
4303 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 4);
4304 case RISCV::TH_MULA:
4305 case RISCV::TH_MULAW:
4306 case RISCV::TH_MULAH:
4307 case RISCV::TH_MULS:
4308 case RISCV::TH_MULSW:
4309 case RISCV::TH_MULSH:
4310 // Operands 2 and 3 are commutable.
4311 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
4312 case RISCV::PseudoCCMOVGPRNoX0:
4313 case RISCV::PseudoCCMOVGPR:
4314 // Operands 1 and 2 are commutable.
4315 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1, 2);
4316 case CASE_RVV_OPCODE(VADD_VV):
4317 case CASE_RVV_OPCODE(VAND_VV):
4318 case CASE_RVV_OPCODE(VOR_VV):
4319 case CASE_RVV_OPCODE(VXOR_VV):
4320 case CASE_RVV_OPCODE_MASK(VMSEQ_VV):
4321 case CASE_RVV_OPCODE_MASK(VMSNE_VV):
4322 case CASE_RVV_OPCODE(VMIN_VV):
4323 case CASE_RVV_OPCODE(VMINU_VV):
4324 case CASE_RVV_OPCODE(VMAX_VV):
4325 case CASE_RVV_OPCODE(VMAXU_VV):
4326 case CASE_RVV_OPCODE(VMUL_VV):
4327 case CASE_RVV_OPCODE(VMULH_VV):
4328 case CASE_RVV_OPCODE(VMULHU_VV):
4329 case CASE_RVV_OPCODE_WIDEN(VWADD_VV):
4330 case CASE_RVV_OPCODE_WIDEN(VWADDU_VV):
4331 case CASE_RVV_OPCODE_WIDEN(VWMUL_VV):
4332 case CASE_RVV_OPCODE_WIDEN(VWMULU_VV):
4333 case CASE_RVV_OPCODE_WIDEN(VWMACC_VV):
4334 case CASE_RVV_OPCODE_WIDEN(VWMACCU_VV):
4335 case CASE_RVV_OPCODE(VABD_VV):
4336 case CASE_RVV_OPCODE(VABDU_VV):
4337 case CASE_RVV_OPCODE_WIDEN(VWABDA_VV):
4338 case CASE_RVV_OPCODE_WIDEN(VWABDAU_VV):
4339 case CASE_RVV_OPCODE_UNMASK(VADC_VVM):
4340 case CASE_RVV_OPCODE(VSADD_VV):
4341 case CASE_RVV_OPCODE(VSADDU_VV):
4342 case CASE_RVV_OPCODE(VAADD_VV):
4343 case CASE_RVV_OPCODE(VAADDU_VV):
4344 case CASE_RVV_OPCODE(VSMUL_VV):
4345 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, MF2):
4346 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M1):
4347 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M2):
4348 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M4):
4349 case CASE_RVV_OPCODE_LMUL(VDOT4A_VV, M8):
4350 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, MF2):
4351 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M1):
4352 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M2):
4353 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M4):
4354 case CASE_RVV_OPCODE_LMUL(VDOT4AU_VV, M8):
4355 // Operands 2 and 3 are commutable.
4356 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
4357 case CASE_VFMA_SPLATS(FMADD):
4358 case CASE_VFMA_SPLATS(FMSUB):
4359 case CASE_VFMA_SPLATS(FMACC):
4360 case CASE_VFMA_SPLATS(FMSAC):
4363 case CASE_VFMA_SPLATS(FNMACC):
4364 case CASE_VFMA_SPLATS(FNMSAC):
4365 case CASE_VFMA_OPCODE_VV(FMACC):
4366 case CASE_VFMA_OPCODE_VV(FMSAC):
4367 case CASE_VFMA_OPCODE_VV(FNMACC):
4368 case CASE_VFMA_OPCODE_VV(FNMSAC):
4369 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4370 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4371 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4372 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4373 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4374 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4375 // If the tail policy is undisturbed we can't commute.
4376 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4377 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
4378 1) == 0)
4379 return false;
4380
4381 // For these instructions we can only swap operand 1 and operand 3 by
4382 // changing the opcode.
4383 unsigned CommutableOpIdx1 = 1;
4384 unsigned CommutableOpIdx2 = 3;
4385 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
4386 CommutableOpIdx2))
4387 return false;
4388 return true;
4389 }
4390 case CASE_VFMA_OPCODE_VV(FMADD):
4394 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4395 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4396 // If the tail policy is undisturbed we can't commute.
4397 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
4398 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
4399 1) == 0)
4400 return false;
4401
4402 // For these instructions we have more freedom. We can commute with the
4403 // other multiplicand or with the addend/subtrahend/minuend.
4404
4405 // Any fixed operand must be from source 1, 2 or 3.
4406 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
4407 return false;
4408 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
4409 return false;
4410
4411 // It both ops are fixed one must be the tied source.
4412 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
4413 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
4414 return false;
4415
4416 // Look for two different register operands assumed to be commutable
4417 // regardless of the FMA opcode. The FMA opcode is adjusted later if
4418 // needed.
4419 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
4420 SrcOpIdx2 == CommuteAnyOperandIndex) {
4421 // At least one of operands to be commuted is not specified and
4422 // this method is free to choose appropriate commutable operands.
4423 unsigned CommutableOpIdx1 = SrcOpIdx1;
4424 if (SrcOpIdx1 == SrcOpIdx2) {
4425 // Both of operands are not fixed. Set one of commutable
4426 // operands to the tied source.
4427 CommutableOpIdx1 = 1;
4428 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
4429 // Only one of the operands is not fixed.
4430 CommutableOpIdx1 = SrcOpIdx2;
4431 }
4432
4433 // CommutableOpIdx1 is well defined now. Let's choose another commutable
4434 // operand and assign its index to CommutableOpIdx2.
4435 unsigned CommutableOpIdx2;
4436 if (CommutableOpIdx1 != 1) {
4437 // If we haven't already used the tied source, we must use it now.
4438 CommutableOpIdx2 = 1;
4439 } else {
4440 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
4441
4442 // The commuted operands should have different registers.
4443 // Otherwise, the commute transformation does not change anything and
4444 // is useless. We use this as a hint to make our decision.
4445 if (Op1Reg != MI.getOperand(2).getReg())
4446 CommutableOpIdx2 = 2;
4447 else
4448 CommutableOpIdx2 = 3;
4449 }
4450
4451 // Assign the found pair of commutable indices to SrcOpIdx1 and
4452 // SrcOpIdx2 to return those values.
4453 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
4454 CommutableOpIdx2))
4455 return false;
4456 }
4457
4458 return true;
4459 }
4460 }
4461
4462 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
4463}
4464
4465// clang-format off
4466#define CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
4467 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
4468 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
4469 break;
4470
4471#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
4472 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
4473 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
4474 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
4475 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
4476 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
4477 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
4478 CASE_VMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
4479
4480// VFMA depends on SEW.
4481#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL, SEW) \
4482 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL##_##SEW: \
4483 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL##_##SEW; \
4484 break;
4485
4486#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW) \
4487 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1, SEW) \
4488 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2, SEW) \
4489 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4, SEW) \
4490 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8, SEW)
4491
4492#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW) \
4493 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2, SEW) \
4494 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE, SEW)
4495
4496#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE, SEW) \
4497 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4, SEW) \
4498 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE, SEW)
4499
4500#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP) \
4501 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VV, E16) \
4502 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VV, E16) \
4503 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VV, E32) \
4504 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VV, E64)
4505
4506#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
4507 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VFPR16, E16) \
4508 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP##_ALT, NEWOP##_ALT, VFPR16, E16) \
4509 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VFPR32, E32) \
4510 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VFPR64, E64)
4511// clang-format on
4512
4514 bool NewMI,
4515 unsigned OpIdx1,
4516 unsigned OpIdx2) const {
4517 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
4518 if (NewMI)
4519 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
4520 return MI;
4521 };
4522
4523 switch (MI.getOpcode()) {
4524 case RISCV::TH_MVEQZ:
4525 case RISCV::TH_MVNEZ: {
4526 auto &WorkingMI = cloneIfNew(MI);
4527 WorkingMI.setDesc(get(MI.getOpcode() == RISCV::TH_MVEQZ ? RISCV::TH_MVNEZ
4528 : RISCV::TH_MVEQZ));
4529 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
4530 OpIdx2);
4531 }
4532 case RISCV::QC_SELECTIEQ:
4533 case RISCV::QC_SELECTINE:
4534 case RISCV::QC_SELECTIIEQ:
4535 case RISCV::QC_SELECTIINE:
4536 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4537 case RISCV::QC_MVEQ:
4538 case RISCV::QC_MVNE:
4539 case RISCV::QC_MVLT:
4540 case RISCV::QC_MVGE:
4541 case RISCV::QC_MVLTU:
4542 case RISCV::QC_MVGEU:
4543 case RISCV::QC_MVEQI:
4544 case RISCV::QC_MVNEI:
4545 case RISCV::QC_MVLTI:
4546 case RISCV::QC_MVGEI:
4547 case RISCV::QC_MVLTUI:
4548 case RISCV::QC_MVGEUI: {
4549 auto &WorkingMI = cloneIfNew(MI);
4550 WorkingMI.setDesc(get(getInverseXqcicmOpcode(MI.getOpcode())));
4551 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, false, OpIdx1,
4552 OpIdx2);
4553 }
4554 case RISCV::PseudoCCMOVGPRNoX0:
4555 case RISCV::PseudoCCMOVGPR: {
4556 // CCMOV can be commuted by inverting the condition.
4557 unsigned BCC = MI.getOperand(MI.getNumExplicitOperands() - 3).getImm();
4559 auto &WorkingMI = cloneIfNew(MI);
4560 WorkingMI.getOperand(MI.getNumExplicitOperands() - 3).setImm(BCC);
4561 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
4562 OpIdx1, OpIdx2);
4563 }
4564 case CASE_VFMA_SPLATS(FMACC):
4565 case CASE_VFMA_SPLATS(FMADD):
4566 case CASE_VFMA_SPLATS(FMSAC):
4567 case CASE_VFMA_SPLATS(FMSUB):
4568 case CASE_VFMA_SPLATS(FNMACC):
4570 case CASE_VFMA_SPLATS(FNMSAC):
4572 case CASE_VFMA_OPCODE_VV(FMACC):
4573 case CASE_VFMA_OPCODE_VV(FMSAC):
4574 case CASE_VFMA_OPCODE_VV(FNMACC):
4575 case CASE_VFMA_OPCODE_VV(FNMSAC):
4576 case CASE_VMA_OPCODE_LMULS(MADD, VX):
4577 case CASE_VMA_OPCODE_LMULS(NMSUB, VX):
4578 case CASE_VMA_OPCODE_LMULS(MACC, VX):
4579 case CASE_VMA_OPCODE_LMULS(NMSAC, VX):
4580 case CASE_VMA_OPCODE_LMULS(MACC, VV):
4581 case CASE_VMA_OPCODE_LMULS(NMSAC, VV): {
4582 // It only make sense to toggle these between clobbering the
4583 // addend/subtrahend/minuend one of the multiplicands.
4584 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4585 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
4586 unsigned Opc;
4587 switch (MI.getOpcode()) {
4588 default:
4589 llvm_unreachable("Unexpected opcode");
4590 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
4591 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
4598 CASE_VFMA_CHANGE_OPCODE_VV(FMACC, FMADD)
4602 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
4603 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
4604 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
4605 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
4606 CASE_VMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
4607 CASE_VMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
4608 }
4609
4610 auto &WorkingMI = cloneIfNew(MI);
4611 WorkingMI.setDesc(get(Opc));
4612 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
4613 OpIdx1, OpIdx2);
4614 }
4615 case CASE_VFMA_OPCODE_VV(FMADD):
4619 case CASE_VMA_OPCODE_LMULS(MADD, VV):
4620 case CASE_VMA_OPCODE_LMULS(NMSUB, VV): {
4621 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
4622 // If one of the operands, is the addend we need to change opcode.
4623 // Otherwise we're just swapping 2 of the multiplicands.
4624 if (OpIdx1 == 3 || OpIdx2 == 3) {
4625 unsigned Opc;
4626 switch (MI.getOpcode()) {
4627 default:
4628 llvm_unreachable("Unexpected opcode");
4629 CASE_VFMA_CHANGE_OPCODE_VV(FMADD, FMACC)
4633 CASE_VMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
4634 CASE_VMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
4635 }
4636
4637 auto &WorkingMI = cloneIfNew(MI);
4638 WorkingMI.setDesc(get(Opc));
4639 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
4640 OpIdx1, OpIdx2);
4641 }
4642 // Let the default code handle it.
4643 break;
4644 }
4645 }
4646
4647 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
4648}
4649
4650#undef CASE_VMA_CHANGE_OPCODE_COMMON
4651#undef CASE_VMA_CHANGE_OPCODE_LMULS
4652#undef CASE_VFMA_CHANGE_OPCODE_COMMON
4653#undef CASE_VFMA_CHANGE_OPCODE_LMULS_M1
4654#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF2
4655#undef CASE_VFMA_CHANGE_OPCODE_LMULS_MF4
4656#undef CASE_VFMA_CHANGE_OPCODE_VV
4657#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
4658
4659#undef CASE_RVV_OPCODE_UNMASK_LMUL
4660#undef CASE_RVV_OPCODE_MASK_LMUL
4661#undef CASE_RVV_OPCODE_LMUL
4662#undef CASE_RVV_OPCODE_UNMASK_WIDEN
4663#undef CASE_RVV_OPCODE_UNMASK
4664#undef CASE_RVV_OPCODE_MASK_WIDEN
4665#undef CASE_RVV_OPCODE_MASK
4666#undef CASE_RVV_OPCODE_WIDEN
4667#undef CASE_RVV_OPCODE
4668
4669#undef CASE_VMA_OPCODE_COMMON
4670#undef CASE_VMA_OPCODE_LMULS
4671#undef CASE_VFMA_OPCODE_COMMON
4672#undef CASE_VFMA_OPCODE_LMULS_M1
4673#undef CASE_VFMA_OPCODE_LMULS_MF2
4674#undef CASE_VFMA_OPCODE_LMULS_MF4
4675#undef CASE_VFMA_OPCODE_VV
4676#undef CASE_VFMA_SPLATS
4677
4679 switch (MI.getOpcode()) {
4680 default:
4681 break;
4682 case RISCV::ADD:
4683 case RISCV::OR:
4684 case RISCV::XOR:
4685 // Normalize (so we hit the next if clause).
4686 // add/[x]or rd, zero, rs => add/[x]or rd, rs, zero
4687 if (MI.getOperand(1).getReg() == RISCV::X0)
4688 commuteInstruction(MI);
4689 // add/[x]or rd, rs, zero => addi rd, rs, 0
4690 if (MI.getOperand(2).getReg() == RISCV::X0) {
4691 MI.getOperand(2).ChangeToImmediate(0);
4692 MI.setDesc(get(RISCV::ADDI));
4693 return true;
4694 }
4695 // xor rd, rs, rs => addi rd, zero, 0
4696 if (MI.getOpcode() == RISCV::XOR &&
4697 MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
4698 MI.getOperand(1).setReg(RISCV::X0);
4699 MI.getOperand(2).ChangeToImmediate(0);
4700 MI.setDesc(get(RISCV::ADDI));
4701 return true;
4702 }
4703 break;
4704 case RISCV::ORI:
4705 case RISCV::XORI:
4706 // [x]ori rd, zero, N => addi rd, zero, N
4707 if (MI.getOperand(1).getReg() == RISCV::X0) {
4708 MI.setDesc(get(RISCV::ADDI));
4709 return true;
4710 }
4711 break;
4712 case RISCV::SUB:
4713 // sub rd, rs, zero => addi rd, rs, 0
4714 if (MI.getOperand(2).getReg() == RISCV::X0) {
4715 MI.getOperand(2).ChangeToImmediate(0);
4716 MI.setDesc(get(RISCV::ADDI));
4717 return true;
4718 }
4719 break;
4720 case RISCV::SUBW:
4721 // subw rd, rs, zero => addiw rd, rs, 0
4722 if (MI.getOperand(2).getReg() == RISCV::X0) {
4723 MI.getOperand(2).ChangeToImmediate(0);
4724 MI.setDesc(get(RISCV::ADDIW));
4725 return true;
4726 }
4727 break;
4728 case RISCV::ADDW:
4729 // Normalize (so we hit the next if clause).
4730 // addw rd, zero, rs => addw rd, rs, zero
4731 if (MI.getOperand(1).getReg() == RISCV::X0)
4732 commuteInstruction(MI);
4733 // addw rd, rs, zero => addiw rd, rs, 0
4734 if (MI.getOperand(2).getReg() == RISCV::X0) {
4735 MI.getOperand(2).ChangeToImmediate(0);
4736 MI.setDesc(get(RISCV::ADDIW));
4737 return true;
4738 }
4739 break;
4740 case RISCV::SH1ADD:
4741 case RISCV::SH1ADD_UW:
4742 case RISCV::SH2ADD:
4743 case RISCV::SH2ADD_UW:
4744 case RISCV::SH3ADD:
4745 case RISCV::SH3ADD_UW:
4746 // shNadd[.uw] rd, zero, rs => addi rd, rs, 0
4747 if (MI.getOperand(1).getReg() == RISCV::X0) {
4748 MI.removeOperand(1);
4749 MI.addOperand(MachineOperand::CreateImm(0));
4750 MI.setDesc(get(RISCV::ADDI));
4751 return true;
4752 }
4753 // shNadd[.uw] rd, rs, zero => slli[.uw] rd, rs, N
4754 if (MI.getOperand(2).getReg() == RISCV::X0) {
4755 MI.removeOperand(2);
4756 unsigned Opc = MI.getOpcode();
4757 if (Opc == RISCV::SH1ADD_UW || Opc == RISCV::SH2ADD_UW ||
4758 Opc == RISCV::SH3ADD_UW) {
4760 MI.setDesc(get(RISCV::SLLI_UW));
4761 return true;
4762 }
4764 MI.setDesc(get(RISCV::SLLI));
4765 return true;
4766 }
4767 break;
4768 case RISCV::AND:
4769 case RISCV::MUL:
4770 case RISCV::MULH:
4771 case RISCV::MULHSU:
4772 case RISCV::MULHU:
4773 case RISCV::MULW:
4774 // and rd, zero, rs => addi rd, zero, 0
4775 // mul* rd, zero, rs => addi rd, zero, 0
4776 // and rd, rs, zero => addi rd, zero, 0
4777 // mul* rd, rs, zero => addi rd, zero, 0
4778 if (MI.getOperand(1).getReg() == RISCV::X0 ||
4779 MI.getOperand(2).getReg() == RISCV::X0) {
4780 MI.getOperand(1).setReg(RISCV::X0);
4781 MI.getOperand(2).ChangeToImmediate(0);
4782 MI.setDesc(get(RISCV::ADDI));
4783 return true;
4784 }
4785 break;
4786 case RISCV::ANDI:
4787 // andi rd, zero, C => addi rd, zero, 0
4788 if (MI.getOperand(1).getReg() == RISCV::X0) {
4789 MI.getOperand(2).setImm(0);
4790 MI.setDesc(get(RISCV::ADDI));
4791 return true;
4792 }
4793 break;
4794 case RISCV::SLL:
4795 case RISCV::SRL:
4796 case RISCV::SRA:
4797 // shift rd, zero, rs => addi rd, zero, 0
4798 if (MI.getOperand(1).getReg() == RISCV::X0) {
4799 MI.getOperand(2).ChangeToImmediate(0);
4800 MI.setDesc(get(RISCV::ADDI));
4801 return true;
4802 }
4803 // shift rd, rs, zero => addi rd, rs, 0
4804 if (MI.getOperand(2).getReg() == RISCV::X0) {
4805 MI.getOperand(2).ChangeToImmediate(0);
4806 MI.setDesc(get(RISCV::ADDI));
4807 return true;
4808 }
4809 break;
4810 case RISCV::SLLW:
4811 case RISCV::SRLW:
4812 case RISCV::SRAW:
4813 // shiftw rd, zero, rs => addi rd, zero, 0
4814 if (MI.getOperand(1).getReg() == RISCV::X0) {
4815 MI.getOperand(2).ChangeToImmediate(0);
4816 MI.setDesc(get(RISCV::ADDI));
4817 return true;
4818 }
4819 break;
4820 case RISCV::SLLI:
4821 case RISCV::SRLI:
4822 case RISCV::SRAI:
4823 case RISCV::SLLIW:
4824 case RISCV::SRLIW:
4825 case RISCV::SRAIW:
4826 case RISCV::SLLI_UW:
4827 // shiftimm rd, zero, N => addi rd, zero, 0
4828 if (MI.getOperand(1).getReg() == RISCV::X0) {
4829 MI.getOperand(2).setImm(0);
4830 MI.setDesc(get(RISCV::ADDI));
4831 return true;
4832 }
4833 break;
4834 case RISCV::SLTU:
4835 case RISCV::ADD_UW:
4836 // sltu rd, zero, zero => addi rd, zero, 0
4837 // add.uw rd, zero, zero => addi rd, zero, 0
4838 if (MI.getOperand(1).getReg() == RISCV::X0 &&
4839 MI.getOperand(2).getReg() == RISCV::X0) {
4840 MI.getOperand(2).ChangeToImmediate(0);
4841 MI.setDesc(get(RISCV::ADDI));
4842 return true;
4843 }
4844 // add.uw rd, zero, rs => addi rd, rs, 0
4845 if (MI.getOpcode() == RISCV::ADD_UW &&
4846 MI.getOperand(1).getReg() == RISCV::X0) {
4847 MI.removeOperand(1);
4848 MI.addOperand(MachineOperand::CreateImm(0));
4849 MI.setDesc(get(RISCV::ADDI));
4850 }
4851 break;
4852 case RISCV::SLTIU:
4853 // sltiu rd, zero, NZC => addi rd, zero, 1
4854 // sltiu rd, zero, 0 => addi rd, zero, 0
4855 if (MI.getOperand(1).getReg() == RISCV::X0) {
4856 MI.getOperand(2).setImm(MI.getOperand(2).getImm() != 0);
4857 MI.setDesc(get(RISCV::ADDI));
4858 return true;
4859 }
4860 break;
4861 case RISCV::SEXT_H:
4862 case RISCV::SEXT_B:
4863 case RISCV::ZEXT_H_RV32:
4864 case RISCV::ZEXT_H_RV64:
4865 // sext.[hb] rd, zero => addi rd, zero, 0
4866 // zext.h rd, zero => addi rd, zero, 0
4867 if (MI.getOperand(1).getReg() == RISCV::X0) {
4868 MI.addOperand(MachineOperand::CreateImm(0));
4869 MI.setDesc(get(RISCV::ADDI));
4870 return true;
4871 }
4872 break;
4873 case RISCV::MIN:
4874 case RISCV::MINU:
4875 case RISCV::MAX:
4876 case RISCV::MAXU:
4877 // min|max rd, rs, rs => addi rd, rs, 0
4878 if (MI.getOperand(1).getReg() == MI.getOperand(2).getReg()) {
4879 MI.getOperand(2).ChangeToImmediate(0);
4880 MI.setDesc(get(RISCV::ADDI));
4881 return true;
4882 }
4883 break;
4884 case RISCV::BEQ:
4885 case RISCV::BNE:
4886 // b{eq,ne} zero, rs, imm => b{eq,ne} rs, zero, imm
4887 if (MI.getOperand(0).getReg() == RISCV::X0) {
4888 MachineOperand MO0 = MI.getOperand(0);
4889 MI.removeOperand(0);
4890 MI.insert(MI.operands_begin() + 1, {MO0});
4891 }
4892 break;
4893 case RISCV::BLTU:
4894 // bltu zero, rs, imm => bne rs, zero, imm
4895 if (MI.getOperand(0).getReg() == RISCV::X0) {
4896 MachineOperand MO0 = MI.getOperand(0);
4897 MI.removeOperand(0);
4898 MI.insert(MI.operands_begin() + 1, {MO0});
4899 MI.setDesc(get(RISCV::BNE));
4900 }
4901 break;
4902 case RISCV::BGEU:
4903 // bgeu zero, rs, imm => beq rs, zero, imm
4904 if (MI.getOperand(0).getReg() == RISCV::X0) {
4905 MachineOperand MO0 = MI.getOperand(0);
4906 MI.removeOperand(0);
4907 MI.insert(MI.operands_begin() + 1, {MO0});
4908 MI.setDesc(get(RISCV::BEQ));
4909 }
4910 break;
4911 }
4912 return false;
4913}
4914
4915// clang-format off
4916#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
4917 RISCV::PseudoV##OP##_##LMUL##_TIED
4918
4919#define CASE_WIDEOP_OPCODE_LMULS(OP) \
4920 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
4921 case CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
4922 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
4923 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
4924 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
4925 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
4926
4927#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
4928 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
4929 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
4930 break;
4931
4932#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4933 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
4934 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
4935 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
4936 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
4937 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
4938 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
4939
4940// FP Widening Ops may by SEW aware. Create SEW aware cases for these cases.
4941#define CASE_FP_WIDEOP_OPCODE_COMMON(OP, LMUL, SEW) \
4942 RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED
4943
4944#define CASE_FP_WIDEOP_OPCODE_LMULS(OP) \
4945 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4946 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4947 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E32): \
4948 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4949 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E32): \
4950 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4951 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E32): \
4952 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16): \
4953 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E32) \
4954
4955#define CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL, SEW) \
4956 case RISCV::PseudoV##OP##_##LMUL##_##SEW##_TIED: \
4957 NewOpc = RISCV::PseudoV##OP##_##LMUL##_##SEW; \
4958 break;
4959
4960#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
4961 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4962 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4963 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E32) \
4964 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4965 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E32) \
4966 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4967 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E32) \
4968 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16) \
4969 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E32) \
4970
4971#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP) \
4972 CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF4, E16): \
4973 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, MF2, E16): \
4974 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M1, E16): \
4975 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M2, E16): \
4976 case CASE_FP_WIDEOP_OPCODE_COMMON(OP, M4, E16)
4977
4978#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP) \
4979 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4, E16) \
4980 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2, E16) \
4981 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1, E16) \
4982 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2, E16) \
4983 CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4, E16)
4984// clang-format on
4985
4987 LiveIntervals *LIS) const {
4989 switch (MI.getOpcode()) {
4990 default:
4991 return nullptr;
4992 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWADD_ALT_WV):
4993 case CASE_FP_WIDEOP_OPCODE_LMULS_ALT(FWSUB_ALT_WV):
4994 case CASE_FP_WIDEOP_OPCODE_LMULS(FWADD_WV):
4995 case CASE_FP_WIDEOP_OPCODE_LMULS(FWSUB_WV): {
4996 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
4997 MI.getNumExplicitOperands() == 7 &&
4998 "Expect 7 explicit operands rd, rs2, rs1, rm, vl, sew, policy");
4999 // If the tail policy is undisturbed we can't convert.
5000 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
5001 1) == 0)
5002 return nullptr;
5003 // clang-format off
5004 unsigned NewOpc;
5005 switch (MI.getOpcode()) {
5006 default:
5007 llvm_unreachable("Unexpected opcode");
5012 }
5013 // clang-format on
5014
5015 MachineBasicBlock &MBB = *MI.getParent();
5016 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
5017 .add(MI.getOperand(0))
5018 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
5019 .add(MI.getOperand(1))
5020 .add(MI.getOperand(2))
5021 .add(MI.getOperand(3))
5022 .add(MI.getOperand(4))
5023 .add(MI.getOperand(5))
5024 .add(MI.getOperand(6));
5025 break;
5026 }
5027 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
5028 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
5029 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
5030 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
5031 // If the tail policy is undisturbed we can't convert.
5032 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
5033 MI.getNumExplicitOperands() == 6);
5034 if ((MI.getOperand(RISCVII::getVecPolicyOpNum(MI.getDesc())).getImm() &
5035 1) == 0)
5036 return nullptr;
5037
5038 // clang-format off
5039 unsigned NewOpc;
5040 switch (MI.getOpcode()) {
5041 default:
5042 llvm_unreachable("Unexpected opcode");
5047 }
5048 // clang-format on
5049
5050 MachineBasicBlock &MBB = *MI.getParent();
5051 MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
5052 .add(MI.getOperand(0))
5053 .addReg(MI.getOperand(0).getReg(), RegState::Undef)
5054 .add(MI.getOperand(1))
5055 .add(MI.getOperand(2))
5056 .add(MI.getOperand(3))
5057 .add(MI.getOperand(4))
5058 .add(MI.getOperand(5));
5059 break;
5060 }
5061 }
5062 MIB.copyImplicitOps(MI);
5063
5064 if (LIS) {
5065 SlotIndex Idx = LIS->ReplaceMachineInstrInMaps(MI, *MIB);
5066
5067 if (MI.getOperand(0).isEarlyClobber()) {
5068 // Use operand 1 was tied to early-clobber def operand 0, so its live
5069 // interval could have ended at an early-clobber slot. Now they are not
5070 // tied we need to update it to the normal register slot.
5071 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());
5073 if (S->end == Idx.getRegSlot(true))
5074 S->end = Idx.getRegSlot();
5075 }
5076 }
5077
5078 return MIB;
5079}
5080
5081#undef CASE_WIDEOP_OPCODE_COMMON
5082#undef CASE_WIDEOP_OPCODE_LMULS
5083#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
5084#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
5085#undef CASE_FP_WIDEOP_OPCODE_COMMON
5086#undef CASE_FP_WIDEOP_OPCODE_LMULS
5087#undef CASE_FP_WIDEOP_CHANGE_OPCODE_COMMON
5088#undef CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS
5089
5092 Register DestReg, uint32_t Amount,
5093 MachineInstr::MIFlag Flag) const {
5095 if (llvm::has_single_bit(Amount)) {
5096 uint32_t ShiftAmount = Log2_32(Amount);
5097 if (ShiftAmount == 0)
5098 return;
5099 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
5100 .addReg(DestReg, RegState::Kill)
5101 .addImm(ShiftAmount)
5102 .setMIFlag(Flag);
5103 } else if (int ShXAmount, ShiftAmount;
5104 STI.hasStdExtZba() &&
5105 (ShXAmount = isShifted359(Amount, ShiftAmount)) != 0) {
5106 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
5107 unsigned Opc;
5108 switch (ShXAmount) {
5109 case 1:
5110 Opc = RISCV::SH1ADD;
5111 break;
5112 case 2:
5113 Opc = RISCV::SH2ADD;
5114 break;
5115 case 3:
5116 Opc = RISCV::SH3ADD;
5117 break;
5118 default:
5119 llvm_unreachable("unexpected result of isShifted359");
5120 }
5121 if (ShiftAmount)
5122 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
5123 .addReg(DestReg, RegState::Kill)
5124 .addImm(ShiftAmount)
5125 .setMIFlag(Flag);
5126 BuildMI(MBB, II, DL, get(Opc), DestReg)
5127 .addReg(DestReg, RegState::Kill)
5128 .addReg(DestReg)
5129 .setMIFlag(Flag);
5130 } else if (llvm::has_single_bit(Amount - 1)) {
5131 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
5132 uint32_t ShiftAmount = Log2_32(Amount - 1);
5133 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
5134 .addReg(DestReg)
5135 .addImm(ShiftAmount)
5136 .setMIFlag(Flag);
5137 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
5138 .addReg(ScaledRegister, RegState::Kill)
5139 .addReg(DestReg, RegState::Kill)
5140 .setMIFlag(Flag);
5141 } else if (llvm::has_single_bit(Amount + 1)) {
5142 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
5143 uint32_t ShiftAmount = Log2_32(Amount + 1);
5144 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
5145 .addReg(DestReg)
5146 .addImm(ShiftAmount)
5147 .setMIFlag(Flag);
5148 BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
5149 .addReg(ScaledRegister, RegState::Kill)
5150 .addReg(DestReg, RegState::Kill)
5151 .setMIFlag(Flag);
5152 } else if (STI.hasStdExtZmmul()) {
5153 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
5154 movImm(MBB, II, DL, N, Amount, Flag);
5155 BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
5156 .addReg(DestReg, RegState::Kill)
5158 .setMIFlag(Flag);
5159 } else {
5160 Register Acc;
5161 uint32_t PrevShiftAmount = 0;
5162 for (uint32_t ShiftAmount = 0; Amount >> ShiftAmount; ShiftAmount++) {
5163 if (Amount & (1U << ShiftAmount)) {
5164 if (ShiftAmount)
5165 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
5166 .addReg(DestReg, RegState::Kill)
5167 .addImm(ShiftAmount - PrevShiftAmount)
5168 .setMIFlag(Flag);
5169 if (Amount >> (ShiftAmount + 1)) {
5170 // If we don't have an accmulator yet, create it and copy DestReg.
5171 if (!Acc) {
5172 Acc = MRI.createVirtualRegister(&RISCV::GPRRegClass);
5173 BuildMI(MBB, II, DL, get(TargetOpcode::COPY), Acc)
5174 .addReg(DestReg)
5175 .setMIFlag(Flag);
5176 } else {
5177 BuildMI(MBB, II, DL, get(RISCV::ADD), Acc)
5178 .addReg(Acc, RegState::Kill)
5179 .addReg(DestReg)
5180 .setMIFlag(Flag);
5181 }
5182 }
5183 PrevShiftAmount = ShiftAmount;
5184 }
5185 }
5186 assert(Acc && "Expected valid accumulator");
5187 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
5188 .addReg(DestReg, RegState::Kill)
5189 .addReg(Acc, RegState::Kill)
5190 .setMIFlag(Flag);
5191 }
5192}
5193
5196 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
5197 {{MONontemporalBit0, "riscv-nontemporal-domain-bit-0"},
5198 {MONontemporalBit1, "riscv-nontemporal-domain-bit-1"}};
5199 return ArrayRef(TargetFlags);
5200}
5201
5203 return OptLevel >= CodeGenOptLevel::Aggressive
5204 ? STI.getTailDupAggressiveThreshold()
5205 : 2;
5206}
5207
5209 // RVV lacks any support for immediate addressing for stack addresses, so be
5210 // conservative.
5211 unsigned Opcode = MI.getOpcode();
5212 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
5214 return false;
5215 return true;
5216}
5217
5218/// Return true if \p MI is a copy that will be lowered to one or more vmvNr.vs.
5220 const MachineInstr &MI) {
5221 return MI.isCopy() && MI.getOperand(0).getReg().isPhysical() &&
5223 TRI->getMinimalPhysRegClass(MI.getOperand(0).getReg()));
5224}
5225
5226std::optional<std::pair<unsigned, unsigned>>
5228 switch (Opcode) {
5229 default:
5230 return std::nullopt;
5231 case RISCV::PseudoVSPILL2_M1:
5232 case RISCV::PseudoVRELOAD2_M1:
5233 return std::make_pair(2u, 1u);
5234 case RISCV::PseudoVSPILL2_M2:
5235 case RISCV::PseudoVRELOAD2_M2:
5236 return std::make_pair(2u, 2u);
5237 case RISCV::PseudoVSPILL2_M4:
5238 case RISCV::PseudoVRELOAD2_M4:
5239 return std::make_pair(2u, 4u);
5240 case RISCV::PseudoVSPILL3_M1:
5241 case RISCV::PseudoVRELOAD3_M1:
5242 return std::make_pair(3u, 1u);
5243 case RISCV::PseudoVSPILL3_M2:
5244 case RISCV::PseudoVRELOAD3_M2:
5245 return std::make_pair(3u, 2u);
5246 case RISCV::PseudoVSPILL4_M1:
5247 case RISCV::PseudoVRELOAD4_M1:
5248 return std::make_pair(4u, 1u);
5249 case RISCV::PseudoVSPILL4_M2:
5250 case RISCV::PseudoVRELOAD4_M2:
5251 return std::make_pair(4u, 2u);
5252 case RISCV::PseudoVSPILL5_M1:
5253 case RISCV::PseudoVRELOAD5_M1:
5254 return std::make_pair(5u, 1u);
5255 case RISCV::PseudoVSPILL6_M1:
5256 case RISCV::PseudoVRELOAD6_M1:
5257 return std::make_pair(6u, 1u);
5258 case RISCV::PseudoVSPILL7_M1:
5259 case RISCV::PseudoVRELOAD7_M1:
5260 return std::make_pair(7u, 1u);
5261 case RISCV::PseudoVSPILL8_M1:
5262 case RISCV::PseudoVRELOAD8_M1:
5263 return std::make_pair(8u, 1u);
5264 }
5265}
5266
5267bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
5268 int16_t MI1FrmOpIdx =
5269 RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
5270 int16_t MI2FrmOpIdx =
5271 RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
5272 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
5273 return false;
5274 MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);
5275 MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);
5276 return FrmOp1.getImm() == FrmOp2.getImm();
5277}
5278
5279std::optional<unsigned>
5280RISCV::getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW) {
5281 switch (Opcode) {
5282 default:
5283 return std::nullopt;
5284
5285 // 11.6. Vector Single-Width Shift Instructions
5286 case RISCV::VSLL_VX:
5287 case RISCV::VSRL_VX:
5288 case RISCV::VSRA_VX:
5289 // 12.4. Vector Single-Width Scaling Shift Instructions
5290 case RISCV::VSSRL_VX:
5291 case RISCV::VSSRA_VX:
5292 // Zvbb
5293 case RISCV::VROL_VX:
5294 case RISCV::VROR_VX:
5295 // Only the low lg2(SEW) bits of the shift-amount value are used.
5296 return Log2SEW;
5297
5298 // 11.7 Vector Narrowing Integer Right Shift Instructions
5299 case RISCV::VNSRL_WX:
5300 case RISCV::VNSRA_WX:
5301 // 12.5. Vector Narrowing Fixed-Point Clip Instructions
5302 case RISCV::VNCLIPU_WX:
5303 case RISCV::VNCLIP_WX:
5304 // Zvbb
5305 case RISCV::VWSLL_VX:
5306 // Only the low lg2(2*SEW) bits of the shift-amount value are used.
5307 return Log2SEW + 1;
5308
5309 // 11.1. Vector Single-Width Integer Add and Subtract
5310 case RISCV::VADD_VX:
5311 case RISCV::VSUB_VX:
5312 case RISCV::VRSUB_VX:
5313 // 11.2. Vector Widening Integer Add/Subtract
5314 case RISCV::VWADDU_VX:
5315 case RISCV::VWSUBU_VX:
5316 case RISCV::VWADD_VX:
5317 case RISCV::VWSUB_VX:
5318 case RISCV::VWADDU_WX:
5319 case RISCV::VWSUBU_WX:
5320 case RISCV::VWADD_WX:
5321 case RISCV::VWSUB_WX:
5322 // 11.4. Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
5323 case RISCV::VADC_VXM:
5324 case RISCV::VADC_VIM:
5325 case RISCV::VMADC_VXM:
5326 case RISCV::VMADC_VIM:
5327 case RISCV::VMADC_VX:
5328 case RISCV::VSBC_VXM:
5329 case RISCV::VMSBC_VXM:
5330 case RISCV::VMSBC_VX:
5331 // 11.5 Vector Bitwise Logical Instructions
5332 case RISCV::VAND_VX:
5333 case RISCV::VOR_VX:
5334 case RISCV::VXOR_VX:
5335 // 11.8. Vector Integer Compare Instructions
5336 case RISCV::VMSEQ_VX:
5337 case RISCV::VMSNE_VX:
5338 case RISCV::VMSLTU_VX:
5339 case RISCV::VMSLT_VX:
5340 case RISCV::VMSLEU_VX:
5341 case RISCV::VMSLE_VX:
5342 case RISCV::VMSGTU_VX:
5343 case RISCV::VMSGT_VX:
5344 // 11.9. Vector Integer Min/Max Instructions
5345 case RISCV::VMINU_VX:
5346 case RISCV::VMIN_VX:
5347 case RISCV::VMAXU_VX:
5348 case RISCV::VMAX_VX:
5349 // 11.10. Vector Single-Width Integer Multiply Instructions
5350 case RISCV::VMUL_VX:
5351 case RISCV::VMULH_VX:
5352 case RISCV::VMULHU_VX:
5353 case RISCV::VMULHSU_VX:
5354 // 11.11. Vector Integer Divide Instructions
5355 case RISCV::VDIVU_VX:
5356 case RISCV::VDIV_VX:
5357 case RISCV::VREMU_VX:
5358 case RISCV::VREM_VX:
5359 // 11.12. Vector Widening Integer Multiply Instructions
5360 case RISCV::VWMUL_VX:
5361 case RISCV::VWMULU_VX:
5362 case RISCV::VWMULSU_VX:
5363 // 11.13. Vector Single-Width Integer Multiply-Add Instructions
5364 case RISCV::VMACC_VX:
5365 case RISCV::VNMSAC_VX:
5366 case RISCV::VMADD_VX:
5367 case RISCV::VNMSUB_VX:
5368 // 11.14. Vector Widening Integer Multiply-Add Instructions
5369 case RISCV::VWMACCU_VX:
5370 case RISCV::VWMACC_VX:
5371 case RISCV::VWMACCSU_VX:
5372 case RISCV::VWMACCUS_VX:
5373 // 11.15. Vector Integer Merge Instructions
5374 case RISCV::VMERGE_VXM:
5375 // 11.16. Vector Integer Move Instructions
5376 case RISCV::VMV_V_X:
5377 // 12.1. Vector Single-Width Saturating Add and Subtract
5378 case RISCV::VSADDU_VX:
5379 case RISCV::VSADD_VX:
5380 case RISCV::VSSUBU_VX:
5381 case RISCV::VSSUB_VX:
5382 // 12.2. Vector Single-Width Averaging Add and Subtract
5383 case RISCV::VAADDU_VX:
5384 case RISCV::VAADD_VX:
5385 case RISCV::VASUBU_VX:
5386 case RISCV::VASUB_VX:
5387 // 12.3. Vector Single-Width Fractional Multiply with Rounding and Saturation
5388 case RISCV::VSMUL_VX:
5389 // 16.1. Integer Scalar Move Instructions
5390 case RISCV::VMV_S_X:
5391 // Zvbb
5392 case RISCV::VANDN_VX:
5393 return 1U << Log2SEW;
5394 }
5395}
5396
5397unsigned RISCV::getRVVMCOpcode(unsigned RVVPseudoOpcode) {
5399 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
5400 if (!RVV)
5401 return 0;
5402 return RVV->BaseInstr;
5403}
5404
5405unsigned RISCV::getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW) {
5406 unsigned DestEEW =
5408 // EEW = 1
5409 if (DestEEW == 0)
5410 return 0;
5411 // EEW = SEW * n
5412 unsigned Scaled = Log2SEW + (DestEEW - 1);
5413 assert(Scaled >= 3 && Scaled <= 6);
5414 return Scaled;
5415}
5416
5417static std::optional<int64_t> getEffectiveImm(const MachineRegisterInfo &MRI,
5418 const MachineOperand &MO) {
5419 assert(MO.isImm() || MO.getReg().isVirtual());
5420 if (MO.isImm())
5421 return MO.getImm();
5422 const MachineInstr *Def = MRI.getVRegDef(MO.getReg());
5423 int64_t Imm;
5424 if (isLoadImm(Def, Imm))
5425 return Imm;
5426 return std::nullopt;
5427}
5428
5429/// Given two VL operands, do we know that LHS <= RHS? Must be used in SSA form.
5431 const MachineOperand &LHS, const MachineOperand &RHS) {
5432 assert((LHS.isImm() || MRI.isSSA()) && (RHS.isImm() || MRI.isSSA()));
5433 if (LHS.isReg() && RHS.isReg() && LHS.getReg().isVirtual() &&
5434 LHS.getReg() == RHS.getReg())
5435 return true;
5436 if (RHS.isImm() && RHS.getImm() == RISCV::VLMaxSentinel)
5437 return true;
5438 if (LHS.isImm() && LHS.getImm() == 0)
5439 return true;
5440 if (LHS.isImm() && LHS.getImm() == RISCV::VLMaxSentinel)
5441 return false;
5442 std::optional<int64_t> LHSImm = getEffectiveImm(MRI, LHS),
5443 RHSImm = getEffectiveImm(MRI, RHS);
5444 if (!LHSImm || !RHSImm)
5445 return false;
5446 return LHSImm <= RHSImm;
5447}
5448
5449namespace {
5450class RISCVPipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
5451 const MachineInstr *LHS;
5452 const MachineInstr *RHS;
5454
5455public:
5456 RISCVPipelinerLoopInfo(const MachineInstr *LHS, const MachineInstr *RHS,
5458 : LHS(LHS), RHS(RHS), Cond(Cond.begin(), Cond.end()) {}
5459
5460 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
5461 // Make the instructions for loop control be placed in stage 0.
5462 // The predecessors of LHS/RHS are considered by the caller.
5463 if (LHS && MI == LHS)
5464 return true;
5465 if (RHS && MI == RHS)
5466 return true;
5467 return false;
5468 }
5469
5470 std::optional<bool> createTripCountGreaterCondition(
5471 int TC, MachineBasicBlock &MBB,
5472 SmallVectorImpl<MachineOperand> &CondParam) override {
5473 // A branch instruction will be inserted as "if (Cond) goto epilogue".
5474 // Cond is normalized for such use.
5475 // The predecessors of the branch are assumed to have already been inserted.
5476 CondParam = Cond;
5477 return {};
5478 }
5479
5480 void setPreheader(MachineBasicBlock *NewPreheader) override {}
5481
5482 void adjustTripCount(int TripCountAdjust) override {}
5483};
5484} // namespace
5485
5486std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
5488 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
5490 if (analyzeBranch(*LoopBB, TBB, FBB, Cond, /*AllowModify=*/false))
5491 return nullptr;
5492
5493 // Infinite loops are not supported
5494 if (TBB == LoopBB && FBB == LoopBB)
5495 return nullptr;
5496
5497 // Must be conditional branch
5498 if (FBB == nullptr)
5499 return nullptr;
5500
5501 assert((TBB == LoopBB || FBB == LoopBB) &&
5502 "The Loop must be a single-basic-block loop");
5503
5504 // Normalization for createTripCountGreaterCondition()
5505 if (TBB == LoopBB)
5507
5508 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
5509 auto FindRegDef = [&MRI](MachineOperand &Op) -> const MachineInstr * {
5510 if (!Op.isReg())
5511 return nullptr;
5512 Register Reg = Op.getReg();
5513 if (!Reg.isVirtual())
5514 return nullptr;
5515 return MRI.getVRegDef(Reg);
5516 };
5517
5518 const MachineInstr *LHS = FindRegDef(Cond[1]);
5519 const MachineInstr *RHS = FindRegDef(Cond[2]);
5520 if (LHS && LHS->isPHI())
5521 return nullptr;
5522 if (RHS && RHS->isPHI())
5523 return nullptr;
5524
5525 return std::make_unique<RISCVPipelinerLoopInfo>(LHS, RHS, Cond);
5526}
5527
5528// FIXME: We should remove this if we have a default generic scheduling model.
5530 unsigned RVVMCOpcode = RISCV::getRVVMCOpcode(Opc);
5531 Opc = RVVMCOpcode ? RVVMCOpcode : Opc;
5532 switch (Opc) {
5533 default:
5534 return false;
5535 // Integer div/rem.
5536 case RISCV::DIV:
5537 case RISCV::DIVW:
5538 case RISCV::DIVU:
5539 case RISCV::DIVUW:
5540 case RISCV::REM:
5541 case RISCV::REMW:
5542 case RISCV::REMU:
5543 case RISCV::REMUW:
5544 // Floating-point div/sqrt.
5545 case RISCV::FDIV_H:
5546 case RISCV::FDIV_S:
5547 case RISCV::FDIV_D:
5548 case RISCV::FDIV_H_INX:
5549 case RISCV::FDIV_S_INX:
5550 case RISCV::FDIV_D_INX:
5551 case RISCV::FDIV_D_IN32X:
5552 case RISCV::FSQRT_H:
5553 case RISCV::FSQRT_S:
5554 case RISCV::FSQRT_D:
5555 case RISCV::FSQRT_H_INX:
5556 case RISCV::FSQRT_S_INX:
5557 case RISCV::FSQRT_D_INX:
5558 case RISCV::FSQRT_D_IN32X:
5559 // Vector integer div/rem
5560 case RISCV::VDIV_VV:
5561 case RISCV::VDIV_VX:
5562 case RISCV::VDIVU_VV:
5563 case RISCV::VDIVU_VX:
5564 case RISCV::VREM_VV:
5565 case RISCV::VREM_VX:
5566 case RISCV::VREMU_VV:
5567 case RISCV::VREMU_VX:
5568 // Vector floating-point div/sqrt.
5569 case RISCV::VFDIV_VV:
5570 case RISCV::VFDIV_VF:
5571 case RISCV::VFRDIV_VF:
5572 case RISCV::VFSQRT_V:
5573 case RISCV::VFRSQRT7_V:
5574 return true;
5575 }
5576}
5577
5578bool RISCVInstrInfo::isVRegCopy(const MachineInstr *MI, unsigned LMul) const {
5579 if (MI->getOpcode() != TargetOpcode::COPY)
5580 return false;
5581 const MachineRegisterInfo &MRI = MI->getMF()->getRegInfo();
5582 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
5583
5584 Register DstReg = MI->getOperand(0).getReg();
5585 const TargetRegisterClass *RC = DstReg.isVirtual()
5586 ? MRI.getRegClass(DstReg)
5587 : TRI->getMinimalPhysRegClass(DstReg);
5588
5590 return false;
5591
5592 if (!LMul)
5593 return true;
5594
5595 // TODO: Perhaps we could distinguish segment register classes (e.g. VRN3M2)
5596 // in the future.
5597 auto [RCLMul, RCFractional] =
5599 return (!RCFractional && LMul == RCLMul) || (RCFractional && LMul == 1);
5600}
5601
5603 if (MI.memoperands_empty())
5604 return false;
5605
5606 MachineMemOperand *MMO = *(MI.memoperands_begin());
5607 if (!MMO->isNonTemporal())
5608 return false;
5609
5610 return true;
5611}
5612
5614 const MachineBasicBlock::iterator &To) {
5615 assert(To == From.getParent()->end() || From.getParent() == To->getParent());
5616 SmallVector<Register> PhysUses, PhysDefs;
5617 for (const MachineOperand &MO : From.all_uses())
5618 if (MO.getReg().isPhysical())
5619 PhysUses.push_back(MO.getReg());
5620 for (const MachineOperand &MO : From.all_defs())
5621 if (MO.getReg().isPhysical())
5622 PhysDefs.push_back(MO.getReg());
5623 bool SawStore = false;
5624 for (auto II = std::next(From.getIterator()); II != To; II++) {
5625 for (Register PhysReg : PhysUses)
5626 if (II->definesRegister(PhysReg, nullptr))
5627 return false;
5628 for (Register PhysReg : PhysDefs)
5629 if (II->definesRegister(PhysReg, nullptr) ||
5630 II->readsRegister(PhysReg, nullptr))
5631 return false;
5632 II->isSafeToMove(SawStore);
5633 if (SawStore)
5634 break;
5635 }
5636 return From.isSafeToMove(SawStore);
5637}
MachineInstrBuilder MachineInstrBuilder & DefMI
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerTailCall
Emit a save, restore, call, and return.
@ MachineOutlinerRegSave
Emit a call and tail-call.
@ MachineOutlinerDefault
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
SmallVector< int16_t, MAX_SRC_OPERANDS_NUM > OperandIndices
uint64_t uint64_t
uint64_t Imm
@ Scaled
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
basic Basic Alias true
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
Module.h This file contains the declarations for the Module class.
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
static int getJumpTableIndexFromLoadAddr(const MachineRegisterInfo &MRI, Register Reg)
static bool isJumpTableLoad(const MachineInstr &MI)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
This file provides utility analysis objects describing memory locations.
uint64_t IntrinsicInst * II
static std::optional< int64_t > getEffectiveImm(const MachineRegisterInfo &MRI, const MachineOperand &MO)
static bool cannotInsertTailCall(const MachineBasicBlock &MBB)
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS_ALT(OP)
#define CASE_FP_WIDEOP_OPCODE_LMULS(OP)
#define CASE_OPERAND_SIMM(NUM)
static std::optional< unsigned > getLMULForRVVWholeLoadStore(unsigned Opcode)
#define CASE_VFMA_CHANGE_OPCODE_VV(OLDOP, NEWOP)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, unsigned Pattern)
std::optional< unsigned > getFoldedOpcode(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, const RISCVSubtarget &ST)
#define RVV_OPC_LMUL_CASE(OPC, INV)
#define CASE_FP_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg, unsigned NumRegs)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static unsigned getAddendOperandIdx(unsigned Pattern)
#define CASE_RVV_OPCODE_UNMASK(OP)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
#define CASE_FP_WIDEOP_OPCODE_LMULS_ALT(OP)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
static bool isMIReadsReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, MCRegister RegNo)
#define OPCODE_LMUL_MASK_CASE(OPC)
#define CASE_OPERAND_UIMM_LSB_ZEROS(BITS, SUFFIX)
static bool isFSUB(unsigned Opc)
#define CASE_VMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
#define CASE_RVV_OPCODE(OP)
#define CASE_VFMA_OPCODE_VV(OP)
MachineOutlinerConstructionID
#define CASE_RVV_OPCODE_WIDEN(OP)
static unsigned getLoadPredicatedOpcode(unsigned Opcode)
static unsigned getSHXADDUWShiftAmount(unsigned Opc)
#define CASE_VMA_OPCODE_LMULS(OP, TYPE)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVVType::VLMUL LMul)
static bool isFMUL(unsigned Opc)
static unsigned getInverseXqcicmOpcode(unsigned Opcode)
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
#define OPCODE_LMUL_CASE(OPC)
#define CASE_OPERAND_UIMM(NUM)
static Register findRegisterToSaveX5To(outliner::Candidate &C, const TargetRegisterInfo &TRI)
static bool canCombineShiftIntoShXAdd(const MachineBasicBlock &MBB, const MachineOperand &MO, unsigned OuterShiftAmt)
Utility routine that checks if.
static bool isCandidatePatchable(const MachineBasicBlock &MBB)
static bool isFADD(unsigned Opc)
static void genShXAddAddShift(MachineInstr &Root, unsigned AddOpIdx, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
static bool isLoadImm(const MachineInstr *MI, int64_t &Imm)
static bool isMIModifiesReg(const MachineInstr &MI, const TargetRegisterInfo *TRI, MCRegister RegNo)
#define CASE_RVV_OPCODE_LMUL(OP, LMUL)
static int getJumpTableIndexFromBase(const MachineRegisterInfo &MRI, Register Reg)
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
static bool getSHXADDPatterns(const MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce)
static unsigned getSHXADDShiftAmount(unsigned Opc)
#define CASE_RVV_OPCODE_MASK(OP)
#define RVV_OPC_LMUL_MASK_CASE(OPC, INV)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII, const RISCVSubtarget &STI)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
SmallVector< MachineOperand, 4 > Cond
int64_t C1
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
int64_t C0
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file declares the machine register scavenger class.
static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, ArrayRef< const MachineOperand * > BaseOps1, const MachineInstr &MI2, ArrayRef< const MachineOperand * > BaseOps2)
This file contains some templates that are useful if you are working with the STL at all.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition Value.cpp:479
This file defines the SmallVector class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
#define LLVM_DEBUG(...)
Definition Debug.h:119
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc=0)
static cl::opt< unsigned > CacheLineSize("cache-line-size", cl::init(0), cl::Hidden, cl::desc("Use this to override the target cache line size when " "specified by the user."))
Value * RHS
Value * LHS
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
const T & front() const
Get the first element.
Definition ArrayRef.h:144
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
static LLVM_ABI DILocation * getMergedLocation(DILocation *LocA, DILocation *LocB)
Attempts to merge LocA and LocB into a single location; see DebugLoc::getMergedLocation for more deta...
bool isBigEndian() const
Definition DataLayout.h:221
A debug info location.
Definition DebugLoc.h:126
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition DenseMap.h:827
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition Function.h:705
LiveInterval - This class represents the liveness of a register, or stack slot.
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
bool hasValue() const
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCInstBuilder & addReg(MCRegister Reg)
Add a new register operand.
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Instances of this class represent a single low-level machine instruction.
Definition MCInst.h:188
Describe properties that are true of each instruction in the target description file.
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition MCInstrDesc.h:88
const uint8_t TSFlags
Configurable target specific flags.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
const FeatureBitset & getFeatureBits() const
Set of metadata that should be preserved when using BuildMI().
MachineInstrBundleIterator< const MachineInstr > const_iterator
MachineInstrBundleIterator< MachineInstr, true > reverse_iterator
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
MachineInstrBundleIterator< MachineInstr > iterator
MachineInstrBundleIterator< const MachineInstr, true > const_reverse_iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
const MachineInstrBuilder & setMemRefs(ArrayRef< MachineMemOperand * > MMOs) const
const MachineInstrBuilder & addUse(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addDef(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a virtual register definition operand.
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
filtered_mop_range all_defs()
Returns an iterator range over all operands that are (explicit or implicit) register defs.
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
LLVM_ABI bool isSafeToMove(bool &SawStore) const
Return true if it is safe to move this instruction.
LLVM_ABI unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const
Return true if the MachineInstr modifies (fully define or partially define) the specified register.
bool mayLoad(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read memory.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
LLVM_ABI bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
LLVM_ABI bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
LLVM_ABI const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
ArrayRef< MachineMemOperand * > memoperands() const
Access to memory operands of the instruction.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
filtered_mop_range all_uses()
Returns an iterator range over all operands that are (explicit or implicit) register uses.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
LLVM_ABI void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
This class contains meta information specific to a module.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
static MachineOperand CreateImm(int64_t Val)
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
LLVM_ABI bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
@ MO_Immediate
Immediate operand.
@ MO_Register
Register operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
const TargetRegisterInfo * getTargetRegisterInfo() const
LLVM_ABI LLVM_READONLY MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
A Module instance is used to store all the information related to an LLVM module.
Definition Module.h:68
MI-level patchpoint operands.
Definition StackMaps.h:77
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given patchpoint should emit.
Definition StackMaps.h:105
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
static bool isSafeToMove(const MachineInstr &From, const MachineBasicBlock::iterator &To)
Return true if moving From down to To won't cause any physical register reads or writes to be clobber...
Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< std::unique_ptr< outliner::OutlinedFunction > > getOutliningCandidateInfo(const MachineModuleInfo &MMI, std::vector< outliner::Candidate > &RepeatedSequenceLocs, unsigned MinRepeats) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags, bool DstRenamable=false, bool DstIsDead=false) const
MachineInstr * emitLdStWithAddr(MachineInstr &MemI, const ExtAddrMode &AM) const override
void mulImm(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, uint32_t Amt, MachineInstr::MIFlag Flag) const
Generate code to multiply the value in DestReg by Amt - handles all the common optimizations for this...
static bool isPairableLdStInstOpc(unsigned Opc)
Return true if pairing the given load or store may be paired with another.
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveIntervals *LIS) const override
RISCVInstrInfo(const RISCVSubtarget &STI)
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, Register VReg, unsigned SubReg=0, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const override
std::unique_ptr< TargetInstrInfo::PipelinerLoopInfo > analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
static bool isLdStSafeToPair(const MachineInstr &LdSt, const TargetRegisterInfo *TRI)
void copyPhysRegVector(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc, const TargetRegisterClass *RegClass) const
bool isReMaterializableImpl(const MachineInstr &MI) const override
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
bool isVRegCopy(const MachineInstr *MI, unsigned LMul=0) const
Return true if MI is a COPY to a vector register of a specific LMul, or any kind of vector registers ...
bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg, const MachineInstr &AddrI, ExtAddrMode &AM) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
unsigned getTailDuplicateSize(CodeGenOptLevel OptLevel) const override
void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const override
bool getMemOperandsWithOffsetWidth(const MachineInstr &MI, SmallVectorImpl< const MachineOperand * > &BaseOps, int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const override
const RISCVSubtarget & STI
Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
bool simplifyInstruction(MachineInstr &MI) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI, MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
MachineTraceStrategy getMachineCombinerTraceStrategy() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
std::optional< RegImmPair > isAddImmediate(const MachineInstr &MI, Register Reg) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
ArrayRef< std::pair< MachineMemOperand::Flags, const char * > > getSerializableMachineMemOperandTargetFlags() const override
MCInst getNop() const override
bool analyzeCandidate(outliner::Candidate &C) const
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, LocationSize &Width) const
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
bool requiresNTLHint(const MachineInstr &MI) const
Return true if the instruction requires an NTL hint to be emitted.
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx) const override
void finalizeInsInstrs(MachineInstr &Root, unsigned &Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
int getJumpTableIndex(const MachineInstr &MI) const override
bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const override
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, Register SrcReg, bool KillSrc, bool RenamableDest=false, bool RenamableSrc=false) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
static RISCVCC::CondCode getCondFromBranchOpc(unsigned Opc)
void buildClearRegister(Register Reg, MachineBasicBlock &MBB, MachineBasicBlock::iterator Iter, DebugLoc &DL, bool AllowSideEffects=true) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
CombinerObjective getCombinerObjective(unsigned Pattern) const override
bool isHighLatencyDef(int Opc) const override
static bool evaluateCondBranch(RISCVCC::CondCode CC, int64_t C0, int64_t C1)
Return the result of the evaluation of C0 CC C1, where CC is a RISCVCC::CondCode.
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const override
bool optimizeCondBranch(MachineInstr &MI) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
static bool isFromLoadImm(const MachineRegisterInfo &MRI, const MachineOperand &Op, int64_t &Imm)
Return true if the operand is a load immediate instruction and sets Imm to the immediate value.
bool shouldClusterMemOps(ArrayRef< const MachineOperand * > BaseOps1, int64_t Offset1, bool OffsetIsScalable1, ArrayRef< const MachineOperand * > BaseOps2, int64_t Offset2, bool OffsetIsScalable2, unsigned ClusterSize, unsigned NumBytes) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
const RISCVOptions & getCLOpts() const
const RISCVRegisterInfo * getRegisterInfo() const override
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isValid() const
Definition Register.h:112
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
SlotIndex - An opaque wrapper around machine indexes.
Definition SlotIndexes.h:70
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
bool erase(PtrType Ptr)
Remove pointer from the set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MI-level stackmap operands.
Definition StackMaps.h:36
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given stackmap should emit.
Definition StackMaps.h:51
MI-level Statepoint operands.
Definition StackMaps.h:159
uint32_t getNumPatchBytes() const
Return the number of patchable bytes the given statepoint should emit.
Definition StackMaps.h:208
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Object returned by analyzeLoopForPipelining.
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool hasReassociableOperands(const MachineInstr &Inst, const MachineBasicBlock *MBB) const
Return true when \P Inst has reassociable operands in the same \P MBB.
virtual void genAlternativeCodeSequence(MachineInstr &Root, unsigned Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isReMaterializableImpl(const MachineInstr &MI) const
For instructions with opcodes for which the M_REMATERIALIZABLE flag is set, this hook lets the target...
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx) const
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void getReassociateOperandIndices(const MachineInstr &Root, unsigned Pattern, std::array< unsigned, 5 > &OperandIndices) const
The returned array encodes the operand index for each parameter because the operands may be commuted;...
virtual CombinerObjective getCombinerObjective(unsigned Pattern) const
Return the objective of a combiner pattern.
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
const MCAsmInfo & getMCAsmInfo() const
Return target specific asm information.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TargetSubtargetInfo - Generic base class for all target subtargets.
virtual const TargetInstrInfo * getInstrInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const =0
Return the target's register information.
Target - Wrapper for Target specific information.
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition TypeSize.h:352
static constexpr TypeSize getZero()
Definition TypeSize.h:358
static constexpr TypeSize getScalable(ScalarTy MinimumSize)
Definition TypeSize.h:355
self_iterator getIterator()
Definition ilist_node.h:123
A raw_ostream that writes to an std::string.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
CondCode getInverseBranchCondition(CondCode)
unsigned getInverseBranchOpcode(unsigned BCC)
unsigned getBrCond(CondCode CC, unsigned SelectOpc=0)
static bool isValidRoundingMode(unsigned Mode)
static StringRef roundingModeToString(RoundingMode RndMode)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool usesMaskPolicy(uint64_t TSFlags)
static bool hasRoundModeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static MCRegister getTailExpandUseRegNo(const FeatureBitset &FeatureBits)
static int getFRMOpNum(const MCInstrDesc &Desc)
static int getVXRMOpNum(const MCInstrDesc &Desc)
static bool hasVecPolicyOp(uint64_t TSFlags)
static bool usesVXRM(uint64_t TSFlags)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isFirstDefTiedToFirstUse(const MCInstrDesc &Desc)
InstSeq generateInstSeq(int64_t Val, const MCSubtargetInfo &STI)
SmallVector< Inst, 8 > InstSeq
Definition RISCVMatInt.h:43
@ OPERAND_UIMMLOG2XLEN_NONZERO
@ OPERAND_UIMM10_LSB00_NONZERO
@ OPERAND_SIMM10_LSB0000_NONZERO
static unsigned getNF(uint8_t TSFlags)
static RISCVVType::VLMUL getLMul(uint8_t TSFlags)
static bool isTailAgnostic(unsigned VType)
LLVM_ABI void printXSfmmVType(unsigned VType, raw_ostream &OS)
LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)
static bool isValidSEW(unsigned SEW)
static bool isValidVType(unsigned VType)
LLVM_ABI void printVType(unsigned VType, raw_ostream &OS)
static bool isValidXSfmmVType(unsigned VTypeI)
static unsigned getSEW(unsigned VType)
static VLMUL getVLMUL(unsigned VType)
static bool isValidRoundingMode(unsigned Mode)
static StringRef roundingModeToString(RoundingMode RndMode)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
bool isValidYBNDSWImm(int64_t Imm)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
unsigned getDestLog2EEW(const MCInstrDesc &Desc, unsigned Log2SEW)
std::optional< unsigned > getVectorLowDemandedScalarBits(unsigned Opcode, unsigned Log2SEW)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(unsigned Opcode)
static constexpr unsigned RVVBitsPerBlock
bool isRVVSpill(const MachineInstr &MI)
static constexpr unsigned RVVBytesPerBlock
static constexpr int64_t VLMaxSentinel
bool isVLKnownLE(const MachineRegisterInfo &MRI, const MachineOperand &LHS, const MachineOperand &RHS)
Given two VL operands, do we know that LHS <= RHS?
bool isVectorCopy(const TargetRegisterInfo *TRI, const MachineInstr &MI)
Return true if MI is a copy that will be lowered to one or more vmvNr.vs.
static bool isValidSMTVTypeMode(unsigned Mode)
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
@ Offset
Definition DWP.cpp:577
@ SHXADD_ADD_SLLI_OP2
@ SHXADD_ADD_SLLI_OP1
MachineTraceStrategy
Strategies for selecting traces.
@ TS_MinInstrCount
Select the trace through a block that has the fewest instructions.
@ TS_Local
Select the trace that contains only the current basic block.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
static const MachineMemOperand::Flags MONontemporalBit1
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
RegState
Flags to represent properties of register accesses.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Kill
The last use of a register.
@ Undef
Value of the register doesn't matter.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
bool isValidAtomicOrdering(Int I)
constexpr RegState getKillRegState(bool B)
static const MachineMemOperand::Flags MONontemporalBit0
constexpr RegState getDeadRegState(bool B)
LLVM_ABI void reportFatalInternalError(Error Err)
Report a fatal error that indicates a bug in LLVM.
Definition Error.cpp:173
Op::Description Desc
unsigned M1(unsigned Val)
Definition VE.h:394
constexpr bool has_single_bit(T Value) noexcept
Definition bit.h:149
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:326
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr RegState getRenamableRegState(bool B)
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr RegState getDefRegState(bool B)
CombinerObjective
The combiner's goal may differ based on which pattern it is attempting to optimize.
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
int isShifted359(T Value, int &Shift)
LLVM_ABI const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=MaxLookupSearchDepth, bool MustPreserveProvenance=false)
This method strips off any GEP address adjustments, pointer casts or llvm.threadlocal....
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
Definition MathExtras.h:183
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2208
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition MathExtras.h:572
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
constexpr bool isShiftedUInt(uint64_t x)
Checks if a unsigned integer is an N bit number shifted left by S.
Definition MathExtras.h:199
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Used to describe addressing mode similar to ExtAddrMode in CodeGenPrepare.
This represents a simple continuous liveness interval for a value.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static bool isRVVRegClass(const TargetRegisterClass *RC)
Used to describe a register and immediate addition.
An individual sequence of instructions to be replaced with a call to an outlined function.
MachineFunction * getMF() const
The information necessary to create an outlined function for some class of candidate.