LLVM 17.0.0git
RISCVInstrInfo.cpp
Go to the documentation of this file.
1//===-- RISCVInstrInfo.cpp - RISCV Instruction Information ------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the RISCV implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "RISCVInstrInfo.h"
15#include "RISCV.h"
17#include "RISCVSubtarget.h"
18#include "RISCVTargetMachine.h"
19#include "llvm/ADT/STLExtras.h"
33
34using namespace llvm;
35
36#define GEN_CHECK_COMPRESS_INSTR
37#include "RISCVGenCompressInstEmitter.inc"
38
39#define GET_INSTRINFO_CTOR_DTOR
40#define GET_INSTRINFO_NAMED_OPS
41#include "RISCVGenInstrInfo.inc"
42
44 "riscv-prefer-whole-register-move", cl::init(false), cl::Hidden,
45 cl::desc("Prefer whole register move for vector registers."));
46
48
49using namespace RISCV;
50
51#define GET_RISCVVPseudosTable_IMPL
52#include "RISCVGenSearchableTables.inc"
53
54} // namespace llvm::RISCVVPseudosTable
55
57 : RISCVGenInstrInfo(RISCV::ADJCALLSTACKDOWN, RISCV::ADJCALLSTACKUP),
58 STI(STI) {}
59
61 if (STI.hasStdExtCOrZca())
62 return MCInstBuilder(RISCV::C_NOP);
63 return MCInstBuilder(RISCV::ADDI)
64 .addReg(RISCV::X0)
65 .addReg(RISCV::X0)
66 .addImm(0);
67}
68
70 int &FrameIndex) const {
71 switch (MI.getOpcode()) {
72 default:
73 return 0;
74 case RISCV::LB:
75 case RISCV::LBU:
76 case RISCV::LH:
77 case RISCV::LHU:
78 case RISCV::FLH:
79 case RISCV::LW:
80 case RISCV::FLW:
81 case RISCV::LWU:
82 case RISCV::LD:
83 case RISCV::FLD:
84 break;
85 }
86
87 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
88 MI.getOperand(2).getImm() == 0) {
89 FrameIndex = MI.getOperand(1).getIndex();
90 return MI.getOperand(0).getReg();
91 }
92
93 return 0;
94}
95
97 int &FrameIndex) const {
98 switch (MI.getOpcode()) {
99 default:
100 return 0;
101 case RISCV::SB:
102 case RISCV::SH:
103 case RISCV::SW:
104 case RISCV::FSH:
105 case RISCV::FSW:
106 case RISCV::SD:
107 case RISCV::FSD:
108 break;
109 }
110
111 if (MI.getOperand(1).isFI() && MI.getOperand(2).isImm() &&
112 MI.getOperand(2).getImm() == 0) {
113 FrameIndex = MI.getOperand(1).getIndex();
114 return MI.getOperand(0).getReg();
115 }
116
117 return 0;
118}
119
120static bool forwardCopyWillClobberTuple(unsigned DstReg, unsigned SrcReg,
121 unsigned NumRegs) {
122 return DstReg > SrcReg && (DstReg - SrcReg) < NumRegs;
123}
124
126 const MachineBasicBlock &MBB,
129 RISCVII::VLMUL LMul) {
131 return false;
132
133 assert(MBBI->getOpcode() == TargetOpcode::COPY &&
134 "Unexpected COPY instruction.");
135 Register SrcReg = MBBI->getOperand(1).getReg();
137
138 bool FoundDef = false;
139 bool FirstVSetVLI = false;
140 unsigned FirstSEW = 0;
141 while (MBBI != MBB.begin()) {
142 --MBBI;
143 if (MBBI->isMetaInstruction())
144 continue;
145
146 if (MBBI->getOpcode() == RISCV::PseudoVSETVLI ||
147 MBBI->getOpcode() == RISCV::PseudoVSETVLIX0 ||
148 MBBI->getOpcode() == RISCV::PseudoVSETIVLI) {
149 // There is a vsetvli between COPY and source define instruction.
150 // vy = def_vop ... (producing instruction)
151 // ...
152 // vsetvli
153 // ...
154 // vx = COPY vy
155 if (!FoundDef) {
156 if (!FirstVSetVLI) {
157 FirstVSetVLI = true;
158 unsigned FirstVType = MBBI->getOperand(2).getImm();
159 RISCVII::VLMUL FirstLMul = RISCVVType::getVLMUL(FirstVType);
160 FirstSEW = RISCVVType::getSEW(FirstVType);
161 // The first encountered vsetvli must have the same lmul as the
162 // register class of COPY.
163 if (FirstLMul != LMul)
164 return false;
165 }
166 // Only permit `vsetvli x0, x0, vtype` between COPY and the source
167 // define instruction.
168 if (MBBI->getOperand(0).getReg() != RISCV::X0)
169 return false;
170 if (MBBI->getOperand(1).isImm())
171 return false;
172 if (MBBI->getOperand(1).getReg() != RISCV::X0)
173 return false;
174 continue;
175 }
176
177 // MBBI is the first vsetvli before the producing instruction.
178 unsigned VType = MBBI->getOperand(2).getImm();
179 // If there is a vsetvli between COPY and the producing instruction.
180 if (FirstVSetVLI) {
181 // If SEW is different, return false.
182 if (RISCVVType::getSEW(VType) != FirstSEW)
183 return false;
184 }
185
186 // If the vsetvli is tail undisturbed, keep the whole register move.
187 if (!RISCVVType::isTailAgnostic(VType))
188 return false;
189
190 // The checking is conservative. We only have register classes for
191 // LMUL = 1/2/4/8. We should be able to convert vmv1r.v to vmv.v.v
192 // for fractional LMUL operations. However, we could not use the vsetvli
193 // lmul for widening operations. The result of widening operation is
194 // 2 x LMUL.
195 return LMul == RISCVVType::getVLMUL(VType);
196 } else if (MBBI->isInlineAsm() || MBBI->isCall()) {
197 return false;
198 } else if (MBBI->getNumDefs()) {
199 // Check all the instructions which will change VL.
200 // For example, vleff has implicit def VL.
201 if (MBBI->modifiesRegister(RISCV::VL))
202 return false;
203
204 // Only converting whole register copies to vmv.v.v when the defining
205 // value appears in the explicit operands.
206 for (const MachineOperand &MO : MBBI->explicit_operands()) {
207 if (!MO.isReg() || !MO.isDef())
208 continue;
209 if (!FoundDef && TRI->isSubRegisterEq(MO.getReg(), SrcReg)) {
210 // We only permit the source of COPY has the same LMUL as the defined
211 // operand.
212 // There are cases we need to keep the whole register copy if the LMUL
213 // is different.
214 // For example,
215 // $x0 = PseudoVSETIVLI 4, 73 // vsetivli zero, 4, e16,m2,ta,m
216 // $v28m4 = PseudoVWADD_VV_M2 $v26m2, $v8m2
217 // # The COPY may be created by vlmul_trunc intrinsic.
218 // $v26m2 = COPY renamable $v28m2, implicit killed $v28m4
219 //
220 // After widening, the valid value will be 4 x e32 elements. If we
221 // convert the COPY to vmv.v.v, it will only copy 4 x e16 elements.
222 // FIXME: The COPY of subregister of Zvlsseg register will not be able
223 // to convert to vmv.v.[v|i] under the constraint.
224 if (MO.getReg() != SrcReg)
225 return false;
226
227 // In widening reduction instructions with LMUL_1 input vector case,
228 // only checking the LMUL is insufficient due to reduction result is
229 // always LMUL_1.
230 // For example,
231 // $x11 = PseudoVSETIVLI 1, 64 // vsetivli a1, 1, e8, m1, ta, mu
232 // $v8m1 = PseudoVWREDSUM_VS_M1 $v26, $v27
233 // $v26 = COPY killed renamable $v8
234 // After widening, The valid value will be 1 x e16 elements. If we
235 // convert the COPY to vmv.v.v, it will only copy 1 x e8 elements.
236 uint64_t TSFlags = MBBI->getDesc().TSFlags;
238 return false;
239
240 // If the producing instruction does not depend on vsetvli, do not
241 // convert COPY to vmv.v.v. For example, VL1R_V or PseudoVRELOAD.
243 return false;
244
245 // Found the definition.
246 FoundDef = true;
247 DefMBBI = MBBI;
248 break;
249 }
250 }
251 }
252 }
253
254 return false;
255}
256
259 const DebugLoc &DL, MCRegister DstReg,
260 MCRegister SrcReg, bool KillSrc) const {
261 if (RISCV::GPRRegClass.contains(DstReg, SrcReg)) {
262 BuildMI(MBB, MBBI, DL, get(RISCV::ADDI), DstReg)
263 .addReg(SrcReg, getKillRegState(KillSrc))
264 .addImm(0);
265 return;
266 }
267
268 // Handle copy from csr
269 if (RISCV::VCSRRegClass.contains(SrcReg) &&
270 RISCV::GPRRegClass.contains(DstReg)) {
272 BuildMI(MBB, MBBI, DL, get(RISCV::CSRRS), DstReg)
273 .addImm(RISCVSysReg::lookupSysRegByName(TRI.getName(SrcReg))->Encoding)
274 .addReg(RISCV::X0);
275 return;
276 }
277
278 // FPR->FPR copies and VR->VR copies.
279 unsigned Opc;
280 bool IsScalableVector = true;
281 unsigned NF = 1;
283 unsigned SubRegIdx = RISCV::sub_vrm1_0;
284 if (RISCV::FPR16RegClass.contains(DstReg, SrcReg)) {
285 if (!STI.hasStdExtZfh() && STI.hasStdExtZfhmin()) {
286 // Zfhmin subset doesn't have FSGNJ_H, replaces FSGNJ_H with FSGNJ_S.
288 DstReg = TRI->getMatchingSuperReg(DstReg, RISCV::sub_16,
289 &RISCV::FPR32RegClass);
290 SrcReg = TRI->getMatchingSuperReg(SrcReg, RISCV::sub_16,
291 &RISCV::FPR32RegClass);
292 Opc = RISCV::FSGNJ_S;
293 } else {
294 Opc = RISCV::FSGNJ_H;
295 }
296 IsScalableVector = false;
297 } else if (RISCV::FPR32RegClass.contains(DstReg, SrcReg)) {
298 Opc = RISCV::FSGNJ_S;
299 IsScalableVector = false;
300 } else if (RISCV::FPR64RegClass.contains(DstReg, SrcReg)) {
301 Opc = RISCV::FSGNJ_D;
302 IsScalableVector = false;
303 } else if (RISCV::VRRegClass.contains(DstReg, SrcReg)) {
304 Opc = RISCV::VMV1R_V;
305 LMul = RISCVII::LMUL_1;
306 } else if (RISCV::VRM2RegClass.contains(DstReg, SrcReg)) {
307 Opc = RISCV::VMV2R_V;
308 LMul = RISCVII::LMUL_2;
309 } else if (RISCV::VRM4RegClass.contains(DstReg, SrcReg)) {
310 Opc = RISCV::VMV4R_V;
311 LMul = RISCVII::LMUL_4;
312 } else if (RISCV::VRM8RegClass.contains(DstReg, SrcReg)) {
313 Opc = RISCV::VMV8R_V;
314 LMul = RISCVII::LMUL_8;
315 } else if (RISCV::VRN2M1RegClass.contains(DstReg, SrcReg)) {
316 Opc = RISCV::VMV1R_V;
317 SubRegIdx = RISCV::sub_vrm1_0;
318 NF = 2;
319 LMul = RISCVII::LMUL_1;
320 } else if (RISCV::VRN2M2RegClass.contains(DstReg, SrcReg)) {
321 Opc = RISCV::VMV2R_V;
322 SubRegIdx = RISCV::sub_vrm2_0;
323 NF = 2;
324 LMul = RISCVII::LMUL_2;
325 } else if (RISCV::VRN2M4RegClass.contains(DstReg, SrcReg)) {
326 Opc = RISCV::VMV4R_V;
327 SubRegIdx = RISCV::sub_vrm4_0;
328 NF = 2;
329 LMul = RISCVII::LMUL_4;
330 } else if (RISCV::VRN3M1RegClass.contains(DstReg, SrcReg)) {
331 Opc = RISCV::VMV1R_V;
332 SubRegIdx = RISCV::sub_vrm1_0;
333 NF = 3;
334 LMul = RISCVII::LMUL_1;
335 } else if (RISCV::VRN3M2RegClass.contains(DstReg, SrcReg)) {
336 Opc = RISCV::VMV2R_V;
337 SubRegIdx = RISCV::sub_vrm2_0;
338 NF = 3;
339 LMul = RISCVII::LMUL_2;
340 } else if (RISCV::VRN4M1RegClass.contains(DstReg, SrcReg)) {
341 Opc = RISCV::VMV1R_V;
342 SubRegIdx = RISCV::sub_vrm1_0;
343 NF = 4;
344 LMul = RISCVII::LMUL_1;
345 } else if (RISCV::VRN4M2RegClass.contains(DstReg, SrcReg)) {
346 Opc = RISCV::VMV2R_V;
347 SubRegIdx = RISCV::sub_vrm2_0;
348 NF = 4;
349 LMul = RISCVII::LMUL_2;
350 } else if (RISCV::VRN5M1RegClass.contains(DstReg, SrcReg)) {
351 Opc = RISCV::VMV1R_V;
352 SubRegIdx = RISCV::sub_vrm1_0;
353 NF = 5;
354 LMul = RISCVII::LMUL_1;
355 } else if (RISCV::VRN6M1RegClass.contains(DstReg, SrcReg)) {
356 Opc = RISCV::VMV1R_V;
357 SubRegIdx = RISCV::sub_vrm1_0;
358 NF = 6;
359 LMul = RISCVII::LMUL_1;
360 } else if (RISCV::VRN7M1RegClass.contains(DstReg, SrcReg)) {
361 Opc = RISCV::VMV1R_V;
362 SubRegIdx = RISCV::sub_vrm1_0;
363 NF = 7;
364 LMul = RISCVII::LMUL_1;
365 } else if (RISCV::VRN8M1RegClass.contains(DstReg, SrcReg)) {
366 Opc = RISCV::VMV1R_V;
367 SubRegIdx = RISCV::sub_vrm1_0;
368 NF = 8;
369 LMul = RISCVII::LMUL_1;
370 } else {
371 llvm_unreachable("Impossible reg-to-reg copy");
372 }
373
374 if (IsScalableVector) {
375 bool UseVMV_V_V = false;
377 unsigned VIOpc;
378 if (isConvertibleToVMV_V_V(STI, MBB, MBBI, DefMBBI, LMul)) {
379 UseVMV_V_V = true;
380 // We only need to handle LMUL = 1/2/4/8 here because we only define
381 // vector register classes for LMUL = 1/2/4/8.
382 switch (LMul) {
383 default:
384 llvm_unreachable("Impossible LMUL for vector register copy.");
385 case RISCVII::LMUL_1:
386 Opc = RISCV::PseudoVMV_V_V_M1;
387 VIOpc = RISCV::PseudoVMV_V_I_M1;
388 break;
389 case RISCVII::LMUL_2:
390 Opc = RISCV::PseudoVMV_V_V_M2;
391 VIOpc = RISCV::PseudoVMV_V_I_M2;
392 break;
393 case RISCVII::LMUL_4:
394 Opc = RISCV::PseudoVMV_V_V_M4;
395 VIOpc = RISCV::PseudoVMV_V_I_M4;
396 break;
397 case RISCVII::LMUL_8:
398 Opc = RISCV::PseudoVMV_V_V_M8;
399 VIOpc = RISCV::PseudoVMV_V_I_M8;
400 break;
401 }
402 }
403
404 bool UseVMV_V_I = false;
405 if (UseVMV_V_V && (DefMBBI->getOpcode() == VIOpc)) {
406 UseVMV_V_I = true;
407 Opc = VIOpc;
408 }
409
410 if (NF == 1) {
411 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc), DstReg);
412 if (UseVMV_V_I)
413 MIB = MIB.add(DefMBBI->getOperand(1));
414 else
415 MIB = MIB.addReg(SrcReg, getKillRegState(KillSrc));
416 if (UseVMV_V_V) {
417 const MCInstrDesc &Desc = DefMBBI->getDesc();
418 MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
419 MIB.add(DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc))); // SEW
420 MIB.addReg(RISCV::VL, RegState::Implicit);
421 MIB.addReg(RISCV::VTYPE, RegState::Implicit);
422 }
423 } else {
425
426 int I = 0, End = NF, Incr = 1;
427 unsigned SrcEncoding = TRI->getEncodingValue(SrcReg);
428 unsigned DstEncoding = TRI->getEncodingValue(DstReg);
429 unsigned LMulVal;
430 bool Fractional;
431 std::tie(LMulVal, Fractional) = RISCVVType::decodeVLMUL(LMul);
432 assert(!Fractional && "It is impossible be fractional lmul here.");
433 if (forwardCopyWillClobberTuple(DstEncoding, SrcEncoding, NF * LMulVal)) {
434 I = NF - 1;
435 End = -1;
436 Incr = -1;
437 }
438
439 for (; I != End; I += Incr) {
440 auto MIB = BuildMI(MBB, MBBI, DL, get(Opc),
441 TRI->getSubReg(DstReg, SubRegIdx + I));
442 if (UseVMV_V_I)
443 MIB = MIB.add(DefMBBI->getOperand(1));
444 else
445 MIB = MIB.addReg(TRI->getSubReg(SrcReg, SubRegIdx + I),
446 getKillRegState(KillSrc));
447 if (UseVMV_V_V) {
448 const MCInstrDesc &Desc = DefMBBI->getDesc();
449 MIB.add(DefMBBI->getOperand(RISCVII::getVLOpNum(Desc))); // AVL
450 MIB.add(DefMBBI->getOperand(RISCVII::getSEWOpNum(Desc))); // SEW
451 MIB.addReg(RISCV::VL, RegState::Implicit);
452 MIB.addReg(RISCV::VTYPE, RegState::Implicit);
453 }
454 }
455 }
456 } else {
457 BuildMI(MBB, MBBI, DL, get(Opc), DstReg)
458 .addReg(SrcReg, getKillRegState(KillSrc))
459 .addReg(SrcReg, getKillRegState(KillSrc));
460 }
461}
462
465 Register SrcReg, bool IsKill, int FI,
466 const TargetRegisterClass *RC,
467 const TargetRegisterInfo *TRI,
468 Register VReg) const {
469 DebugLoc DL;
470 if (I != MBB.end())
471 DL = I->getDebugLoc();
472
474 MachineFrameInfo &MFI = MF->getFrameInfo();
475
476 unsigned Opcode;
477 bool IsScalableVector = true;
478 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
479 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
480 RISCV::SW : RISCV::SD;
481 IsScalableVector = false;
482 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
483 Opcode = RISCV::FSH;
484 IsScalableVector = false;
485 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
486 Opcode = RISCV::FSW;
487 IsScalableVector = false;
488 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
489 Opcode = RISCV::FSD;
490 IsScalableVector = false;
491 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
492 Opcode = RISCV::VS1R_V;
493 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
494 Opcode = RISCV::VS2R_V;
495 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
496 Opcode = RISCV::VS4R_V;
497 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
498 Opcode = RISCV::VS8R_V;
499 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
500 Opcode = RISCV::PseudoVSPILL2_M1;
501 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
502 Opcode = RISCV::PseudoVSPILL2_M2;
503 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
504 Opcode = RISCV::PseudoVSPILL2_M4;
505 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
506 Opcode = RISCV::PseudoVSPILL3_M1;
507 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
508 Opcode = RISCV::PseudoVSPILL3_M2;
509 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
510 Opcode = RISCV::PseudoVSPILL4_M1;
511 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
512 Opcode = RISCV::PseudoVSPILL4_M2;
513 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
514 Opcode = RISCV::PseudoVSPILL5_M1;
515 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
516 Opcode = RISCV::PseudoVSPILL6_M1;
517 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
518 Opcode = RISCV::PseudoVSPILL7_M1;
519 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
520 Opcode = RISCV::PseudoVSPILL8_M1;
521 else
522 llvm_unreachable("Can't store this register to stack slot");
523
524 if (IsScalableVector) {
528
530 BuildMI(MBB, I, DL, get(Opcode))
531 .addReg(SrcReg, getKillRegState(IsKill))
532 .addFrameIndex(FI)
533 .addMemOperand(MMO);
534 } else {
537 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
538
539 BuildMI(MBB, I, DL, get(Opcode))
540 .addReg(SrcReg, getKillRegState(IsKill))
541 .addFrameIndex(FI)
542 .addImm(0)
543 .addMemOperand(MMO);
544 }
545}
546
549 Register DstReg, int FI,
550 const TargetRegisterClass *RC,
551 const TargetRegisterInfo *TRI,
552 Register VReg) const {
553 DebugLoc DL;
554 if (I != MBB.end())
555 DL = I->getDebugLoc();
556
558 MachineFrameInfo &MFI = MF->getFrameInfo();
559
560 unsigned Opcode;
561 bool IsScalableVector = true;
562 if (RISCV::GPRRegClass.hasSubClassEq(RC)) {
563 Opcode = TRI->getRegSizeInBits(RISCV::GPRRegClass) == 32 ?
564 RISCV::LW : RISCV::LD;
565 IsScalableVector = false;
566 } else if (RISCV::FPR16RegClass.hasSubClassEq(RC)) {
567 Opcode = RISCV::FLH;
568 IsScalableVector = false;
569 } else if (RISCV::FPR32RegClass.hasSubClassEq(RC)) {
570 Opcode = RISCV::FLW;
571 IsScalableVector = false;
572 } else if (RISCV::FPR64RegClass.hasSubClassEq(RC)) {
573 Opcode = RISCV::FLD;
574 IsScalableVector = false;
575 } else if (RISCV::VRRegClass.hasSubClassEq(RC)) {
576 Opcode = RISCV::VL1RE8_V;
577 } else if (RISCV::VRM2RegClass.hasSubClassEq(RC)) {
578 Opcode = RISCV::VL2RE8_V;
579 } else if (RISCV::VRM4RegClass.hasSubClassEq(RC)) {
580 Opcode = RISCV::VL4RE8_V;
581 } else if (RISCV::VRM8RegClass.hasSubClassEq(RC)) {
582 Opcode = RISCV::VL8RE8_V;
583 } else if (RISCV::VRN2M1RegClass.hasSubClassEq(RC))
584 Opcode = RISCV::PseudoVRELOAD2_M1;
585 else if (RISCV::VRN2M2RegClass.hasSubClassEq(RC))
586 Opcode = RISCV::PseudoVRELOAD2_M2;
587 else if (RISCV::VRN2M4RegClass.hasSubClassEq(RC))
588 Opcode = RISCV::PseudoVRELOAD2_M4;
589 else if (RISCV::VRN3M1RegClass.hasSubClassEq(RC))
590 Opcode = RISCV::PseudoVRELOAD3_M1;
591 else if (RISCV::VRN3M2RegClass.hasSubClassEq(RC))
592 Opcode = RISCV::PseudoVRELOAD3_M2;
593 else if (RISCV::VRN4M1RegClass.hasSubClassEq(RC))
594 Opcode = RISCV::PseudoVRELOAD4_M1;
595 else if (RISCV::VRN4M2RegClass.hasSubClassEq(RC))
596 Opcode = RISCV::PseudoVRELOAD4_M2;
597 else if (RISCV::VRN5M1RegClass.hasSubClassEq(RC))
598 Opcode = RISCV::PseudoVRELOAD5_M1;
599 else if (RISCV::VRN6M1RegClass.hasSubClassEq(RC))
600 Opcode = RISCV::PseudoVRELOAD6_M1;
601 else if (RISCV::VRN7M1RegClass.hasSubClassEq(RC))
602 Opcode = RISCV::PseudoVRELOAD7_M1;
603 else if (RISCV::VRN8M1RegClass.hasSubClassEq(RC))
604 Opcode = RISCV::PseudoVRELOAD8_M1;
605 else
606 llvm_unreachable("Can't load this register from stack slot");
607
608 if (IsScalableVector) {
612
614 BuildMI(MBB, I, DL, get(Opcode), DstReg)
615 .addFrameIndex(FI)
616 .addMemOperand(MMO);
617 } else {
620 MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
621
622 BuildMI(MBB, I, DL, get(Opcode), DstReg)
623 .addFrameIndex(FI)
624 .addImm(0)
625 .addMemOperand(MMO);
626 }
627}
628
631 MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS,
632 VirtRegMap *VRM) const {
633 const MachineFrameInfo &MFI = MF.getFrameInfo();
634
635 // The below optimizations narrow the load so they are only valid for little
636 // endian.
637 // TODO: Support big endian by adding an offset into the frame object?
638 if (MF.getDataLayout().isBigEndian())
639 return nullptr;
640
641 // Fold load from stack followed by sext.w into lw.
642 // TODO: Fold with sext.b, sext.h, zext.b, zext.h, zext.w?
643 if (Ops.size() != 1 || Ops[0] != 1)
644 return nullptr;
645
646 unsigned LoadOpc;
647 switch (MI.getOpcode()) {
648 default:
649 if (RISCV::isSEXT_W(MI)) {
650 LoadOpc = RISCV::LW;
651 break;
652 }
653 if (RISCV::isZEXT_W(MI)) {
654 LoadOpc = RISCV::LWU;
655 break;
656 }
657 if (RISCV::isZEXT_B(MI)) {
658 LoadOpc = RISCV::LBU;
659 break;
660 }
661 return nullptr;
662 case RISCV::SEXT_H:
663 LoadOpc = RISCV::LH;
664 break;
665 case RISCV::SEXT_B:
666 LoadOpc = RISCV::LB;
667 break;
668 case RISCV::ZEXT_H_RV32:
669 case RISCV::ZEXT_H_RV64:
670 LoadOpc = RISCV::LHU;
671 break;
672 }
673
675 MachinePointerInfo::getFixedStack(MF, FrameIndex),
677 MFI.getObjectAlign(FrameIndex));
678
679 Register DstReg = MI.getOperand(0).getReg();
680 return BuildMI(*MI.getParent(), InsertPt, MI.getDebugLoc(), get(LoadOpc),
681 DstReg)
682 .addFrameIndex(FrameIndex)
683 .addImm(0)
684 .addMemOperand(MMO);
685}
686
689 const DebugLoc &DL, Register DstReg, uint64_t Val,
690 MachineInstr::MIFlag Flag) const {
691 Register SrcReg = RISCV::X0;
692
693 if (!STI.is64Bit() && !isInt<32>(Val))
694 report_fatal_error("Should only materialize 32-bit constants for RV32");
695
697 RISCVMatInt::generateInstSeq(Val, STI.getFeatureBits());
698 assert(!Seq.empty());
699
700 for (RISCVMatInt::Inst &Inst : Seq) {
701 switch (Inst.getOpndKind()) {
702 case RISCVMatInt::Imm:
703 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
704 .addImm(Inst.getImm())
705 .setMIFlag(Flag);
706 break;
708 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
709 .addReg(SrcReg, RegState::Kill)
710 .addReg(RISCV::X0)
711 .setMIFlag(Flag);
712 break;
714 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
715 .addReg(SrcReg, RegState::Kill)
716 .addReg(SrcReg, RegState::Kill)
717 .setMIFlag(Flag);
718 break;
720 BuildMI(MBB, MBBI, DL, get(Inst.getOpcode()), DstReg)
721 .addReg(SrcReg, RegState::Kill)
722 .addImm(Inst.getImm())
723 .setMIFlag(Flag);
724 break;
725 }
726
727 // Only the first instruction has X0 as its source.
728 SrcReg = DstReg;
729 }
730}
731
733 switch (Opc) {
734 default:
736 case RISCV::BEQ:
737 return RISCVCC::COND_EQ;
738 case RISCV::BNE:
739 return RISCVCC::COND_NE;
740 case RISCV::BLT:
741 return RISCVCC::COND_LT;
742 case RISCV::BGE:
743 return RISCVCC::COND_GE;
744 case RISCV::BLTU:
745 return RISCVCC::COND_LTU;
746 case RISCV::BGEU:
747 return RISCVCC::COND_GEU;
748 }
749}
750
751// The contents of values added to Cond are not examined outside of
752// RISCVInstrInfo, giving us flexibility in what to push to it. For RISCV, we
753// push BranchOpcode, Reg1, Reg2.
756 // Block ends with fall-through condbranch.
757 assert(LastInst.getDesc().isConditionalBranch() &&
758 "Unknown conditional branch");
759 Target = LastInst.getOperand(2).getMBB();
760 unsigned CC = getCondFromBranchOpc(LastInst.getOpcode());
762 Cond.push_back(LastInst.getOperand(0));
763 Cond.push_back(LastInst.getOperand(1));
764}
765
767 switch (CC) {
768 default:
769 llvm_unreachable("Unknown condition code!");
770 case RISCVCC::COND_EQ:
771 return get(RISCV::BEQ);
772 case RISCVCC::COND_NE:
773 return get(RISCV::BNE);
774 case RISCVCC::COND_LT:
775 return get(RISCV::BLT);
776 case RISCVCC::COND_GE:
777 return get(RISCV::BGE);
779 return get(RISCV::BLTU);
781 return get(RISCV::BGEU);
782 }
783}
784
786 switch (CC) {
787 default:
788 llvm_unreachable("Unrecognized conditional branch");
789 case RISCVCC::COND_EQ:
790 return RISCVCC::COND_NE;
791 case RISCVCC::COND_NE:
792 return RISCVCC::COND_EQ;
793 case RISCVCC::COND_LT:
794 return RISCVCC::COND_GE;
795 case RISCVCC::COND_GE:
796 return RISCVCC::COND_LT;
798 return RISCVCC::COND_GEU;
800 return RISCVCC::COND_LTU;
801 }
802}
803
806 MachineBasicBlock *&FBB,
808 bool AllowModify) const {
809 TBB = FBB = nullptr;
810 Cond.clear();
811
812 // If the block has no terminators, it just falls into the block after it.
814 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
815 return false;
816
817 // Count the number of terminators and find the first unconditional or
818 // indirect branch.
819 MachineBasicBlock::iterator FirstUncondOrIndirectBr = MBB.end();
820 int NumTerminators = 0;
821 for (auto J = I.getReverse(); J != MBB.rend() && isUnpredicatedTerminator(*J);
822 J++) {
823 NumTerminators++;
824 if (J->getDesc().isUnconditionalBranch() ||
825 J->getDesc().isIndirectBranch()) {
826 FirstUncondOrIndirectBr = J.getReverse();
827 }
828 }
829
830 // If AllowModify is true, we can erase any terminators after
831 // FirstUncondOrIndirectBR.
832 if (AllowModify && FirstUncondOrIndirectBr != MBB.end()) {
833 while (std::next(FirstUncondOrIndirectBr) != MBB.end()) {
834 std::next(FirstUncondOrIndirectBr)->eraseFromParent();
835 NumTerminators--;
836 }
837 I = FirstUncondOrIndirectBr;
838 }
839
840 // We can't handle blocks that end in an indirect branch.
841 if (I->getDesc().isIndirectBranch())
842 return true;
843
844 // We can't handle blocks with more than 2 terminators.
845 if (NumTerminators > 2)
846 return true;
847
848 // Handle a single unconditional branch.
849 if (NumTerminators == 1 && I->getDesc().isUnconditionalBranch()) {
851 return false;
852 }
853
854 // Handle a single conditional branch.
855 if (NumTerminators == 1 && I->getDesc().isConditionalBranch()) {
857 return false;
858 }
859
860 // Handle a conditional branch followed by an unconditional branch.
861 if (NumTerminators == 2 && std::prev(I)->getDesc().isConditionalBranch() &&
862 I->getDesc().isUnconditionalBranch()) {
863 parseCondBranch(*std::prev(I), TBB, Cond);
864 FBB = getBranchDestBlock(*I);
865 return false;
866 }
867
868 // Otherwise, we can't handle this.
869 return true;
870}
871
873 int *BytesRemoved) const {
874 if (BytesRemoved)
875 *BytesRemoved = 0;
877 if (I == MBB.end())
878 return 0;
879
880 if (!I->getDesc().isUnconditionalBranch() &&
881 !I->getDesc().isConditionalBranch())
882 return 0;
883
884 // Remove the branch.
885 if (BytesRemoved)
886 *BytesRemoved += getInstSizeInBytes(*I);
887 I->eraseFromParent();
888
889 I = MBB.end();
890
891 if (I == MBB.begin())
892 return 1;
893 --I;
894 if (!I->getDesc().isConditionalBranch())
895 return 1;
896
897 // Remove the branch.
898 if (BytesRemoved)
899 *BytesRemoved += getInstSizeInBytes(*I);
900 I->eraseFromParent();
901 return 2;
902}
903
904// Inserts a branch into the end of the specific MachineBasicBlock, returning
905// the number of instructions inserted.
908 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
909 if (BytesAdded)
910 *BytesAdded = 0;
911
912 // Shouldn't be a fall through.
913 assert(TBB && "insertBranch must not be told to insert a fallthrough");
914 assert((Cond.size() == 3 || Cond.size() == 0) &&
915 "RISCV branch conditions have two components!");
916
917 // Unconditional branch.
918 if (Cond.empty()) {
919 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(TBB);
920 if (BytesAdded)
921 *BytesAdded += getInstSizeInBytes(MI);
922 return 1;
923 }
924
925 // Either a one or two-way conditional branch.
926 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
927 MachineInstr &CondMI =
928 *BuildMI(&MBB, DL, getBrCond(CC)).add(Cond[1]).add(Cond[2]).addMBB(TBB);
929 if (BytesAdded)
930 *BytesAdded += getInstSizeInBytes(CondMI);
931
932 // One-way conditional branch.
933 if (!FBB)
934 return 1;
935
936 // Two-way conditional branch.
937 MachineInstr &MI = *BuildMI(&MBB, DL, get(RISCV::PseudoBR)).addMBB(FBB);
938 if (BytesAdded)
939 *BytesAdded += getInstSizeInBytes(MI);
940 return 2;
941}
942
944 MachineBasicBlock &DestBB,
945 MachineBasicBlock &RestoreBB,
946 const DebugLoc &DL, int64_t BrOffset,
947 RegScavenger *RS) const {
948 assert(RS && "RegScavenger required for long branching");
949 assert(MBB.empty() &&
950 "new block should be inserted for expanding unconditional branch");
951 assert(MBB.pred_size() == 1);
952 assert(RestoreBB.empty() &&
953 "restore block should be inserted for restoring clobbered registers");
954
959
960 if (!isInt<32>(BrOffset))
962 "Branch offsets outside of the signed 32-bit range not supported");
963
964 // FIXME: A virtual register must be used initially, as the register
965 // scavenger won't work with empty blocks (SIInstrInfo::insertIndirectBranch
966 // uses the same workaround).
967 Register ScratchReg = MRI.createVirtualRegister(&RISCV::GPRRegClass);
968 auto II = MBB.end();
969 // We may also update the jump target to RestoreBB later.
970 MachineInstr &MI = *BuildMI(MBB, II, DL, get(RISCV::PseudoJump))
972 .addMBB(&DestBB, RISCVII::MO_CALL);
973
975 Register TmpGPR =
976 RS->scavengeRegisterBackwards(RISCV::GPRRegClass, MI.getIterator(),
977 /*RestoreAfter=*/false, /*SpAdj=*/0,
978 /*AllowSpill=*/false);
979 if (TmpGPR != RISCV::NoRegister)
980 RS->setRegUsed(TmpGPR);
981 else {
982 // The case when there is no scavenged register needs special handling.
983
984 // Pick s11 because it doesn't make a difference.
985 TmpGPR = RISCV::X27;
986
987 int FrameIndex = RVFI->getBranchRelaxationScratchFrameIndex();
988 if (FrameIndex == -1)
989 report_fatal_error("underestimated function size");
990
991 storeRegToStackSlot(MBB, MI, TmpGPR, /*IsKill=*/true, FrameIndex,
992 &RISCV::GPRRegClass, TRI, Register());
993 TRI->eliminateFrameIndex(std::prev(MI.getIterator()),
994 /*SpAdj=*/0, /*FIOperandNum=*/1);
995
996 MI.getOperand(1).setMBB(&RestoreBB);
997
998 loadRegFromStackSlot(RestoreBB, RestoreBB.end(), TmpGPR, FrameIndex,
999 &RISCV::GPRRegClass, TRI, Register());
1000 TRI->eliminateFrameIndex(RestoreBB.back(),
1001 /*SpAdj=*/0, /*FIOperandNum=*/1);
1002 }
1003
1004 MRI.replaceRegWith(ScratchReg, TmpGPR);
1005 MRI.clearVirtRegs();
1006}
1007
1010 assert((Cond.size() == 3) && "Invalid branch condition!");
1011 auto CC = static_cast<RISCVCC::CondCode>(Cond[0].getImm());
1013 return false;
1014}
1015
1018 assert(MI.getDesc().isBranch() && "Unexpected opcode!");
1019 // The branch target is always the last operand.
1020 int NumOp = MI.getNumExplicitOperands();
1021 return MI.getOperand(NumOp - 1).getMBB();
1022}
1023
1025 int64_t BrOffset) const {
1026 unsigned XLen = STI.getXLen();
1027 // Ideally we could determine the supported branch offset from the
1028 // RISCVII::FormMask, but this can't be used for Pseudo instructions like
1029 // PseudoBR.
1030 switch (BranchOp) {
1031 default:
1032 llvm_unreachable("Unexpected opcode!");
1033 case RISCV::BEQ:
1034 case RISCV::BNE:
1035 case RISCV::BLT:
1036 case RISCV::BGE:
1037 case RISCV::BLTU:
1038 case RISCV::BGEU:
1039 return isIntN(13, BrOffset);
1040 case RISCV::JAL:
1041 case RISCV::PseudoBR:
1042 return isIntN(21, BrOffset);
1043 case RISCV::PseudoJump:
1044 return isIntN(32, SignExtend64(BrOffset + 0x800, XLen));
1045 }
1046}
1047
1048// If the operation has a predicated pseudo instruction, return the pseudo
1049// instruction opcode. Otherwise, return RISCV::INSTRUCTION_LIST_END.
1050// TODO: Support more operations.
1051unsigned getPredicatedOpcode(unsigned Opcode) {
1052 switch (Opcode) {
1053 case RISCV::ADD: return RISCV::PseudoCCADD; break;
1054 case RISCV::SUB: return RISCV::PseudoCCSUB; break;
1055 case RISCV::AND: return RISCV::PseudoCCAND; break;
1056 case RISCV::OR: return RISCV::PseudoCCOR; break;
1057 case RISCV::XOR: return RISCV::PseudoCCXOR; break;
1058
1059 case RISCV::ADDW: return RISCV::PseudoCCADDW; break;
1060 case RISCV::SUBW: return RISCV::PseudoCCSUBW; break;
1061 }
1062
1063 return RISCV::INSTRUCTION_LIST_END;
1064}
1065
1066/// Identify instructions that can be folded into a CCMOV instruction, and
1067/// return the defining instruction.
1069 const MachineRegisterInfo &MRI,
1070 const TargetInstrInfo *TII) {
1071 if (!Reg.isVirtual())
1072 return nullptr;
1073 if (!MRI.hasOneNonDBGUse(Reg))
1074 return nullptr;
1075 MachineInstr *MI = MRI.getVRegDef(Reg);
1076 if (!MI)
1077 return nullptr;
1078 // Check if MI can be predicated and folded into the CCMOV.
1079 if (getPredicatedOpcode(MI->getOpcode()) == RISCV::INSTRUCTION_LIST_END)
1080 return nullptr;
1081 // Check if MI has any other defs or physreg uses.
1082 for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
1083 const MachineOperand &MO = MI->getOperand(i);
1084 // Reject frame index operands, PEI can't handle the predicated pseudos.
1085 if (MO.isFI() || MO.isCPI() || MO.isJTI())
1086 return nullptr;
1087 if (!MO.isReg())
1088 continue;
1089 // MI can't have any tied operands, that would conflict with predication.
1090 if (MO.isTied())
1091 return nullptr;
1092 if (MO.isDef())
1093 return nullptr;
1094 // Allow constant physregs.
1095 if (MO.getReg().isPhysical() && !MRI.isConstantPhysReg(MO.getReg()))
1096 return nullptr;
1097 }
1098 bool DontMoveAcrossStores = true;
1099 if (!MI->isSafeToMove(/* AliasAnalysis = */ nullptr, DontMoveAcrossStores))
1100 return nullptr;
1101 return MI;
1102}
1103
1106 unsigned &TrueOp, unsigned &FalseOp,
1107 bool &Optimizable) const {
1108 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1109 "Unknown select instruction");
1110 // CCMOV operands:
1111 // 0: Def.
1112 // 1: LHS of compare.
1113 // 2: RHS of compare.
1114 // 3: Condition code.
1115 // 4: False use.
1116 // 5: True use.
1117 TrueOp = 5;
1118 FalseOp = 4;
1119 Cond.push_back(MI.getOperand(1));
1120 Cond.push_back(MI.getOperand(2));
1121 Cond.push_back(MI.getOperand(3));
1122 // We can only fold when we support short forward branch opt.
1123 Optimizable = STI.hasShortForwardBranchOpt();
1124 return false;
1125}
1126
1130 bool PreferFalse) const {
1131 assert(MI.getOpcode() == RISCV::PseudoCCMOVGPR &&
1132 "Unknown select instruction");
1133 if (!STI.hasShortForwardBranchOpt())
1134 return nullptr;
1135
1136 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1138 canFoldAsPredicatedOp(MI.getOperand(5).getReg(), MRI, this);
1139 bool Invert = !DefMI;
1140 if (!DefMI)
1141 DefMI = canFoldAsPredicatedOp(MI.getOperand(4).getReg(), MRI, this);
1142 if (!DefMI)
1143 return nullptr;
1144
1145 // Find new register class to use.
1146 MachineOperand FalseReg = MI.getOperand(Invert ? 5 : 4);
1147 Register DestReg = MI.getOperand(0).getReg();
1148 const TargetRegisterClass *PreviousClass = MRI.getRegClass(FalseReg.getReg());
1149 if (!MRI.constrainRegClass(DestReg, PreviousClass))
1150 return nullptr;
1151
1152 unsigned PredOpc = getPredicatedOpcode(DefMI->getOpcode());
1153 assert(PredOpc != RISCV::INSTRUCTION_LIST_END && "Unexpected opcode!");
1154
1155 // Create a new predicated version of DefMI.
1156 MachineInstrBuilder NewMI =
1157 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(PredOpc), DestReg);
1158
1159 // Copy the condition portion.
1160 NewMI.add(MI.getOperand(1));
1161 NewMI.add(MI.getOperand(2));
1162
1163 // Add condition code, inverting if necessary.
1164 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
1165 if (Invert)
1167 NewMI.addImm(CC);
1168
1169 // Copy the false register.
1170 NewMI.add(FalseReg);
1171
1172 // Copy all the DefMI operands.
1173 const MCInstrDesc &DefDesc = DefMI->getDesc();
1174 for (unsigned i = 1, e = DefDesc.getNumOperands(); i != e; ++i)
1175 NewMI.add(DefMI->getOperand(i));
1176
1177 // Update SeenMIs set: register newly created MI and erase removed DefMI.
1178 SeenMIs.insert(NewMI);
1179 SeenMIs.erase(DefMI);
1180
1181 // If MI is inside a loop, and DefMI is outside the loop, then kill flags on
1182 // DefMI would be invalid when tranferred inside the loop. Checking for a
1183 // loop is expensive, but at least remove kill flags if they are in different
1184 // BBs.
1185 if (DefMI->getParent() != MI.getParent())
1186 NewMI->clearKillInfo();
1187
1188 // The caller will erase MI, but not DefMI.
1190 return NewMI;
1191}
1192
1194 if (MI.isMetaInstruction())
1195 return 0;
1196
1197 unsigned Opcode = MI.getOpcode();
1198
1199 if (Opcode == TargetOpcode::INLINEASM ||
1200 Opcode == TargetOpcode::INLINEASM_BR) {
1201 const MachineFunction &MF = *MI.getParent()->getParent();
1202 const auto &TM = static_cast<const RISCVTargetMachine &>(MF.getTarget());
1203 return getInlineAsmLength(MI.getOperand(0).getSymbolName(),
1204 *TM.getMCAsmInfo());
1205 }
1206
1207 if (MI.getParent() && MI.getParent()->getParent()) {
1208 if (isCompressibleInst(MI, STI))
1209 return 2;
1210 }
1211 return get(Opcode).getSize();
1212}
1213
1215 const unsigned Opcode = MI.getOpcode();
1216 switch (Opcode) {
1217 default:
1218 break;
1219 case RISCV::FSGNJ_D:
1220 case RISCV::FSGNJ_S:
1221 case RISCV::FSGNJ_H:
1222 // The canonical floating-point move is fsgnj rd, rs, rs.
1223 return MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1224 MI.getOperand(1).getReg() == MI.getOperand(2).getReg();
1225 case RISCV::ADDI:
1226 case RISCV::ORI:
1227 case RISCV::XORI:
1228 return (MI.getOperand(1).isReg() &&
1229 MI.getOperand(1).getReg() == RISCV::X0) ||
1230 (MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0);
1231 }
1232 return MI.isAsCheapAsAMove();
1233}
1234
1235std::optional<DestSourcePair>
1237 if (MI.isMoveReg())
1238 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1239 switch (MI.getOpcode()) {
1240 default:
1241 break;
1242 case RISCV::ADDI:
1243 // Operand 1 can be a frameindex but callers expect registers
1244 if (MI.getOperand(1).isReg() && MI.getOperand(2).isImm() &&
1245 MI.getOperand(2).getImm() == 0)
1246 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1247 break;
1248 case RISCV::FSGNJ_D:
1249 case RISCV::FSGNJ_S:
1250 case RISCV::FSGNJ_H:
1251 // The canonical floating-point move is fsgnj rd, rs, rs.
1252 if (MI.getOperand(1).isReg() && MI.getOperand(2).isReg() &&
1253 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
1254 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
1255 break;
1256 }
1257 return std::nullopt;
1258}
1259
1261 MachineInstr &OldMI2,
1262 MachineInstr &NewMI1,
1263 MachineInstr &NewMI2) const {
1264 uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
1265 NewMI1.setFlags(IntersectedFlags);
1266 NewMI2.setFlags(IntersectedFlags);
1267}
1268
1271 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
1272 int16_t FrmOpIdx =
1273 RISCV::getNamedOperandIdx(Root.getOpcode(), RISCV::OpName::frm);
1274 if (FrmOpIdx < 0) {
1275 assert(all_of(InsInstrs,
1276 [](MachineInstr *MI) {
1277 return RISCV::getNamedOperandIdx(MI->getOpcode(),
1278 RISCV::OpName::frm) < 0;
1279 }) &&
1280 "New instructions require FRM whereas the old one does not have it");
1281 return;
1282 }
1283
1284 const MachineOperand &FRM = Root.getOperand(FrmOpIdx);
1285 MachineFunction &MF = *Root.getMF();
1286
1287 for (auto *NewMI : InsInstrs) {
1288 assert(static_cast<unsigned>(RISCV::getNamedOperandIdx(
1289 NewMI->getOpcode(), RISCV::OpName::frm)) ==
1290 NewMI->getNumOperands() &&
1291 "Instruction has unexpected number of operands");
1292 MachineInstrBuilder MIB(MF, NewMI);
1293 MIB.add(FRM);
1294 if (FRM.getImm() == RISCVFPRndMode::DYN)
1295 MIB.addUse(RISCV::FRM, RegState::Implicit);
1296 }
1297}
1298
1299static bool isFADD(unsigned Opc) {
1300 switch (Opc) {
1301 default:
1302 return false;
1303 case RISCV::FADD_H:
1304 case RISCV::FADD_S:
1305 case RISCV::FADD_D:
1306 return true;
1307 }
1308}
1309
1310static bool isFSUB(unsigned Opc) {
1311 switch (Opc) {
1312 default:
1313 return false;
1314 case RISCV::FSUB_H:
1315 case RISCV::FSUB_S:
1316 case RISCV::FSUB_D:
1317 return true;
1318 }
1319}
1320
1321static bool isFMUL(unsigned Opc) {
1322 switch (Opc) {
1323 default:
1324 return false;
1325 case RISCV::FMUL_H:
1326 case RISCV::FMUL_S:
1327 case RISCV::FMUL_D:
1328 return true;
1329 }
1330}
1331
1333 bool &Commuted) const {
1334 if (!TargetInstrInfo::hasReassociableSibling(Inst, Commuted))
1335 return false;
1336
1337 const MachineRegisterInfo &MRI = Inst.getMF()->getRegInfo();
1338 unsigned OperandIdx = Commuted ? 2 : 1;
1339 const MachineInstr &Sibling =
1340 *MRI.getVRegDef(Inst.getOperand(OperandIdx).getReg());
1341
1342 int16_t InstFrmOpIdx =
1343 RISCV::getNamedOperandIdx(Inst.getOpcode(), RISCV::OpName::frm);
1344 int16_t SiblingFrmOpIdx =
1345 RISCV::getNamedOperandIdx(Sibling.getOpcode(), RISCV::OpName::frm);
1346
1347 return (InstFrmOpIdx < 0 && SiblingFrmOpIdx < 0) ||
1348 RISCV::hasEqualFRM(Inst, Sibling);
1349}
1350
1352 bool Invert) const {
1353 unsigned Opc = Inst.getOpcode();
1354 if (Invert) {
1355 auto InverseOpcode = getInverseOpcode(Opc);
1356 if (!InverseOpcode)
1357 return false;
1358 Opc = *InverseOpcode;
1359 }
1360
1361 if (isFADD(Opc) || isFMUL(Opc))
1364
1365 switch (Opc) {
1366 default:
1367 return false;
1368 case RISCV::ADD:
1369 case RISCV::ADDW:
1370 case RISCV::AND:
1371 case RISCV::OR:
1372 case RISCV::XOR:
1373 // From RISC-V ISA spec, if both the high and low bits of the same product
1374 // are required, then the recommended code sequence is:
1375 //
1376 // MULH[[S]U] rdh, rs1, rs2
1377 // MUL rdl, rs1, rs2
1378 // (source register specifiers must be in same order and rdh cannot be the
1379 // same as rs1 or rs2)
1380 //
1381 // Microarchitectures can then fuse these into a single multiply operation
1382 // instead of performing two separate multiplies.
1383 // MachineCombiner may reassociate MUL operands and lose the fusion
1384 // opportunity.
1385 case RISCV::MUL:
1386 case RISCV::MULW:
1387 case RISCV::MIN:
1388 case RISCV::MINU:
1389 case RISCV::MAX:
1390 case RISCV::MAXU:
1391 case RISCV::FMIN_H:
1392 case RISCV::FMIN_S:
1393 case RISCV::FMIN_D:
1394 case RISCV::FMAX_H:
1395 case RISCV::FMAX_S:
1396 case RISCV::FMAX_D:
1397 return true;
1398 }
1399
1400 return false;
1401}
1402
1403std::optional<unsigned>
1404RISCVInstrInfo::getInverseOpcode(unsigned Opcode) const {
1405 switch (Opcode) {
1406 default:
1407 return std::nullopt;
1408 case RISCV::FADD_H:
1409 return RISCV::FSUB_H;
1410 case RISCV::FADD_S:
1411 return RISCV::FSUB_S;
1412 case RISCV::FADD_D:
1413 return RISCV::FSUB_D;
1414 case RISCV::FSUB_H:
1415 return RISCV::FADD_H;
1416 case RISCV::FSUB_S:
1417 return RISCV::FADD_S;
1418 case RISCV::FSUB_D:
1419 return RISCV::FADD_D;
1420 case RISCV::ADD:
1421 return RISCV::SUB;
1422 case RISCV::SUB:
1423 return RISCV::ADD;
1424 case RISCV::ADDW:
1425 return RISCV::SUBW;
1426 case RISCV::SUBW:
1427 return RISCV::ADDW;
1428 }
1429}
1430
1432 const MachineOperand &MO,
1433 bool DoRegPressureReduce) {
1434 if (!MO.isReg() || !MO.getReg().isVirtual())
1435 return false;
1436 const MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
1437 MachineInstr *MI = MRI.getVRegDef(MO.getReg());
1438 if (!MI || !isFMUL(MI->getOpcode()))
1439 return false;
1440
1443 return false;
1444
1445 // Try combining even if fmul has more than one use as it eliminates
1446 // dependency between fadd(fsub) and fmul. However, it can extend liveranges
1447 // for fmul operands, so reject the transformation in register pressure
1448 // reduction mode.
1449 if (DoRegPressureReduce && !MRI.hasOneNonDBGUse(MI->getOperand(0).getReg()))
1450 return false;
1451
1452 // Do not combine instructions from different basic blocks.
1453 if (Root.getParent() != MI->getParent())
1454 return false;
1455 return RISCV::hasEqualFRM(Root, *MI);
1456}
1457
1458static bool
1461 bool DoRegPressureReduce) {
1462 unsigned Opc = Root.getOpcode();
1463 bool IsFAdd = isFADD(Opc);
1464 if (!IsFAdd && !isFSUB(Opc))
1465 return false;
1466 bool Added = false;
1467 if (canCombineFPFusedMultiply(Root, Root.getOperand(1),
1468 DoRegPressureReduce)) {
1471 Added = true;
1472 }
1473 if (canCombineFPFusedMultiply(Root, Root.getOperand(2),
1474 DoRegPressureReduce)) {
1477 Added = true;
1478 }
1479 return Added;
1480}
1481
1482static bool getFPPatterns(MachineInstr &Root,
1484 bool DoRegPressureReduce) {
1485 return getFPFusedMultiplyPatterns(Root, Patterns, DoRegPressureReduce);
1486}
1487
1490 bool DoRegPressureReduce) const {
1491
1492 if (getFPPatterns(Root, Patterns, DoRegPressureReduce))
1493 return true;
1494
1495 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
1496 DoRegPressureReduce);
1497}
1498
1499static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc,
1501 switch (RootOpc) {
1502 default:
1503 llvm_unreachable("Unexpected opcode");
1504 case RISCV::FADD_H:
1505 return RISCV::FMADD_H;
1506 case RISCV::FADD_S:
1507 return RISCV::FMADD_S;
1508 case RISCV::FADD_D:
1509 return RISCV::FMADD_D;
1510 case RISCV::FSUB_H:
1511 return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_H
1512 : RISCV::FNMSUB_H;
1513 case RISCV::FSUB_S:
1514 return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_S
1515 : RISCV::FNMSUB_S;
1516 case RISCV::FSUB_D:
1517 return Pattern == MachineCombinerPattern::FMSUB ? RISCV::FMSUB_D
1518 : RISCV::FNMSUB_D;
1519 }
1520}
1521
1523 switch (Pattern) {
1524 default:
1525 llvm_unreachable("Unexpected pattern");
1528 return 2;
1531 return 1;
1532 }
1533}
1534
1539 MachineFunction *MF = Root.getMF();
1542
1543 MachineOperand &Mul1 = Prev.getOperand(1);
1544 MachineOperand &Mul2 = Prev.getOperand(2);
1545 MachineOperand &Dst = Root.getOperand(0);
1547
1548 Register DstReg = Dst.getReg();
1549 unsigned FusedOpc = getFPFusedMultiplyOpcode(Root.getOpcode(), Pattern);
1550 auto IntersectedFlags = Root.getFlags() & Prev.getFlags();
1551 DebugLoc MergedLoc =
1553
1555 BuildMI(*MF, MergedLoc, TII->get(FusedOpc), DstReg)
1556 .addReg(Mul1.getReg(), getKillRegState(Mul1.isKill()))
1557 .addReg(Mul2.getReg(), getKillRegState(Mul2.isKill()))
1558 .addReg(Addend.getReg(), getKillRegState(Addend.isKill()))
1559 .setMIFlags(IntersectedFlags);
1560
1561 // Mul operands are not killed anymore.
1562 Mul1.setIsKill(false);
1563 Mul2.setIsKill(false);
1564
1565 InsInstrs.push_back(MIB);
1566 if (MRI.hasOneNonDBGUse(Prev.getOperand(0).getReg()))
1567 DelInstrs.push_back(&Prev);
1568 DelInstrs.push_back(&Root);
1569}
1570
1575 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
1577 switch (Pattern) {
1578 default:
1580 DelInstrs, InstrIdxForVirtReg);
1581 return;
1584 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(1).getReg());
1585 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
1586 return;
1587 }
1590 MachineInstr &Prev = *MRI.getVRegDef(Root.getOperand(2).getReg());
1591 combineFPFusedMultiply(Root, Prev, Pattern, InsInstrs, DelInstrs);
1592 return;
1593 }
1594 }
1595}
1596
1598 StringRef &ErrInfo) const {
1599 MCInstrDesc const &Desc = MI.getDesc();
1600
1601 for (auto &OI : enumerate(Desc.operands())) {
1602 unsigned OpType = OI.value().OperandType;
1603 if (OpType >= RISCVOp::OPERAND_FIRST_RISCV_IMM &&
1605 const MachineOperand &MO = MI.getOperand(OI.index());
1606 if (MO.isImm()) {
1607 int64_t Imm = MO.getImm();
1608 bool Ok;
1609 switch (OpType) {
1610 default:
1611 llvm_unreachable("Unexpected operand type");
1612
1613 // clang-format off
1614#define CASE_OPERAND_UIMM(NUM) \
1615 case RISCVOp::OPERAND_UIMM##NUM: \
1616 Ok = isUInt<NUM>(Imm); \
1617 break;
1625 // clang-format on
1627 Ok = isShiftedUInt<1, 1>(Imm);
1628 break;
1630 Ok = isShiftedUInt<5, 2>(Imm);
1631 break;
1633 Ok = isShiftedUInt<6, 2>(Imm);
1634 break;
1636 Ok = isShiftedUInt<5, 3>(Imm);
1637 break;
1639 Ok = isShiftedInt<6, 4>(Imm) && (Imm != 0);
1640 break;
1642 Ok = Imm == 0;
1643 break;
1645 Ok = isInt<5>(Imm);
1646 break;
1648 Ok = (isInt<5>(Imm) && Imm != -16) || Imm == 16;
1649 break;
1651 Ok = isInt<6>(Imm);
1652 break;
1654 Ok = Imm != 0 && isInt<6>(Imm);
1655 break;
1657 Ok = isUInt<10>(Imm);
1658 break;
1660 Ok = isUInt<11>(Imm);
1661 break;
1663 Ok = isInt<12>(Imm);
1664 break;
1666 Ok = isShiftedInt<7, 5>(Imm);
1667 break;
1669 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
1670 break;
1672 Ok = STI.is64Bit() ? isUInt<6>(Imm) : isUInt<5>(Imm);
1673 Ok = Ok && Imm != 0;
1674 break;
1676 Ok = STI.is64Bit() ? isUInt<5>(Imm) : isUInt<4>(Imm);
1677 break;
1679 Ok = Imm >= 0 && Imm <= 10;
1680 break;
1681 }
1682 if (!Ok) {
1683 ErrInfo = "Invalid immediate";
1684 return false;
1685 }
1686 }
1687 }
1688 }
1689
1690 const uint64_t TSFlags = Desc.TSFlags;
1692 unsigned OpIdx = RISCVII::getMergeOpNum(Desc);
1693 if (MI.findTiedOperandIdx(0) != OpIdx) {
1694 ErrInfo = "Merge op improperly tied";
1695 return false;
1696 }
1697 }
1699 const MachineOperand &Op = MI.getOperand(RISCVII::getVLOpNum(Desc));
1700 if (!Op.isImm() && !Op.isReg()) {
1701 ErrInfo = "Invalid operand type for VL operand";
1702 return false;
1703 }
1704 if (Op.isReg() && Op.getReg() != RISCV::NoRegister) {
1705 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
1706 auto *RC = MRI.getRegClass(Op.getReg());
1707 if (!RISCV::GPRRegClass.hasSubClassEq(RC)) {
1708 ErrInfo = "Invalid register class for VL operand";
1709 return false;
1710 }
1711 }
1712 if (!RISCVII::hasSEWOp(TSFlags)) {
1713 ErrInfo = "VL operand w/o SEW operand?";
1714 return false;
1715 }
1716 }
1718 unsigned OpIdx = RISCVII::getSEWOpNum(Desc);
1719 uint64_t Log2SEW = MI.getOperand(OpIdx).getImm();
1720 if (Log2SEW > 31) {
1721 ErrInfo = "Unexpected SEW value";
1722 return false;
1723 }
1724 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
1726 ErrInfo = "Unexpected SEW value";
1727 return false;
1728 }
1729 }
1731 unsigned OpIdx = RISCVII::getVecPolicyOpNum(Desc);
1732 uint64_t Policy = MI.getOperand(OpIdx).getImm();
1734 ErrInfo = "Invalid Policy Value";
1735 return false;
1736 }
1737 if (!RISCVII::hasVLOp(TSFlags)) {
1738 ErrInfo = "policy operand w/o VL operand?";
1739 return false;
1740 }
1741
1742 // VecPolicy operands can only exist on instructions with passthru/merge
1743 // arguments. Note that not all arguments with passthru have vec policy
1744 // operands- some instructions have implicit policies.
1745 unsigned UseOpIdx;
1746 if (!MI.isRegTiedToUseOperand(0, &UseOpIdx)) {
1747 ErrInfo = "policy operand w/o tied operand?";
1748 return false;
1749 }
1750 }
1751
1752 return true;
1753}
1754
1755// Return true if get the base operand, byte offset of an instruction and the
1756// memory width. Width is the size of memory that is being loaded/stored.
1758 const MachineInstr &LdSt, const MachineOperand *&BaseReg, int64_t &Offset,
1759 unsigned &Width, const TargetRegisterInfo *TRI) const {
1760 if (!LdSt.mayLoadOrStore())
1761 return false;
1762
1763 // Here we assume the standard RISC-V ISA, which uses a base+offset
1764 // addressing mode. You'll need to relax these conditions to support custom
1765 // load/stores instructions.
1766 if (LdSt.getNumExplicitOperands() != 3)
1767 return false;
1768 if (!LdSt.getOperand(1).isReg() || !LdSt.getOperand(2).isImm())
1769 return false;
1770
1771 if (!LdSt.hasOneMemOperand())
1772 return false;
1773
1774 Width = (*LdSt.memoperands_begin())->getSize();
1775 BaseReg = &LdSt.getOperand(1);
1776 Offset = LdSt.getOperand(2).getImm();
1777 return true;
1778}
1779
1781 const MachineInstr &MIa, const MachineInstr &MIb) const {
1782 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1783 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1784
1787 return false;
1788
1789 // Retrieve the base register, offset from the base register and width. Width
1790 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4). If
1791 // base registers are identical, and the offset of a lower memory access +
1792 // the width doesn't overlap the offset of a higher memory access,
1793 // then the memory accesses are different.
1795 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1796 int64_t OffsetA = 0, OffsetB = 0;
1797 unsigned int WidthA = 0, WidthB = 0;
1798 if (getMemOperandWithOffsetWidth(MIa, BaseOpA, OffsetA, WidthA, TRI) &&
1799 getMemOperandWithOffsetWidth(MIb, BaseOpB, OffsetB, WidthB, TRI)) {
1800 if (BaseOpA->isIdenticalTo(*BaseOpB)) {
1801 int LowOffset = std::min(OffsetA, OffsetB);
1802 int HighOffset = std::max(OffsetA, OffsetB);
1803 int LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1804 if (LowOffset + LowWidth <= HighOffset)
1805 return true;
1806 }
1807 }
1808 return false;
1809}
1810
1811std::pair<unsigned, unsigned>
1813 const unsigned Mask = RISCVII::MO_DIRECT_FLAG_MASK;
1814 return std::make_pair(TF & Mask, TF & ~Mask);
1815}
1816
1819 using namespace RISCVII;
1820 static const std::pair<unsigned, const char *> TargetFlags[] = {
1821 {MO_CALL, "riscv-call"},
1822 {MO_PLT, "riscv-plt"},
1823 {MO_LO, "riscv-lo"},
1824 {MO_HI, "riscv-hi"},
1825 {MO_PCREL_LO, "riscv-pcrel-lo"},
1826 {MO_PCREL_HI, "riscv-pcrel-hi"},
1827 {MO_GOT_HI, "riscv-got-hi"},
1828 {MO_TPREL_LO, "riscv-tprel-lo"},
1829 {MO_TPREL_HI, "riscv-tprel-hi"},
1830 {MO_TPREL_ADD, "riscv-tprel-add"},
1831 {MO_TLS_GOT_HI, "riscv-tls-got-hi"},
1832 {MO_TLS_GD_HI, "riscv-tls-gd-hi"}};
1833 return ArrayRef(TargetFlags);
1834}
1836 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
1837 const Function &F = MF.getFunction();
1838
1839 // Can F be deduplicated by the linker? If it can, don't outline from it.
1840 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
1841 return false;
1842
1843 // Don't outline from functions with section markings; the program could
1844 // expect that all the code is in the named section.
1845 if (F.hasSection())
1846 return false;
1847
1848 // It's safe to outline from MF.
1849 return true;
1850}
1851
1853 unsigned &Flags) const {
1854 // More accurate safety checking is done in getOutliningCandidateInfo.
1856}
1857
1858// Enum values indicating how an outlined call should be constructed.
1862
1864 MachineFunction &MF) const {
1865 return MF.getFunction().hasMinSize();
1866}
1867
1869 std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
1870
1871 // First we need to filter out candidates where the X5 register (IE t0) can't
1872 // be used to setup the function call.
1873 auto CannotInsertCall = [](outliner::Candidate &C) {
1874 const TargetRegisterInfo *TRI = C.getMF()->getSubtarget().getRegisterInfo();
1875 return !C.isAvailableAcrossAndOutOfSeq(RISCV::X5, *TRI);
1876 };
1877
1878 llvm::erase_if(RepeatedSequenceLocs, CannotInsertCall);
1879
1880 // If the sequence doesn't have enough candidates left, then we're done.
1881 if (RepeatedSequenceLocs.size() < 2)
1883
1884 unsigned SequenceSize = 0;
1885
1886 auto I = RepeatedSequenceLocs[0].front();
1887 auto E = std::next(RepeatedSequenceLocs[0].back());
1888 for (; I != E; ++I)
1889 SequenceSize += getInstSizeInBytes(*I);
1890
1891 // call t0, function = 8 bytes.
1892 unsigned CallOverhead = 8;
1893 for (auto &C : RepeatedSequenceLocs)
1894 C.setCallInfo(MachineOutlinerDefault, CallOverhead);
1895
1896 // jr t0 = 4 bytes, 2 bytes if compressed instructions are enabled.
1897 unsigned FrameOverhead = 4;
1898 if (RepeatedSequenceLocs[0]
1899 .getMF()
1900 ->getSubtarget<RISCVSubtarget>()
1901 .hasStdExtCOrZca())
1902 FrameOverhead = 2;
1903
1904 return outliner::OutlinedFunction(RepeatedSequenceLocs, SequenceSize,
1905 FrameOverhead, MachineOutlinerDefault);
1906}
1907
1910 unsigned Flags) const {
1911 MachineInstr &MI = *MBBI;
1912 MachineBasicBlock *MBB = MI.getParent();
1913 const TargetRegisterInfo *TRI =
1915 const auto &F = MI.getMF()->getFunction();
1916
1917 // Positions generally can't safely be outlined.
1918 if (MI.isPosition()) {
1919 // We can manually strip out CFI instructions later.
1920 if (MI.isCFIInstruction())
1921 // If current function has exception handling code, we can't outline &
1922 // strip these CFI instructions since it may break .eh_frame section
1923 // needed in unwinding.
1924 return F.needsUnwindTableEntry() ? outliner::InstrType::Illegal
1926
1928 }
1929
1930 // Don't trust the user to write safe inline assembly.
1931 if (MI.isInlineAsm())
1933
1934 // We can't outline branches to other basic blocks.
1935 if (MI.isTerminator() && !MBB->succ_empty())
1937
1938 // We need support for tail calls to outlined functions before return
1939 // statements can be allowed.
1940 if (MI.isReturn())
1942
1943 // Don't allow modifying the X5 register which we use for return addresses for
1944 // these outlined functions.
1945 if (MI.modifiesRegister(RISCV::X5, TRI) ||
1946 MI.getDesc().hasImplicitDefOfPhysReg(RISCV::X5))
1948
1949 // Make sure the operands don't reference something unsafe.
1950 for (const auto &MO : MI.operands()) {
1951 if (MO.isMBB() || MO.isBlockAddress() || MO.isCPI() || MO.isJTI())
1953
1954 // pcrel-hi and pcrel-lo can't put in separate sections, filter that out
1955 // if any possible.
1956 if (MO.getTargetFlags() == RISCVII::MO_PCREL_LO &&
1957 (MI.getMF()->getTarget().getFunctionSections() || F.hasComdat() ||
1958 F.hasSection()))
1960 }
1961
1962 // Don't allow instructions which won't be materialized to impact outlining
1963 // analysis.
1964 if (MI.isMetaInstruction())
1966
1968}
1969
1972 const outliner::OutlinedFunction &OF) const {
1973
1974 // Strip out any CFI instructions
1975 bool Changed = true;
1976 while (Changed) {
1977 Changed = false;
1978 auto I = MBB.begin();
1979 auto E = MBB.end();
1980 for (; I != E; ++I) {
1981 if (I->isCFIInstruction()) {
1982 I->removeFromParent();
1983 Changed = true;
1984 break;
1985 }
1986 }
1987 }
1988
1989 MBB.addLiveIn(RISCV::X5);
1990
1991 // Add in a return instruction to the end of the outlined frame.
1992 MBB.insert(MBB.end(), BuildMI(MF, DebugLoc(), get(RISCV::JALR))
1993 .addReg(RISCV::X0, RegState::Define)
1994 .addReg(RISCV::X5)
1995 .addImm(0));
1996}
1997
2001
2002 // Add in a call instruction to the outlined function at the given location.
2003 It = MBB.insert(It,
2004 BuildMI(MF, DebugLoc(), get(RISCV::PseudoCALLReg), RISCV::X5)
2005 .addGlobalAddress(M.getNamedValue(MF.getName()), 0,
2007 return It;
2008}
2009
2010// MIR printer helper function to annotate Operands with a comment.
2012 const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx,
2013 const TargetRegisterInfo *TRI) const {
2014 // Print a generic comment for this operand if there is one.
2015 std::string GenericComment =
2017 if (!GenericComment.empty())
2018 return GenericComment;
2019
2020 // If not, we must have an immediate operand.
2021 if (!Op.isImm())
2022 return std::string();
2023
2024 std::string Comment;
2025 raw_string_ostream OS(Comment);
2026
2027 uint64_t TSFlags = MI.getDesc().TSFlags;
2028
2029 // Print the full VType operand of vsetvli/vsetivli instructions, and the SEW
2030 // operand of vector codegen pseudos.
2031 if ((MI.getOpcode() == RISCV::VSETVLI || MI.getOpcode() == RISCV::VSETIVLI ||
2032 MI.getOpcode() == RISCV::PseudoVSETVLI ||
2033 MI.getOpcode() == RISCV::PseudoVSETIVLI ||
2034 MI.getOpcode() == RISCV::PseudoVSETVLIX0) &&
2035 OpIdx == 2) {
2036 unsigned Imm = MI.getOperand(OpIdx).getImm();
2037 RISCVVType::printVType(Imm, OS);
2038 } else if (RISCVII::hasSEWOp(TSFlags) &&
2039 OpIdx == RISCVII::getSEWOpNum(MI.getDesc())) {
2040 unsigned Log2SEW = MI.getOperand(OpIdx).getImm();
2041 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
2042 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
2043 OS << "e" << SEW;
2044 } else if (RISCVII::hasVecPolicyOp(TSFlags) &&
2045 OpIdx == RISCVII::getVecPolicyOpNum(MI.getDesc())) {
2046 unsigned Policy = MI.getOperand(OpIdx).getImm();
2048 "Invalid Policy Value");
2049 OS << (Policy & RISCVII::TAIL_AGNOSTIC ? "ta" : "tu") << ", "
2050 << (Policy & RISCVII::MASK_AGNOSTIC ? "ma" : "mu");
2051 }
2052
2053 OS.flush();
2054 return Comment;
2055}
2056
2057// clang-format off
2058#define CASE_VFMA_OPCODE_COMMON(OP, TYPE, LMUL) \
2059 RISCV::PseudoV##OP##_##TYPE##_##LMUL
2060
2061#define CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE) \
2062 CASE_VFMA_OPCODE_COMMON(OP, TYPE, M1): \
2063 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M2): \
2064 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M4): \
2065 case CASE_VFMA_OPCODE_COMMON(OP, TYPE, M8)
2066
2067#define CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE) \
2068 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF2): \
2069 case CASE_VFMA_OPCODE_LMULS_M1(OP, TYPE)
2070
2071#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE) \
2072 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF4): \
2073 case CASE_VFMA_OPCODE_LMULS_MF2(OP, TYPE)
2074
2075#define CASE_VFMA_OPCODE_LMULS(OP, TYPE) \
2076 CASE_VFMA_OPCODE_COMMON(OP, TYPE, MF8): \
2077 case CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)
2078
2079#define CASE_VFMA_SPLATS(OP) \
2080 CASE_VFMA_OPCODE_LMULS_MF4(OP, VF16): \
2081 case CASE_VFMA_OPCODE_LMULS_MF2(OP, VF32): \
2082 case CASE_VFMA_OPCODE_LMULS_M1(OP, VF64)
2083// clang-format on
2084
2086 unsigned &SrcOpIdx1,
2087 unsigned &SrcOpIdx2) const {
2088 const MCInstrDesc &Desc = MI.getDesc();
2089 if (!Desc.isCommutable())
2090 return false;
2091
2092 switch (MI.getOpcode()) {
2093 case RISCV::PseudoCCMOVGPR:
2094 // Operands 4 and 5 are commutable.
2095 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 4, 5);
2096 case CASE_VFMA_SPLATS(FMADD):
2097 case CASE_VFMA_SPLATS(FMSUB):
2098 case CASE_VFMA_SPLATS(FMACC):
2099 case CASE_VFMA_SPLATS(FMSAC):
2100 case CASE_VFMA_SPLATS(FNMADD):
2102 case CASE_VFMA_SPLATS(FNMACC):
2103 case CASE_VFMA_SPLATS(FNMSAC):
2104 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
2105 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
2106 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
2107 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
2108 case CASE_VFMA_OPCODE_LMULS(MADD, VX):
2109 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
2110 case CASE_VFMA_OPCODE_LMULS(MACC, VX):
2111 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
2112 case CASE_VFMA_OPCODE_LMULS(MACC, VV):
2113 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
2114 // If the tail policy is undisturbed we can't commute.
2115 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2116 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2117 return false;
2118
2119 // For these instructions we can only swap operand 1 and operand 3 by
2120 // changing the opcode.
2121 unsigned CommutableOpIdx1 = 1;
2122 unsigned CommutableOpIdx2 = 3;
2123 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2124 CommutableOpIdx2))
2125 return false;
2126 return true;
2127 }
2128 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
2130 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
2132 case CASE_VFMA_OPCODE_LMULS(MADD, VV):
2133 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
2134 // If the tail policy is undisturbed we can't commute.
2135 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags));
2136 if ((MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 1) == 0)
2137 return false;
2138
2139 // For these instructions we have more freedom. We can commute with the
2140 // other multiplicand or with the addend/subtrahend/minuend.
2141
2142 // Any fixed operand must be from source 1, 2 or 3.
2143 if (SrcOpIdx1 != CommuteAnyOperandIndex && SrcOpIdx1 > 3)
2144 return false;
2145 if (SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx2 > 3)
2146 return false;
2147
2148 // It both ops are fixed one must be the tied source.
2149 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2150 SrcOpIdx2 != CommuteAnyOperandIndex && SrcOpIdx1 != 1 && SrcOpIdx2 != 1)
2151 return false;
2152
2153 // Look for two different register operands assumed to be commutable
2154 // regardless of the FMA opcode. The FMA opcode is adjusted later if
2155 // needed.
2156 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2157 SrcOpIdx2 == CommuteAnyOperandIndex) {
2158 // At least one of operands to be commuted is not specified and
2159 // this method is free to choose appropriate commutable operands.
2160 unsigned CommutableOpIdx1 = SrcOpIdx1;
2161 if (SrcOpIdx1 == SrcOpIdx2) {
2162 // Both of operands are not fixed. Set one of commutable
2163 // operands to the tied source.
2164 CommutableOpIdx1 = 1;
2165 } else if (SrcOpIdx1 == CommuteAnyOperandIndex) {
2166 // Only one of the operands is not fixed.
2167 CommutableOpIdx1 = SrcOpIdx2;
2168 }
2169
2170 // CommutableOpIdx1 is well defined now. Let's choose another commutable
2171 // operand and assign its index to CommutableOpIdx2.
2172 unsigned CommutableOpIdx2;
2173 if (CommutableOpIdx1 != 1) {
2174 // If we haven't already used the tied source, we must use it now.
2175 CommutableOpIdx2 = 1;
2176 } else {
2177 Register Op1Reg = MI.getOperand(CommutableOpIdx1).getReg();
2178
2179 // The commuted operands should have different registers.
2180 // Otherwise, the commute transformation does not change anything and
2181 // is useless. We use this as a hint to make our decision.
2182 if (Op1Reg != MI.getOperand(2).getReg())
2183 CommutableOpIdx2 = 2;
2184 else
2185 CommutableOpIdx2 = 3;
2186 }
2187
2188 // Assign the found pair of commutable indices to SrcOpIdx1 and
2189 // SrcOpIdx2 to return those values.
2190 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2191 CommutableOpIdx2))
2192 return false;
2193 }
2194
2195 return true;
2196 }
2197 }
2198
2199 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2200}
2201
2202#define CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, LMUL) \
2203 case RISCV::PseudoV##OLDOP##_##TYPE##_##LMUL: \
2204 Opc = RISCV::PseudoV##NEWOP##_##TYPE##_##LMUL; \
2205 break;
2206
2207#define CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE) \
2208 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M1) \
2209 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M2) \
2210 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M4) \
2211 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, M8)
2212
2213#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE) \
2214 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF2) \
2215 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, TYPE)
2216
2217#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE) \
2218 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF4) \
2219 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, TYPE)
2220
2221#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE) \
2222 CASE_VFMA_CHANGE_OPCODE_COMMON(OLDOP, NEWOP, TYPE, MF8) \
2223 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
2224
2225#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP) \
2226 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, VF16) \
2227 CASE_VFMA_CHANGE_OPCODE_LMULS_MF2(OLDOP, NEWOP, VF32) \
2228 CASE_VFMA_CHANGE_OPCODE_LMULS_M1(OLDOP, NEWOP, VF64)
2229
2231 bool NewMI,
2232 unsigned OpIdx1,
2233 unsigned OpIdx2) const {
2234 auto cloneIfNew = [NewMI](MachineInstr &MI) -> MachineInstr & {
2235 if (NewMI)
2236 return *MI.getParent()->getParent()->CloneMachineInstr(&MI);
2237 return MI;
2238 };
2239
2240 switch (MI.getOpcode()) {
2241 case RISCV::PseudoCCMOVGPR: {
2242 // CCMOV can be commuted by inverting the condition.
2243 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
2245 auto &WorkingMI = cloneIfNew(MI);
2246 WorkingMI.getOperand(3).setImm(CC);
2247 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI*/ false,
2248 OpIdx1, OpIdx2);
2249 }
2250 case CASE_VFMA_SPLATS(FMACC):
2251 case CASE_VFMA_SPLATS(FMADD):
2252 case CASE_VFMA_SPLATS(FMSAC):
2253 case CASE_VFMA_SPLATS(FMSUB):
2254 case CASE_VFMA_SPLATS(FNMACC):
2255 case CASE_VFMA_SPLATS(FNMADD):
2256 case CASE_VFMA_SPLATS(FNMSAC):
2258 case CASE_VFMA_OPCODE_LMULS_MF4(FMACC, VV):
2259 case CASE_VFMA_OPCODE_LMULS_MF4(FMSAC, VV):
2260 case CASE_VFMA_OPCODE_LMULS_MF4(FNMACC, VV):
2261 case CASE_VFMA_OPCODE_LMULS_MF4(FNMSAC, VV):
2262 case CASE_VFMA_OPCODE_LMULS(MADD, VX):
2263 case CASE_VFMA_OPCODE_LMULS(NMSUB, VX):
2264 case CASE_VFMA_OPCODE_LMULS(MACC, VX):
2265 case CASE_VFMA_OPCODE_LMULS(NMSAC, VX):
2266 case CASE_VFMA_OPCODE_LMULS(MACC, VV):
2267 case CASE_VFMA_OPCODE_LMULS(NMSAC, VV): {
2268 // It only make sense to toggle these between clobbering the
2269 // addend/subtrahend/minuend one of the multiplicands.
2270 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
2271 assert((OpIdx1 == 3 || OpIdx2 == 3) && "Unexpected opcode index");
2272 unsigned Opc;
2273 switch (MI.getOpcode()) {
2274 default:
2275 llvm_unreachable("Unexpected opcode");
2276 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMACC, FMADD)
2277 CASE_VFMA_CHANGE_OPCODE_SPLATS(FMADD, FMACC)
2280 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMACC, FNMADD)
2281 CASE_VFMA_CHANGE_OPCODE_SPLATS(FNMADD, FNMACC)
2284 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMACC, FMADD, VV)
2286 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMACC, FNMADD, VV)
2288 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VX)
2289 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VX)
2290 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VX)
2291 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VX)
2292 CASE_VFMA_CHANGE_OPCODE_LMULS(MACC, MADD, VV)
2293 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSAC, NMSUB, VV)
2294 }
2295
2296 auto &WorkingMI = cloneIfNew(MI);
2297 WorkingMI.setDesc(get(Opc));
2298 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
2299 OpIdx1, OpIdx2);
2300 }
2301 case CASE_VFMA_OPCODE_LMULS_MF4(FMADD, VV):
2303 case CASE_VFMA_OPCODE_LMULS_MF4(FNMADD, VV):
2305 case CASE_VFMA_OPCODE_LMULS(MADD, VV):
2306 case CASE_VFMA_OPCODE_LMULS(NMSUB, VV): {
2307 assert((OpIdx1 == 1 || OpIdx2 == 1) && "Unexpected opcode index");
2308 // If one of the operands, is the addend we need to change opcode.
2309 // Otherwise we're just swapping 2 of the multiplicands.
2310 if (OpIdx1 == 3 || OpIdx2 == 3) {
2311 unsigned Opc;
2312 switch (MI.getOpcode()) {
2313 default:
2314 llvm_unreachable("Unexpected opcode");
2315 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FMADD, FMACC, VV)
2317 CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(FNMADD, FNMACC, VV)
2319 CASE_VFMA_CHANGE_OPCODE_LMULS(MADD, MACC, VV)
2320 CASE_VFMA_CHANGE_OPCODE_LMULS(NMSUB, NMSAC, VV)
2321 }
2322
2323 auto &WorkingMI = cloneIfNew(MI);
2324 WorkingMI.setDesc(get(Opc));
2325 return TargetInstrInfo::commuteInstructionImpl(WorkingMI, /*NewMI=*/false,
2326 OpIdx1, OpIdx2);
2327 }
2328 // Let the default code handle it.
2329 break;
2330 }
2331 }
2332
2333 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2334}
2335
2336#undef CASE_VFMA_CHANGE_OPCODE_SPLATS
2337#undef CASE_VFMA_CHANGE_OPCODE_LMULS
2338#undef CASE_VFMA_CHANGE_OPCODE_COMMON
2339#undef CASE_VFMA_SPLATS
2340#undef CASE_VFMA_OPCODE_LMULS
2341#undef CASE_VFMA_OPCODE_COMMON
2342
2343// clang-format off
2344#define CASE_WIDEOP_OPCODE_COMMON(OP, LMUL) \
2345 RISCV::PseudoV##OP##_##LMUL##_TIED
2346
2347#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP) \
2348 CASE_WIDEOP_OPCODE_COMMON(OP, MF4): \
2349 case CASE_WIDEOP_OPCODE_COMMON(OP, MF2): \
2350 case CASE_WIDEOP_OPCODE_COMMON(OP, M1): \
2351 case CASE_WIDEOP_OPCODE_COMMON(OP, M2): \
2352 case CASE_WIDEOP_OPCODE_COMMON(OP, M4)
2353
2354#define CASE_WIDEOP_OPCODE_LMULS(OP) \
2355 CASE_WIDEOP_OPCODE_COMMON(OP, MF8): \
2356 case CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
2357// clang-format on
2358
2359#define CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, LMUL) \
2360 case RISCV::PseudoV##OP##_##LMUL##_TIED: \
2361 NewOpc = RISCV::PseudoV##OP##_##LMUL; \
2362 break;
2363
2364#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP) \
2365 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF4) \
2366 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF2) \
2367 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M1) \
2368 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M2) \
2369 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, M4)
2370
2371#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP) \
2372 CASE_WIDEOP_CHANGE_OPCODE_COMMON(OP, MF8) \
2373 CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
2374
2376 LiveVariables *LV,
2377 LiveIntervals *LIS) const {
2378 switch (MI.getOpcode()) {
2379 default:
2380 break;
2381 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWADD_WV):
2382 case CASE_WIDEOP_OPCODE_LMULS_MF4(FWSUB_WV):
2383 case CASE_WIDEOP_OPCODE_LMULS(WADD_WV):
2384 case CASE_WIDEOP_OPCODE_LMULS(WADDU_WV):
2385 case CASE_WIDEOP_OPCODE_LMULS(WSUB_WV):
2386 case CASE_WIDEOP_OPCODE_LMULS(WSUBU_WV): {
2387 // If the tail policy is undisturbed we can't convert.
2388 assert(RISCVII::hasVecPolicyOp(MI.getDesc().TSFlags) &&
2389 MI.getNumExplicitOperands() == 6);
2390 if ((MI.getOperand(5).getImm() & 1) == 0)
2391 return nullptr;
2392
2393 // clang-format off
2394 unsigned NewOpc;
2395 switch (MI.getOpcode()) {
2396 default:
2397 llvm_unreachable("Unexpected opcode");
2404 }
2405 // clang-format on
2406
2407 MachineBasicBlock &MBB = *MI.getParent();
2408 MachineInstrBuilder MIB = BuildMI(MBB, MI, MI.getDebugLoc(), get(NewOpc))
2409 .add(MI.getOperand(0))
2410 .add(MI.getOperand(1))
2411 .add(MI.getOperand(2))
2412 .add(MI.getOperand(3))
2413 .add(MI.getOperand(4));
2414 MIB.copyImplicitOps(MI);
2415
2416 if (LV) {
2417 unsigned NumOps = MI.getNumOperands();
2418 for (unsigned I = 1; I < NumOps; ++I) {
2419 MachineOperand &Op = MI.getOperand(I);
2420 if (Op.isReg() && Op.isKill())
2421 LV->replaceKillInstruction(Op.getReg(), MI, *MIB);
2422 }
2423 }
2424
2425 if (LIS) {
2427
2428 if (MI.getOperand(0).isEarlyClobber()) {
2429 // Use operand 1 was tied to early-clobber def operand 0, so its live
2430 // interval could have ended at an early-clobber slot. Now they are not
2431 // tied we need to update it to the normal register slot.
2432 LiveInterval &LI = LIS->getInterval(MI.getOperand(1).getReg());
2434 if (S->end == Idx.getRegSlot(true))
2435 S->end = Idx.getRegSlot();
2436 }
2437 }
2438
2439 return MIB;
2440 }
2441 }
2442
2443 return nullptr;
2444}
2445
2446#undef CASE_WIDEOP_CHANGE_OPCODE_LMULS
2447#undef CASE_WIDEOP_CHANGE_OPCODE_COMMON
2448#undef CASE_WIDEOP_OPCODE_LMULS
2449#undef CASE_WIDEOP_OPCODE_COMMON
2450
2454 const DebugLoc &DL, Register DestReg,
2455 int64_t Amount,
2456 MachineInstr::MIFlag Flag) const {
2457 assert(Amount > 0 && "There is no need to get VLEN scaled value.");
2458 assert(Amount % 8 == 0 &&
2459 "Reserve the stack by the multiple of one vector size.");
2460
2462 int64_t NumOfVReg = Amount / 8;
2463
2464 BuildMI(MBB, II, DL, get(RISCV::PseudoReadVLENB), DestReg).setMIFlag(Flag);
2465 assert(isInt<32>(NumOfVReg) &&
2466 "Expect the number of vector registers within 32-bits.");
2467 if (isPowerOf2_32(NumOfVReg)) {
2468 uint32_t ShiftAmount = Log2_32(NumOfVReg);
2469 if (ShiftAmount == 0)
2470 return;
2471 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
2472 .addReg(DestReg, RegState::Kill)
2473 .addImm(ShiftAmount)
2474 .setMIFlag(Flag);
2475 } else if (STI.hasStdExtZba() &&
2476 ((NumOfVReg % 3 == 0 && isPowerOf2_64(NumOfVReg / 3)) ||
2477 (NumOfVReg % 5 == 0 && isPowerOf2_64(NumOfVReg / 5)) ||
2478 (NumOfVReg % 9 == 0 && isPowerOf2_64(NumOfVReg / 9)))) {
2479 // We can use Zba SHXADD+SLLI instructions for multiply in some cases.
2480 unsigned Opc;
2481 uint32_t ShiftAmount;
2482 if (NumOfVReg % 9 == 0) {
2483 Opc = RISCV::SH3ADD;
2484 ShiftAmount = Log2_64(NumOfVReg / 9);
2485 } else if (NumOfVReg % 5 == 0) {
2486 Opc = RISCV::SH2ADD;
2487 ShiftAmount = Log2_64(NumOfVReg / 5);
2488 } else if (NumOfVReg % 3 == 0) {
2489 Opc = RISCV::SH1ADD;
2490 ShiftAmount = Log2_64(NumOfVReg / 3);
2491 } else {
2492 llvm_unreachable("Unexpected number of vregs");
2493 }
2494 if (ShiftAmount)
2495 BuildMI(MBB, II, DL, get(RISCV::SLLI), DestReg)
2496 .addReg(DestReg, RegState::Kill)
2497 .addImm(ShiftAmount)
2498 .setMIFlag(Flag);
2499 BuildMI(MBB, II, DL, get(Opc), DestReg)
2500 .addReg(DestReg, RegState::Kill)
2501 .addReg(DestReg)
2502 .setMIFlag(Flag);
2503 } else if (isPowerOf2_32(NumOfVReg - 1)) {
2504 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2505 uint32_t ShiftAmount = Log2_32(NumOfVReg - 1);
2506 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
2507 .addReg(DestReg)
2508 .addImm(ShiftAmount)
2509 .setMIFlag(Flag);
2510 BuildMI(MBB, II, DL, get(RISCV::ADD), DestReg)
2511 .addReg(ScaledRegister, RegState::Kill)
2512 .addReg(DestReg, RegState::Kill)
2513 .setMIFlag(Flag);
2514 } else if (isPowerOf2_32(NumOfVReg + 1)) {
2515 Register ScaledRegister = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2516 uint32_t ShiftAmount = Log2_32(NumOfVReg + 1);
2517 BuildMI(MBB, II, DL, get(RISCV::SLLI), ScaledRegister)
2518 .addReg(DestReg)
2519 .addImm(ShiftAmount)
2520 .setMIFlag(Flag);
2521 BuildMI(MBB, II, DL, get(RISCV::SUB), DestReg)
2522 .addReg(ScaledRegister, RegState::Kill)
2523 .addReg(DestReg, RegState::Kill)
2524 .setMIFlag(Flag);
2525 } else {
2526 Register N = MRI.createVirtualRegister(&RISCV::GPRRegClass);
2527 movImm(MBB, II, DL, N, NumOfVReg, Flag);
2528 if (!STI.hasStdExtM() && !STI.hasStdExtZmmul())
2530 MF.getFunction(),
2531 "M- or Zmmul-extension must be enabled to calculate the vscaled size/"
2532 "offset."});
2533 BuildMI(MBB, II, DL, get(RISCV::MUL), DestReg)
2534 .addReg(DestReg, RegState::Kill)
2536 .setMIFlag(Flag);
2537 }
2538}
2539
2540// Checks if all users only demand the lower \p OrigBits of the original
2541// instruction's result.
2542// TODO: handle multiple interdependent transformations
2544 const MachineRegisterInfo &MRI,
2545 unsigned OrigBits) const {
2546
2549
2550 Worklist.push_back(std::make_pair(&OrigMI, OrigBits));
2551
2552 while (!Worklist.empty()) {
2553 auto P = Worklist.pop_back_val();
2554 const MachineInstr *MI = P.first;
2555 unsigned Bits = P.second;
2556
2557 if (!Visited.insert(P).second)
2558 continue;
2559
2560 // Only handle instructions with one def.
2561 if (MI->getNumExplicitDefs() != 1)
2562 return false;
2563
2564 for (auto &UserOp : MRI.use_operands(MI->getOperand(0).getReg())) {
2565 const MachineInstr *UserMI = UserOp.getParent();
2566 unsigned OpIdx = UserOp.getOperandNo();
2567
2568 switch (UserMI->getOpcode()) {
2569 default:
2570 return false;
2571
2572 case RISCV::ADDIW:
2573 case RISCV::ADDW:
2574 case RISCV::DIVUW:
2575 case RISCV::DIVW:
2576 case RISCV::MULW:
2577 case RISCV::REMUW:
2578 case RISCV::REMW:
2579 case RISCV::SLLIW:
2580 case RISCV::SLLW:
2581 case RISCV::SRAIW:
2582 case RISCV::SRAW:
2583 case RISCV::SRLIW:
2584 case RISCV::SRLW:
2585 case RISCV::SUBW:
2586 case RISCV::ROLW:
2587 case RISCV::RORW:
2588 case RISCV::RORIW:
2589 case RISCV::CLZW:
2590 case RISCV::CTZW:
2591 case RISCV::CPOPW:
2592 case RISCV::SLLI_UW:
2593 case RISCV::FMV_W_X:
2594 case RISCV::FCVT_H_W:
2595 case RISCV::FCVT_H_WU:
2596 case RISCV::FCVT_S_W:
2597 case RISCV::FCVT_S_WU:
2598 case RISCV::FCVT_D_W:
2599 case RISCV::FCVT_D_WU:
2600 if (Bits >= 32)
2601 break;
2602 return false;
2603 case RISCV::SEXT_B:
2604 case RISCV::PACKH:
2605 if (Bits >= 8)
2606 break;
2607 return false;
2608 case RISCV::SEXT_H:
2609 case RISCV::FMV_H_X:
2610 case RISCV::ZEXT_H_RV32:
2611 case RISCV::ZEXT_H_RV64:
2612 case RISCV::PACKW:
2613 if (Bits >= 16)
2614 break;
2615 return false;
2616
2617 case RISCV::PACK:
2618 if (Bits >= (STI.getXLen() / 2))
2619 break;
2620 return false;
2621
2622 case RISCV::SRLI: {
2623 // If we are shifting right by less than Bits, and users don't demand
2624 // any bits that were shifted into [Bits-1:0], then we can consider this
2625 // as an N-Bit user.
2626 unsigned ShAmt = UserMI->getOperand(2).getImm();
2627 if (Bits > ShAmt) {
2628 Worklist.push_back(std::make_pair(UserMI, Bits - ShAmt));
2629 break;
2630 }
2631 return false;
2632 }
2633
2634 // these overwrite higher input bits, otherwise the lower word of output
2635 // depends only on the lower word of input. So check their uses read W.
2636 case RISCV::SLLI:
2637 if (Bits >= (STI.getXLen() - UserMI->getOperand(2).getImm()))
2638 break;
2639 Worklist.push_back(std::make_pair(UserMI, Bits));
2640 break;
2641 case RISCV::ANDI: {
2642 uint64_t Imm = UserMI->getOperand(2).getImm();
2643 if (Bits >= (unsigned)llvm::bit_width(Imm))
2644 break;
2645 Worklist.push_back(std::make_pair(UserMI, Bits));
2646 break;
2647 }
2648 case RISCV::ORI: {
2649 uint64_t Imm = UserMI->getOperand(2).getImm();
2650 if (Bits >= (unsigned)llvm::bit_width<uint64_t>(~Imm))
2651 break;
2652 Worklist.push_back(std::make_pair(UserMI, Bits));
2653 break;
2654 }
2655
2656 case RISCV::SLL:
2657 case RISCV::BSET:
2658 case RISCV::BCLR:
2659 case RISCV::BINV:
2660 // Operand 2 is the shift amount which uses log2(xlen) bits.
2661 if (OpIdx == 2) {
2662 if (Bits >= Log2_32(STI.getXLen()))
2663 break;
2664 return false;
2665 }
2666 Worklist.push_back(std::make_pair(UserMI, Bits));
2667 break;
2668
2669 case RISCV::SRA:
2670 case RISCV::SRL:
2671 case RISCV::ROL:
2672 case RISCV::ROR:
2673 // Operand 2 is the shift amount which uses 6 bits.
2674 if (OpIdx == 2 && Bits >= Log2_32(STI.getXLen()))
2675 break;
2676 return false;
2677
2678 case RISCV::ADD_UW:
2679 case RISCV::SH1ADD_UW:
2680 case RISCV::SH2ADD_UW:
2681 case RISCV::SH3ADD_UW:
2682 // Operand 1 is implicitly zero extended.
2683 if (OpIdx == 1 && Bits >= 32)
2684 break;
2685 Worklist.push_back(std::make_pair(UserMI, Bits));
2686 break;
2687
2688 case RISCV::BEXTI:
2689 if (UserMI->getOperand(2).getImm() >= Bits)
2690 return false;
2691 break;
2692
2693 case RISCV::SB:
2694 // The first argument is the value to store.
2695 if (OpIdx == 0 && Bits >= 8)
2696 break;
2697 return false;
2698 case RISCV::SH:
2699 // The first argument is the value to store.
2700 if (OpIdx == 0 && Bits >= 16)
2701 break;
2702 return false;
2703 case RISCV::SW:
2704 // The first argument is the value to store.
2705 if (OpIdx == 0 && Bits >= 32)
2706 break;
2707 return false;
2708
2709 // For these, lower word of output in these operations, depends only on
2710 // the lower word of input. So, we check all uses only read lower word.
2711 case RISCV::COPY:
2712 case RISCV::PHI:
2713
2714 case RISCV::ADD:
2715 case RISCV::ADDI:
2716 case RISCV::AND:
2717 case RISCV::MUL:
2718 case RISCV::OR:
2719 case RISCV::SUB:
2720 case RISCV::XOR:
2721 case RISCV::XORI:
2722
2723 case RISCV::ANDN:
2724 case RISCV::BREV8:
2725 case RISCV::CLMUL:
2726 case RISCV::ORC_B:
2727 case RISCV::ORN:
2728 case RISCV::SH1ADD:
2729 case RISCV::SH2ADD:
2730 case RISCV::SH3ADD:
2731 case RISCV::XNOR:
2732 case RISCV::BSETI:
2733 case RISCV::BCLRI:
2734 case RISCV::BINVI:
2735 Worklist.push_back(std::make_pair(UserMI, Bits));
2736 break;
2737
2738 case RISCV::PseudoCCMOVGPR:
2739 // Either operand 4 or operand 5 is returned by this instruction. If
2740 // only the lower word of the result is used, then only the lower word
2741 // of operand 4 and 5 is used.
2742 if (OpIdx != 4 && OpIdx != 5)
2743 return false;
2744 Worklist.push_back(std::make_pair(UserMI, Bits));
2745 break;
2746
2747 case RISCV::VT_MASKC:
2748 case RISCV::VT_MASKCN:
2749 if (OpIdx != 1)
2750 return false;
2751 Worklist.push_back(std::make_pair(UserMI, Bits));
2752 break;
2753 }
2754 }
2755 }
2756
2757 return true;
2758}
2759
2760// Returns true if this is the sext.w pattern, addiw rd, rs1, 0.
2762 return MI.getOpcode() == RISCV::ADDIW && MI.getOperand(1).isReg() &&
2763 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 0;
2764}
2765
2766// Returns true if this is the zext.w pattern, adduw rd, rs1, x0.
2768 return MI.getOpcode() == RISCV::ADD_UW && MI.getOperand(1).isReg() &&
2769 MI.getOperand(2).isReg() && MI.getOperand(2).getReg() == RISCV::X0;
2770}
2771
2772// Returns true if this is the zext.b pattern, andi rd, rs1, 255.
2774 return MI.getOpcode() == RISCV::ANDI && MI.getOperand(1).isReg() &&
2775 MI.getOperand(2).isImm() && MI.getOperand(2).getImm() == 255;
2776}
2777
2778static bool isRVVWholeLoadStore(unsigned Opcode) {
2779 switch (Opcode) {
2780 default:
2781 return false;
2782 case RISCV::VS1R_V:
2783 case RISCV::VS2R_V:
2784 case RISCV::VS4R_V:
2785 case RISCV::VS8R_V:
2786 case RISCV::VL1RE8_V:
2787 case RISCV::VL2RE8_V:
2788 case RISCV::VL4RE8_V:
2789 case RISCV::VL8RE8_V:
2790 case RISCV::VL1RE16_V:
2791 case RISCV::VL2RE16_V:
2792 case RISCV::VL4RE16_V:
2793 case RISCV::VL8RE16_V:
2794 case RISCV::VL1RE32_V:
2795 case RISCV::VL2RE32_V:
2796 case RISCV::VL4RE32_V:
2797 case RISCV::VL8RE32_V:
2798 case RISCV::VL1RE64_V:
2799 case RISCV::VL2RE64_V:
2800 case RISCV::VL4RE64_V:
2801 case RISCV::VL8RE64_V:
2802 return true;
2803 }
2804}
2805
2807 // RVV lacks any support for immediate addressing for stack addresses, so be
2808 // conservative.
2809 unsigned Opcode = MI.getOpcode();
2810 if (!RISCVVPseudosTable::getPseudoInfo(Opcode) &&
2811 !isRVVWholeLoadStore(Opcode) && !isRVVSpillForZvlsseg(Opcode))
2812 return false;
2813 return true;
2814}
2815
2816std::optional<std::pair<unsigned, unsigned>>
2818 switch (Opcode) {
2819 default:
2820 return std::nullopt;
2821 case RISCV::PseudoVSPILL2_M1:
2822 case RISCV::PseudoVRELOAD2_M1:
2823 return std::make_pair(2u, 1u);
2824 case RISCV::PseudoVSPILL2_M2:
2825 case RISCV::PseudoVRELOAD2_M2:
2826 return std::make_pair(2u, 2u);
2827 case RISCV::PseudoVSPILL2_M4:
2828 case RISCV::PseudoVRELOAD2_M4:
2829 return std::make_pair(2u, 4u);
2830 case RISCV::PseudoVSPILL3_M1:
2831 case RISCV::PseudoVRELOAD3_M1:
2832 return std::make_pair(3u, 1u);
2833 case RISCV::PseudoVSPILL3_M2:
2834 case RISCV::PseudoVRELOAD3_M2:
2835 return std::make_pair(3u, 2u);
2836 case RISCV::PseudoVSPILL4_M1:
2837 case RISCV::PseudoVRELOAD4_M1:
2838 return std::make_pair(4u, 1u);
2839 case RISCV::PseudoVSPILL4_M2:
2840 case RISCV::PseudoVRELOAD4_M2:
2841 return std::make_pair(4u, 2u);
2842 case RISCV::PseudoVSPILL5_M1:
2843 case RISCV::PseudoVRELOAD5_M1:
2844 return std::make_pair(5u, 1u);
2845 case RISCV::PseudoVSPILL6_M1:
2846 case RISCV::PseudoVRELOAD6_M1:
2847 return std::make_pair(6u, 1u);
2848 case RISCV::PseudoVSPILL7_M1:
2849 case RISCV::PseudoVRELOAD7_M1:
2850 return std::make_pair(7u, 1u);
2851 case RISCV::PseudoVSPILL8_M1:
2852 case RISCV::PseudoVRELOAD8_M1:
2853 return std::make_pair(8u, 1u);
2854 }
2855}
2856
2858 return MI.getNumExplicitDefs() == 2 && MI.modifiesRegister(RISCV::VL) &&
2859 !MI.isInlineAsm();
2860}
2861
2862bool RISCV::hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2) {
2863 int16_t MI1FrmOpIdx =
2864 RISCV::getNamedOperandIdx(MI1.getOpcode(), RISCV::OpName::frm);
2865 int16_t MI2FrmOpIdx =
2866 RISCV::getNamedOperandIdx(MI2.getOpcode(), RISCV::OpName::frm);
2867 if (MI1FrmOpIdx < 0 || MI2FrmOpIdx < 0)
2868 return false;
2869 MachineOperand FrmOp1 = MI1.getOperand(MI1FrmOpIdx);
2870 MachineOperand FrmOp2 = MI2.getOperand(MI2FrmOpIdx);
2871 return FrmOp1.getImm() == FrmOp2.getImm();
2872}
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
@ MachineOutlinerDefault
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static ARCCC::CondCode getOppositeBranchCondition(ARCCC::CondCode CC)
Return the inverse of passed condition, i.e. turning COND_E to COND_NE.
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
static M68k::CondCode getCondFromBranchOpc(unsigned BrOpc)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
#define P(N)
const char LLVMTargetMachineRef TM
unsigned UseOpIdx
unsigned Log2SEW
unsigned SEW
uint64_t TSFlags
#define CASE_VFMA_CHANGE_OPCODE_SPLATS(OLDOP, NEWOP)
static bool isRVVWholeLoadStore(unsigned Opcode)
static void combineFPFusedMultiply(MachineInstr &Root, MachineInstr &Prev, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs)
static bool getFPFusedMultiplyPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS(OP)
static cl::opt< bool > PreferWholeRegisterMove("riscv-prefer-whole-register-move", cl::init(false), cl::Hidden, cl::desc("Prefer whole register move for vector registers."))
#define CASE_VFMA_SPLATS(OP)
unsigned getPredicatedOpcode(unsigned Opcode)
static unsigned getFPFusedMultiplyOpcode(unsigned RootOpc, MachineCombinerPattern Pattern)
#define CASE_WIDEOP_OPCODE_LMULS(OP)
static bool isFSUB(unsigned Opc)
MachineOutlinerConstructionID
static bool isFMUL(unsigned Opc)
#define CASE_WIDEOP_CHANGE_OPCODE_LMULS_MF4(OP)
#define CASE_OPERAND_UIMM(NUM)
#define CASE_VFMA_CHANGE_OPCODE_LMULS(OLDOP, NEWOP, TYPE)
static bool isFADD(unsigned Opc)
#define CASE_VFMA_CHANGE_OPCODE_LMULS_MF4(OLDOP, NEWOP, TYPE)
#define CASE_WIDEOP_OPCODE_LMULS_MF4(OP)
static unsigned getAddendOperandIdx(MachineCombinerPattern Pattern)
static bool isConvertibleToVMV_V_V(const RISCVSubtarget &STI, const MachineBasicBlock &MBB, MachineBasicBlock::const_iterator MBBI, MachineBasicBlock::const_iterator &DefMBBI, RISCVII::VLMUL LMul)
#define CASE_VFMA_OPCODE_LMULS(OP, TYPE)
static MachineInstr * canFoldAsPredicatedOp(Register Reg, const MachineRegisterInfo &MRI, const TargetInstrInfo *TII)
Identify instructions that can be folded into a CCMOV instruction, and return the defining instructio...
static bool getFPPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce)
static bool canCombineFPFusedMultiply(const MachineInstr &Root, const MachineOperand &MO, bool DoRegPressureReduce)
#define CASE_VFMA_OPCODE_LMULS_MF4(OP, TYPE)
const SmallVectorImpl< MachineOperand > MachineBasicBlock * TBB
This file declares the machine register scavenger class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:467
static unsigned getSize(unsigned Kind)
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
static const DILocation * getMergedLocation(const DILocation *LocA, const DILocation *LocB)
When two instructions are combined into a single instruction we also need to combine the original loc...
bool isBigEndian() const
Definition: DataLayout.h:246
A debug info location.
Definition: DebugLoc.h:33
Diagnostic information for unsupported feature in backend.
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:641
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:315
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
LiveInterval - This class represents the liveness of a register, or stack slot.
Definition: LiveInterval.h:686
LiveInterval & getInterval(Register Reg)
SlotIndex ReplaceMachineInstrInMaps(MachineInstr &MI, MachineInstr &NewMI)
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
Definition: LiveInterval.h:408
void replaceKillInstruction(Register Reg, MachineInstr &OldMI, MachineInstr &NewMI)
replaceKillInstruction - Update register kill info by replacing a kill instruction with a new one.
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:31
MCInstBuilder & addImm(int64_t Val)
Add a new integer immediate operand.
Definition: MCInstBuilder.h:37
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
unsigned getNumOperands() const
Return the number of declared MachineOperands for this MachineInstruction.
Definition: MCInstrDesc.h:237
ArrayRef< MCOperandInfo > operands() const
Definition: MCInstrDesc.h:239
bool isCommutable() const
Return true if this may be a 2- or 3-address instruction (of the form "X = op Y, Z,...
Definition: MCInstrDesc.h:480
bool isConditionalBranch() const
Return true if this is a branch which may fall through to the next instruction or may transfer contro...
Definition: MCInstrDesc.h:316
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
unsigned pred_size() const
reverse_iterator rend()
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
iterator getLastNonDebugInstr(bool SkipPseudoOp=true)
Returns an iterator to the last non-debug instruction in the basic block, or end().
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
void setStackID(int ObjectIdx, uint8_t ID)
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineInstrBuilder & setMIFlag(MachineInstr::MIFlag Flag) const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addFrameIndex(int Idx) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & setMIFlags(unsigned Flags) const
const MachineInstrBuilder & copyImplicitOps(const MachineInstr &OtherMI) const
Copy all the implicit operands from OtherMI onto this one.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
reverse_iterator getReverse() const
Get a reverse iterator to the same node.
Representation of each machine instruction.
Definition: MachineInstr.h:68
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:516
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:313
void setFlags(unsigned flags)
Definition: MachineInstr.h:366
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
Definition: MachineInstr.h:357
unsigned getNumExplicitOperands() const
Returns the number of non-implicit operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
Definition: MachineInstr.h:513
bool hasUnmodeledSideEffects() const
Return true if this instruction has side effects that are not modeled by mayLoad / mayStore,...
bool hasOneMemOperand() const
Return true if this instruction has exactly one MachineMemOperand.
Definition: MachineInstr.h:746
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:731
bool hasOrderedMemoryRef() const
Return true if this instruction may have an ordered or volatile memory reference, or if the informati...
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:445
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:352
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
void clearKillInfo()
Clears kill flags on all operands.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
int64_t getImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isCPI() const
isCPI - Tests if this is a MO_ConstantPoolIndex operand.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
bool isJTI() const
isJTI - Tests if this is a MO_JumpTableIndex operand.
static MachineOperand CreateImm(int64_t Val)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
bool isIdenticalTo(const MachineOperand &Other) const
Returns true if this operand is identical to the specified operand except for liveness related flags ...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
void setSpecialOperandAttr(MachineInstr &OldMI1, MachineInstr &OldMI2, MachineInstr &NewMI1, MachineInstr &NewMI2) const override
MachineInstr * convertToThreeAddress(MachineInstr &MI, LiveVariables *LV, LiveIntervals *LIS) const override
unsigned isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex) const override
unsigned removeBranch(MachineBasicBlock &MBB, int *BytesRemoved=nullptr) const override
unsigned isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex) const override
bool hasAllNBitUsers(const MachineInstr &MI, const MachineRegisterInfo &MRI, unsigned NBits) const
const MCInstrDesc & getBrCond(RISCVCC::CondCode CC) const
bool isFunctionSafeToOutlineFrom(MachineFunction &MF, bool OutlineFromLinkOnceODRs) const override
unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef< MachineOperand > Cond, const DebugLoc &dl, int *BytesAdded=nullptr) const override
bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const override
RISCVInstrInfo(RISCVSubtarget &STI)
void genAlternativeCodeSequence(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg) const override
outliner::InstrType getOutliningType(MachineBasicBlock::iterator &MBBI, unsigned Flags) const override
MachineInstr * optimizeSelect(MachineInstr &MI, SmallPtrSetImpl< MachineInstr * > &SeenMIs, bool) const override
void insertIndirectBranch(MachineBasicBlock &MBB, MachineBasicBlock &NewDestBB, MachineBasicBlock &RestoreBB, const DebugLoc &DL, int64_t BrOffset, RegScavenger *RS) const override
bool isAsCheapAsAMove(const MachineInstr &MI) const override
bool verifyInstruction(const MachineInstr &MI, StringRef &ErrInfo) const override
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, MCRegister DstReg, MCRegister SrcReg, bool KillSrc) const override
outliner::OutlinedFunction getOutliningCandidateInfo(std::vector< outliner::Candidate > &RepeatedSequenceLocs) const override
const RISCVSubtarget & STI
bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce) const override
std::optional< unsigned > getInverseOpcode(unsigned Opcode) const override
ArrayRef< std::pair< unsigned, const char * > > getSerializableDirectMachineOperandTargetFlags() const override
unsigned getInstSizeInBytes(const MachineInstr &MI) const override
bool reverseBranchCondition(SmallVectorImpl< MachineOperand > &Cond) const override
MCInst getNop() const override
void finalizeInsInstrs(MachineInstr &Root, MachineCombinerPattern &P, SmallVectorImpl< MachineInstr * > &InsInstrs) const override
MachineInstr * foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef< unsigned > Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS=nullptr, VirtRegMap *VRM=nullptr) const override
bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const override
void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF, const outliner::OutlinedFunction &OF) const override
std::pair< unsigned, unsigned > decomposeMachineOperandsTargetFlags(unsigned TF) const override
MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const override
void movImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, Register DstReg, uint64_t Val, MachineInstr::MIFlag Flag=MachineInstr::NoFlags) const
void getVLENFactoredAmount(MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator II, const DebugLoc &DL, Register DestReg, int64_t Amount, MachineInstr::MIFlag Flag=MachineInstr::NoFlags) const
MachineBasicBlock * getBranchDestBlock(const MachineInstr &MI) const override
std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const override
bool getMemOperandWithOffsetWidth(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const
bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override
bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const override
bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl< MachineOperand > &Cond, bool AllowModify) const override
MachineBasicBlock::iterator insertOutlinedCall(Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It, MachineFunction &MF, outliner::Candidate &C) const override
bool isBranchOffsetInRange(unsigned BranchOpc, int64_t BrOffset) const override
void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register DstReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
bool isAssociativeAndCommutative(const MachineInstr &Inst, bool Invert) const override
std::optional< DestSourcePair > isCopyInstrImpl(const MachineInstr &MI) const override
bool analyzeSelect(const MachineInstr &MI, SmallVectorImpl< MachineOperand > &Cond, unsigned &TrueOp, unsigned &FalseOp, bool &Optimizable) const override
bool areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const override
void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg, bool IsKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI, Register VReg) const override
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
bool hasStdExtCOrZca() const
unsigned getXLen() const
const RISCVRegisterInfo * getRegisterInfo() const override
void enterBasicBlockEnd(MachineBasicBlock &MBB)
Start tracking liveness from the end of basic block MBB.
void setRegUsed(Register Reg, LaneBitmask LaneMask=LaneBitmask::getAll())
Tell the scavenger a register is used.
Register scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, bool RestoreAfter, int SPAdj, bool AllowSpill=true)
Make a register of the specific register class available from the current position backwards to the p...
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:97
bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
SlotIndex - An opaque wrapper around machine indexes.
Definition: SlotIndexes.h:82
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:379
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:177
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
virtual bool findCommutedOpIndices(const MachineInstr &MI, unsigned &SrcOpIdx1, unsigned &SrcOpIdx2) const
Returns true iff the routine could find two commutable operands in the given machine instruction.
virtual bool getMachineCombinerPatterns(MachineInstr &Root, SmallVectorImpl< MachineCombinerPattern > &Patterns, bool DoRegPressureReduce) const
Return true when there is potentially a faster code sequence for an instruction chain ending in Root.
virtual bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB, unsigned &Flags) const
Optional target hook that returns true if MBB is safe to outline from, and returns any target-specifi...
virtual void genAlternativeCodeSequence(MachineInstr &Root, MachineCombinerPattern Pattern, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< unsigned, unsigned > &InstIdxForVirtReg) const
When getMachineCombinerPatterns() finds patterns, this function generates the instructions that could...
virtual MachineInstr * commuteInstructionImpl(MachineInstr &MI, bool NewMI, unsigned OpIdx1, unsigned OpIdx2) const
This method commutes the operands of the given machine instruction MI.
virtual bool hasReassociableSibling(const MachineInstr &Inst, bool &Commuted) const
Return true when \P Inst has reassociable sibling.
virtual std::string createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op, unsigned OpIdx, const TargetRegisterInfo *TRI) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetInstrInfo * getInstrInfo() const
Target - Wrapper for Target specific information.
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
CondCode getOppositeBranchCondition(CondCode)
static unsigned getVecPolicyOpNum(const MCInstrDesc &Desc)
static bool hasMergeOp(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getMergeOpNum(const MCInstrDesc &Desc)
static bool isRVVWideningReduction(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
InstSeq generateInstSeq(int64_t Val, const FeatureBitset &ActiveFeatures)
@ OPERAND_UIMMLOG2XLEN_NONZERO
@ OPERAND_SIMM10_LSB0000_NONZERO
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
static bool isValidSEW(unsigned SEW)
void printVType(unsigned VType, raw_ostream &OS)
static unsigned getSEW(unsigned VType)
bool hasEqualFRM(const MachineInstr &MI1, const MachineInstr &MI2)
int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex)
bool isSEXT_W(const MachineInstr &MI)
bool isFaultFirstLoad(const MachineInstr &MI)
std::optional< std::pair< unsigned, unsigned > > isRVVSpillForZvlsseg(unsigned Opcode)
bool isZEXT_B(const MachineInstr &MI)
bool isRVVSpill(const MachineInstr &MI)
bool isZEXT_W(const MachineInstr &MI)
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Dead
Unused definition.
@ Define
Register definition.
@ Kill
The last use of a register.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
InstrType
Represents how an instruction should be mapped by the outliner.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1735
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:281
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:293
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:379
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:373
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:288
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
MachineCombinerPattern
These are instruction patterns matched by the machine combiner pass.
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:2264
unsigned getKillRegState(bool B)
bool isIntN(unsigned N, int64_t x)
Checks if an signed integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:257
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition: STLExtras.h:1998
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:543
#define N
This represents a simple continuous liveness interval for a value.
Definition: LiveInterval.h:162
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
An individual sequence of instructions to be replaced with a call to an outlined function.
The information necessary to create an outlined function for some class of candidate.