37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
50 cl::desc(
"Enable strict assertion checking for the dataflow algorithm"));
63 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
64 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
65 MI.getOpcode() == RISCV::PseudoVSETIVLI;
71 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
73 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
74 return RISCV::X0 ==
MI.getOperand(0).getReg();
77static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
102 case RISCV::VFMV_S_F:
113 case RISCV::VFMV_V_F:
122 case RISCV::VSLIDEDOWN_VX:
123 case RISCV::VSLIDEDOWN_VI:
124 case RISCV::VSLIDEUP_VX:
125 case RISCV::VSLIDEUP_VI:
132static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
142 case RISCV::VLSE16_V:
144 case RISCV::VSSE16_V:
147 case RISCV::VLSE32_V:
149 case RISCV::VSSE32_V:
152 case RISCV::VLSE64_V:
154 case RISCV::VSSE64_V:
160 return MI.getOpcode() == RISCV::ADDI &&
161 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
162 MI.getOperand(1).getReg() == RISCV::X0 &&
163 MI.getOperand(2).getImm() != 0;
171 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
184 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
193 if (UseMO.
getReg() == RISCV::NoRegister)
218struct DemandedFields {
223 bool VLZeroness =
false;
227 SEWGreaterThanOrEqual = 2,
229 SEWGreaterThanOrEqualAndLessThan64 =
236 bool SEWLMULRatio =
false;
237 bool TailPolicy =
false;
238 bool MaskPolicy =
false;
241 bool usedVTYPE()
const {
242 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
247 return VLAny || VLZeroness;
266 void doUnion(
const DemandedFields &
B) {
268 VLZeroness |=
B.VLZeroness;
269 SEW = std::max(SEW,
B.SEW);
271 SEWLMULRatio |=
B.SEWLMULRatio;
272 TailPolicy |=
B.TailPolicy;
273 MaskPolicy |=
B.MaskPolicy;
276#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
286 OS <<
"VLAny=" << VLAny <<
", ";
287 OS <<
"VLZeroness=" << VLZeroness <<
", ";
293 case SEWGreaterThanOrEqual:
294 OS <<
"SEWGreaterThanOrEqual";
296 case SEWGreaterThanOrEqualAndLessThan64:
297 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
304 OS <<
"LMUL=" <<
LMUL <<
", ";
305 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
306 OS <<
"TailPolicy=" << TailPolicy <<
", ";
307 OS <<
"MaskPolicy=" << MaskPolicy;
313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
325 const DemandedFields &Used) {
327 case DemandedFields::SEWNone:
329 case DemandedFields::SEWEqual:
333 case DemandedFields::SEWGreaterThanOrEqual:
337 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
348 if (
Used.SEWLMULRatio) {
353 if (Ratio1 != Ratio2)
378 if (
MI.isCall() ||
MI.isInlineAsm() ||
379 MI.readsRegister(RISCV::VL,
nullptr))
381 if (
MI.isCall() ||
MI.isInlineAsm() ||
382 MI.readsRegister(RISCV::VTYPE,
nullptr))
393 Res.MaskPolicy =
false;
402 if (getEEWForLoadStore(
MI)) {
403 Res.SEW = DemandedFields::SEWNone;
409 Res.TailPolicy =
false;
410 Res.MaskPolicy =
false;
417 if (isMaskRegOp(
MI)) {
418 Res.SEW = DemandedFields::SEWNone;
423 if (isScalarInsertInstr(
MI)) {
425 Res.SEWLMULRatio =
false;
433 if (hasUndefinedMergeOp(
MI, *
MRI)) {
434 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
435 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
437 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
438 Res.TailPolicy =
false;
443 if (isScalarExtractInstr(
MI)) {
446 Res.SEWLMULRatio =
false;
447 Res.TailPolicy =
false;
448 Res.MaskPolicy =
false;
478 uint8_t TailAgnostic : 1;
479 uint8_t MaskAgnostic : 1;
480 uint8_t SEWLMULRatioOnly : 1;
484 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
485 SEWLMULRatioOnly(
false) {}
487 static VSETVLIInfo getUnknown() {
494 void setUnknown() { State =
Unknown; }
495 bool isUnknown()
const {
return State ==
Unknown; }
499 AVLRegDef.DefMI =
DefMI;
500 AVLRegDef.DefReg =
AVLReg;
504 void setAVLImm(
unsigned Imm) {
509 void setAVLVLMAX() { State = AVLIsVLMAX; }
511 void setAVLIgnored() { State = AVLIsIgnored; }
513 bool hasAVLImm()
const {
return State == AVLIsImm; }
514 bool hasAVLReg()
const {
return State == AVLIsReg; }
515 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
516 bool hasAVLIgnored()
const {
return State == AVLIsIgnored; }
518 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());
519 return AVLRegDef.DefReg;
521 unsigned getAVLImm()
const {
526 assert(hasAVLReg() && AVLRegDef.DefMI);
527 return *AVLRegDef.DefMI;
530 void setAVL(VSETVLIInfo Info) {
532 if (
Info.isUnknown())
534 else if (
Info.hasAVLReg())
536 else if (
Info.hasAVLVLMAX())
538 else if (
Info.hasAVLIgnored())
542 setAVLImm(
Info.getAVLImm());
546 unsigned getSEW()
const {
return SEW; }
548 bool getTailAgnostic()
const {
return TailAgnostic; }
549 bool getMaskAgnostic()
const {
return MaskAgnostic; }
551 bool hasNonZeroAVL()
const {
553 return getAVLImm() > 0;
555 return isNonZeroLoadImmediate(getAVLDefMI());
563 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other)
const {
564 if (hasSameAVL(
Other))
566 return (hasNonZeroAVL() &&
Other.hasNonZeroAVL());
569 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
570 if (hasAVLReg() &&
Other.hasAVLReg())
571 return AVLRegDef.DefMI ==
Other.AVLRegDef.DefMI &&
572 AVLRegDef.DefReg ==
Other.AVLRegDef.DefReg;
574 if (hasAVLImm() &&
Other.hasAVLImm())
575 return getAVLImm() ==
Other.getAVLImm();
578 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
581 return Other.hasAVLIgnored();
588 "Can't set VTYPE for uninitialized or unknown");
596 "Can't set VTYPE for uninitialized or unknown");
607 "Can't encode VTYPE for uninitialized or unknown");
611 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
613 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
615 "Can't compare invalid VSETVLIInfos");
617 "Can't compare VTYPE in unknown state");
618 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
619 "Can't compare when only LMUL/SEW ratio is valid.");
620 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
627 "Can't use VTYPE for uninitialized or unknown");
635 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
637 "Can't compare invalid VSETVLIInfos");
639 "Can't compare VTYPE in unknown state");
643 bool hasCompatibleVTYPE(
const DemandedFields &Used,
644 const VSETVLIInfo &Require)
const {
645 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
651 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
654 "Can't compare invalid VSETVLIInfos");
655 assert(!Require.SEWLMULRatioOnly &&
656 "Expected a valid VTYPE for instruction!");
658 if (isUnknown() || Require.isUnknown())
662 if (SEWLMULRatioOnly)
665 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
668 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require))
671 return hasCompatibleVTYPE(Used, Require);
677 return !
Other.isValid();
678 if (!
Other.isValid())
683 return Other.isUnknown();
684 if (
Other.isUnknown())
687 if (!hasSameAVL(
Other))
691 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
695 if (SEWLMULRatioOnly)
696 return hasSameVLMAX(
Other);
699 return hasSameVTYPE(
Other);
703 return !(*
this ==
Other);
710 if (!
Other.isValid())
718 if (isUnknown() ||
Other.isUnknown())
719 return VSETVLIInfo::getUnknown();
727 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
728 VSETVLIInfo MergeInfo = *
this;
729 MergeInfo.SEWLMULRatioOnly =
true;
734 return VSETVLIInfo::getUnknown();
737#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
749 OS <<
"Uninitialized";
761 <<
"VLMul=" << (
unsigned)VLMul <<
", "
762 <<
"SEW=" << (
unsigned)SEW <<
", "
763 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
764 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
765 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
770#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
788 bool InQueue =
false;
798 std::vector<BlockData> BlockInfo;
799 std::queue<const MachineBasicBlock *> WorkList;
815 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
816 const VSETVLIInfo &CurInfo)
const;
817 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
820 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
823 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
825 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
826 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
828 VSETVLIInfo &Info)
const;
867char RISCVInsertVSETVLI::ID = 0;
872char RISCVCoalesceVSETVLI::
ID = 0;
882 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
883 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
885 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
886 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
889 "Can't handle X0, X0 vsetvli yet");
913 VSETVLIInfo InstrInfo;
915 bool TailAgnostic =
true;
916 bool MaskAgnostic =
true;
917 if (!hasUndefinedMergeOp(
MI, *
MRI)) {
919 TailAgnostic =
false;
920 MaskAgnostic =
false;
927 "Invalid Policy Value");
943 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
945 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
951 int64_t Imm = VLOp.
getImm();
956 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
957 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
958 InstrInfo.setAVLImm(VLMAX);
960 InstrInfo.setAVLVLMAX();
963 InstrInfo.setAVLImm(Imm);
972 InstrInfo.setAVLIgnored();
975 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
976 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
979 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
985 if (InstrInfo.hasAVLReg()) {
987 if (isVectorConfigInstr(
DefMI)) {
988 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(
DefMI, *
MRI);
989 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
990 (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
991 InstrInfo.setAVL(DefInstrInfo);
999 const VSETVLIInfo &Info,
1000 const VSETVLIInfo &PrevInfo) {
1007 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1009 ++NumInsertedVSETVL;
1010 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1013 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1025 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1027 if (isVectorConfigInstr(
DefMI)) {
1028 VSETVLIInfo DefInfo = getInfoForVSETVLI(
DefMI, *
MRI);
1029 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1041 if (
Info.hasAVLImm()) {
1049 if (
Info.hasAVLIgnored()) {
1052 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
1053 Info.hasSameVLMAX(PrevInfo)) {
1069 if (
Info.hasAVLVLMAX()) {
1070 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1079 MRI->constrainRegClass(
AVLReg, &RISCV::GPRNoX0RegClass);
1088 return Fractional || LMul == 1;
1094 const VSETVLIInfo &Require,
1095 const VSETVLIInfo &CurInfo)
const {
1098 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1101 DemandedFields
Used = getDemanded(
MI,
MRI, ST);
1111 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1114 Used.VLZeroness =
true;
1116 Used.TailPolicy =
false;
1124 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1127 Used.SEWLMULRatio =
false;
1129 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1130 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1132 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1133 Used.TailPolicy =
false;
1136 if (CurInfo.isCompatible(Used, Require, *
MRI))
1143 if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
1145 if (isVectorConfigInstr(
DefMI)) {
1146 VSETVLIInfo DefInfo = getInfoForVSETVLI(
DefMI, *
MRI);
1147 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1159 DemandedFields &Demanded) {
1162 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1163 !PrevInfo.isUnknown()) {
1165 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1166 Info.setVLMul(*NewVLMul);
1167 Demanded.LMUL =
true;
1176void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1187 const VSETVLIInfo PrevInfo =
Info;
1188 if (!
Info.isValid() ||
Info.isUnknown())
1191 DemandedFields Demanded = getDemanded(
MI,
MRI, ST);
1201 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo) &&
1202 IncomingInfo.hasSameVLMAX(PrevInfo);
1203 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1204 Info.setAVL(IncomingInfo);
1207 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1209 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1212 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1213 IncomingInfo.getTailAgnostic(),
1214 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1215 IncomingInfo.getMaskAgnostic());
1219 if (
Info.hasSEWLMULRatioOnly()) {
1220 VSETVLIInfo RatiolessInfo = IncomingInfo;
1221 RatiolessInfo.setAVL(Info);
1222 Info = RatiolessInfo;
1229void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1231 if (isVectorConfigInstr(
MI)) {
1238 Info.setAVLRegDef(
MRI->getVRegDef(
MI.getOperand(1).getReg()),
1239 MI.getOperand(1).getReg());
1245 if (
MI.isCall() ||
MI.isInlineAsm() ||
1246 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1247 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1248 Info = VSETVLIInfo::getUnknown();
1252 VSETVLIInfo &Info)
const {
1253 bool HadVectorOp =
false;
1257 transferBefore(Info,
MI);
1262 transferAfter(Info,
MI);
1272 BBInfo.InQueue =
false;
1276 VSETVLIInfo InInfo = BBInfo.
Pred;
1279 InInfo.setUnknown();
1282 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1286 if (!InInfo.isValid())
1290 if (InInfo == BBInfo.
Pred)
1293 BBInfo.
Pred = InInfo;
1295 <<
" changed to " << BBInfo.
Pred <<
"\n");
1301 VSETVLIInfo TmpStatus;
1302 computeVLVTYPEChanges(
MBB, TmpStatus);
1306 if (BBInfo.
Exit == TmpStatus)
1309 BBInfo.
Exit = TmpStatus;
1311 <<
" changed to " << BBInfo.
Exit <<
"\n");
1316 if (!BlockInfo[S->getNumber()].InQueue) {
1317 BlockInfo[S->getNumber()].InQueue =
true;
1325bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1330 if (!Require.hasAVLReg())
1335 if (
PHI->getOpcode() != RISCV::PHI ||
PHI->getParent() != &
MBB)
1338 for (
unsigned PHIOp = 1, NumOps =
PHI->getNumOperands(); PHIOp != NumOps;
1342 const VSETVLIInfo &PBBExit = BlockInfo[PBB->
getNumber()].Exit;
1351 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI, *
MRI);
1352 if (DefInfo != PBBExit)
1358 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1371 bool PrefixTransparent =
true;
1373 const VSETVLIInfo PrevInfo = CurInfo;
1374 transferBefore(CurInfo,
MI);
1377 if (isVectorConfigInstr(
MI)) {
1379 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1380 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1381 "Unexpected operands where VL and VTYPE should be");
1382 MI.getOperand(3).setIsDead(
false);
1383 MI.getOperand(4).setIsDead(
false);
1384 PrefixTransparent =
false;
1389 if (PrevInfo != CurInfo) {
1397 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1398 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1399 PrefixTransparent =
false;
1409 VLOp.
setReg(RISCV::NoRegister);
1416 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, Reg) &&
1417 MRI->use_nodbg_empty(Reg))
1427 if (
MI.isCall() ||
MI.isInlineAsm() ||
1428 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1429 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1430 PrefixTransparent =
false;
1432 transferAfter(CurInfo,
MI);
1438 const VSETVLIInfo &ExitInfo = BlockInfo[
MBB.
getNumber()].Exit;
1439 if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() &&
1440 CurInfo != ExitInfo) {
1452 if (CurInfo !=
Info.Exit) {
1459 "InsertVSETVLI dataflow invariant violated");
1473 VSETVLIInfo AvailableInfo;
1475 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1476 if (PredInfo.isUnknown()) {
1477 if (UnavailablePred)
1479 UnavailablePred =
P;
1480 }
else if (!AvailableInfo.isValid()) {
1481 AvailableInfo = PredInfo;
1482 }
else if (AvailableInfo != PredInfo) {
1489 if (!UnavailablePred || !AvailableInfo.isValid())
1494 if (AvailableInfo.hasSEWLMULRatioOnly())
1504 if (AvailableInfo.hasAVLReg()) {
1505 const MachineInstr *AVLDefMI = &AvailableInfo.getAVLDefMI();
1508 if (AVLDefMI->
getParent() != UnavailablePred)
1510 for (
auto &TermMI : UnavailablePred->
terminators())
1511 if (&TermMI == AVLDefMI)
1517 if (AvailableInfo.hasAVLIgnored())
1529 VSETVLIInfo CurInfo = AvailableInfo;
1530 int TransitionsRemoved = 0;
1532 const VSETVLIInfo LastInfo = CurInfo;
1533 const VSETVLIInfo LastOldInfo = OldInfo;
1534 transferBefore(CurInfo,
MI);
1535 transferBefore(OldInfo,
MI);
1536 if (CurInfo == LastInfo)
1537 TransitionsRemoved++;
1538 if (LastOldInfo == OldInfo)
1539 TransitionsRemoved--;
1540 transferAfter(CurInfo,
MI);
1541 transferAfter(OldInfo,
MI);
1542 if (CurInfo == OldInfo)
1546 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1553 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1555 << UnavailablePred->
getName() <<
" with state "
1556 << AvailableInfo <<
"\n");
1557 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1563 insertVSETVLI(*UnavailablePred, InsertPt,
1565 AvailableInfo, OldExit);
1572 const DemandedFields &Used,
1577 if (!isVLPreservingConfig(
MI)) {
1581 if (Used.VLZeroness) {
1582 if (isVLPreservingConfig(PrevMI))
1584 if (!getInfoForVSETVLI(PrevMI,
MRI)
1585 .hasEquallyZeroAVL(getInfoForVSETVLI(
MI,
MRI)))
1589 auto &AVL =
MI.getOperand(1);
1594 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1595 (!
MRI.hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1596 PrevAVL.getReg() != AVL.getReg()))
1602 auto VType =
MI.getOperand(2).getImm();
1603 return areCompatibleVTYPEs(PriorVType, VType, Used);
1610 DemandedFields
Used;
1616 if (!isVectorConfigInstr(
MI)) {
1618 if (
MI.isCall() ||
MI.isInlineAsm() ||
1619 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1620 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1627 if (RegDef != RISCV::X0 && !
MRI->use_nodbg_empty(RegDef))
1631 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1638 if (!isVLPreservingConfig(*NextMI)) {
1641 MI.getOperand(0).setReg(DefReg);
1642 MI.getOperand(0).setIsDead(
false);
1652 DefVNI->
def = MISlot;
1658 LIS->shrinkToUses(&DefLI);
1662 if (
MI.getOperand(1).isReg())
1663 OldVLReg =
MI.getOperand(1).getReg();
1675 LIS->shrinkToUses(&LIS->getInterval(OldVLReg));
1678 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1679 MRI->use_nodbg_empty(OldVLReg)) {
1681 LIS->removeInterval(OldVLReg);
1695 NumCoalescedVSETVL += ToDelete.
size();
1696 for (
auto *
MI : ToDelete) {
1697 LIS->RemoveMachineInstrFromMaps(*
MI);
1698 MI->eraseFromParent();
1701 return !ToDelete.empty();
1708 Register VLOutput =
MI.getOperand(1).getReg();
1709 if (!
MRI->use_nodbg_empty(VLOutput))
1713 MI.getOperand(1).setReg(RISCV::X0);
1721 if (!
ST->hasVInstructions())
1726 TII =
ST->getInstrInfo();
1729 assert(BlockInfo.empty() &&
"Expect empty block infos");
1732 bool HaveVectorOp =
false;
1736 VSETVLIInfo TmpStatus;
1737 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1740 BBInfo.
Exit = TmpStatus;
1742 <<
" is " << BBInfo.
Exit <<
"\n");
1747 if (!HaveVectorOp) {
1756 WorkList.push(&
MBB);
1759 while (!WorkList.empty()) {
1762 computeIncomingVLVTYPE(
MBB);
1782 return HaveVectorOp;
1787 return new RISCVInsertVSETVLI();
1799 if (!ST->hasVInstructions())
1801 TII = ST->getInstrInfo();
1803 LIS = &getAnalysis<LiveIntervals>();
1805 bool Changed =
false;
1807 Changed |= coalesceVSETVLIs(
MBB);
1813 return new RISCVCoalesceVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
NewInfo setVTYPE(MI.getOperand(2).getImm())
#define RISCV_COALESCE_VSETVLI_NAME
assert((AVLReg !=RISCV::X0||MI.getOperand(0).getReg() !=RISCV::X0) &&"Can't handle X0, X0 vsetvli yet")
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const MachineRegisterInfo *MRI)
#define RISCV_INSERT_VSETVLI_NAME
else NewInfo setAVLRegDef(MRI.getVRegDef(AVLReg), AVLReg)
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI)
static cl::opt< bool > UseStrictAsserts("riscv-insert-vsetvl-strict-asserts", cl::init(true), cl::Hidden, cl::desc("Enable strict assertion checking for the dataflow algorithm"))
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME, false, false) char RISCVCoalesceVSETVLI const MachineRegisterInfo & MRI
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool isRegSequence() const
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
FunctionPass * createRISCVCoalesceVSETVLIPass()
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.