37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
50 cl::desc(
"Enable strict assertion checking for the dataflow algorithm"));
63 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
64 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
65 MI.getOpcode() == RISCV::PseudoVSETIVLI;
71 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
73 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
74 return RISCV::X0 ==
MI.getOperand(0).getReg();
77static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
102 case RISCV::VFMV_S_F:
113 case RISCV::VFMV_V_F:
122 case RISCV::VSLIDEDOWN_VX:
123 case RISCV::VSLIDEDOWN_VI:
124 case RISCV::VSLIDEUP_VX:
125 case RISCV::VSLIDEUP_VI:
132static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
142 case RISCV::VLSE16_V:
144 case RISCV::VSSE16_V:
147 case RISCV::VLSE32_V:
149 case RISCV::VSSE32_V:
152 case RISCV::VLSE64_V:
154 case RISCV::VSSE64_V:
160 return MI.getOpcode() == RISCV::ADDI &&
161 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
162 MI.getOperand(1).getReg() == RISCV::X0 &&
163 MI.getOperand(2).getImm() != 0;
171 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
184 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
193 if (UseMO.
getReg() == RISCV::NoRegister)
218struct DemandedFields {
223 bool VLZeroness =
false;
227 SEWGreaterThanOrEqual = 2,
229 SEWGreaterThanOrEqualAndLessThan64 =
236 bool SEWLMULRatio =
false;
237 bool TailPolicy =
false;
238 bool MaskPolicy =
false;
241 bool usedVTYPE()
const {
242 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
247 return VLAny || VLZeroness;
266 void doUnion(
const DemandedFields &
B) {
268 VLZeroness |=
B.VLZeroness;
269 SEW = std::max(SEW,
B.SEW);
271 SEWLMULRatio |=
B.SEWLMULRatio;
272 TailPolicy |=
B.TailPolicy;
273 MaskPolicy |=
B.MaskPolicy;
276#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
286 OS <<
"VLAny=" << VLAny <<
", ";
287 OS <<
"VLZeroness=" << VLZeroness <<
", ";
293 case SEWGreaterThanOrEqual:
294 OS <<
"SEWGreaterThanOrEqual";
296 case SEWGreaterThanOrEqualAndLessThan64:
297 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
304 OS <<
"LMUL=" <<
LMUL <<
", ";
305 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
306 OS <<
"TailPolicy=" << TailPolicy <<
", ";
307 OS <<
"MaskPolicy=" << MaskPolicy;
313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
325 const DemandedFields &Used) {
327 case DemandedFields::SEWNone:
329 case DemandedFields::SEWEqual:
333 case DemandedFields::SEWGreaterThanOrEqual:
337 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
348 if (
Used.SEWLMULRatio) {
353 if (Ratio1 != Ratio2)
378 if (
MI.isCall() ||
MI.isInlineAsm() ||
379 MI.readsRegister(RISCV::VL,
nullptr))
381 if (
MI.isCall() ||
MI.isInlineAsm() ||
382 MI.readsRegister(RISCV::VTYPE,
nullptr))
393 Res.MaskPolicy =
false;
402 if (getEEWForLoadStore(
MI)) {
403 Res.SEW = DemandedFields::SEWNone;
409 Res.TailPolicy =
false;
410 Res.MaskPolicy =
false;
417 if (isMaskRegOp(
MI)) {
418 Res.SEW = DemandedFields::SEWNone;
423 if (isScalarInsertInstr(
MI)) {
425 Res.SEWLMULRatio =
false;
433 if (hasUndefinedMergeOp(
MI, *
MRI)) {
434 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
435 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
437 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
438 Res.TailPolicy =
false;
443 if (isScalarExtractInstr(
MI)) {
446 Res.SEWLMULRatio =
false;
447 Res.TailPolicy =
false;
448 Res.MaskPolicy =
false;
474 uint8_t TailAgnostic : 1;
475 uint8_t MaskAgnostic : 1;
476 uint8_t SEWLMULRatioOnly : 1;
480 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
481 SEWLMULRatioOnly(
false) {}
483 static VSETVLIInfo getUnknown() {
490 void setUnknown() { State =
Unknown; }
491 bool isUnknown()
const {
return State ==
Unknown; }
499 void setAVLImm(
unsigned Imm) {
504 void setAVLVLMAX() { State = AVLIsVLMAX; }
506 void setAVLIgnored() { State = AVLIsIgnored; }
508 bool hasAVLImm()
const {
return State == AVLIsImm; }
509 bool hasAVLReg()
const {
return State == AVLIsReg; }
510 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
511 bool hasAVLIgnored()
const {
return State == AVLIsIgnored; }
516 unsigned getAVLImm()
const {
521 void setAVL(VSETVLIInfo Info) {
523 if (
Info.isUnknown())
525 else if (
Info.hasAVLReg())
526 setAVLReg(
Info.getAVLReg());
527 else if (
Info.hasAVLVLMAX())
529 else if (
Info.hasAVLIgnored())
533 setAVLImm(
Info.getAVLImm());
537 unsigned getSEW()
const {
return SEW; }
539 bool getTailAgnostic()
const {
return TailAgnostic; }
540 bool getMaskAgnostic()
const {
return MaskAgnostic; }
544 return getAVLImm() > 0;
548 return isNonZeroLoadImmediate(*
MI);
557 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other,
559 if (hasSameAVL(
Other))
561 return (hasNonZeroAVL(
MRI) &&
Other.hasNonZeroAVL(
MRI));
564 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
565 if (hasAVLReg() &&
Other.hasAVLReg())
566 return getAVLReg() ==
Other.getAVLReg();
568 if (hasAVLImm() &&
Other.hasAVLImm())
569 return getAVLImm() ==
Other.getAVLImm();
572 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
575 return Other.hasAVLIgnored();
580 void setVTYPE(
unsigned VType) {
582 "Can't set VTYPE for uninitialized or unknown");
590 "Can't set VTYPE for uninitialized or unknown");
601 "Can't encode VTYPE for uninitialized or unknown");
605 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
607 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
609 "Can't compare invalid VSETVLIInfos");
611 "Can't compare VTYPE in unknown state");
612 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
613 "Can't compare when only LMUL/SEW ratio is valid.");
614 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
621 "Can't use VTYPE for uninitialized or unknown");
629 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
631 "Can't compare invalid VSETVLIInfos");
633 "Can't compare VTYPE in unknown state");
637 bool hasCompatibleVTYPE(
const DemandedFields &Used,
638 const VSETVLIInfo &Require)
const {
639 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
645 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
648 "Can't compare invalid VSETVLIInfos");
649 assert(!Require.SEWLMULRatioOnly &&
650 "Expected a valid VTYPE for instruction!");
652 if (isUnknown() || Require.isUnknown())
656 if (SEWLMULRatioOnly)
659 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
662 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require,
MRI))
665 return hasCompatibleVTYPE(Used, Require);
671 return !
Other.isValid();
672 if (!
Other.isValid())
677 return Other.isUnknown();
678 if (
Other.isUnknown())
681 if (!hasSameAVL(
Other))
685 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
689 if (SEWLMULRatioOnly)
690 return hasSameVLMAX(
Other);
693 return hasSameVTYPE(
Other);
697 return !(*
this ==
Other);
704 if (!
Other.isValid())
712 if (isUnknown() ||
Other.isUnknown())
713 return VSETVLIInfo::getUnknown();
721 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
722 VSETVLIInfo MergeInfo = *
this;
723 MergeInfo.SEWLMULRatioOnly =
true;
728 return VSETVLIInfo::getUnknown();
731#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
743 OS <<
"Uninitialized";
749 OS <<
"AVLImm=" << (
unsigned)AVLImm;
755 <<
"VLMul=" << (
unsigned)VLMul <<
", "
756 <<
"SEW=" << (
unsigned)SEW <<
", "
757 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
758 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
759 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
764#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
782 bool InQueue =
false;
792 std::vector<BlockData> BlockInfo;
793 std::queue<const MachineBasicBlock *> WorkList;
809 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
810 const VSETVLIInfo &CurInfo)
const;
811 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
814 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
817 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
819 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
820 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
822 VSETVLIInfo &Info)
const;
861char RISCVInsertVSETVLI::ID = 0;
866char RISCVCoalesceVSETVLI::
ID = 0;
875 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
876 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
878 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
879 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
881 assert((AVLReg != RISCV::X0 ||
MI.getOperand(0).getReg() != RISCV::X0) &&
882 "Can't handle X0, X0 vsetvli yet");
883 if (AVLReg == RISCV::X0)
884 NewInfo.setAVLVLMAX();
886 NewInfo.setAVLReg(AVLReg);
888 NewInfo.setVTYPE(
MI.getOperand(2).getImm());
906 VSETVLIInfo InstrInfo;
908 bool TailAgnostic =
true;
909 bool MaskAgnostic =
true;
910 if (!hasUndefinedMergeOp(
MI, *
MRI)) {
912 TailAgnostic =
false;
913 MaskAgnostic =
false;
920 "Invalid Policy Value");
936 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
938 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
944 int64_t Imm = VLOp.
getImm();
949 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
950 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
951 InstrInfo.setAVLImm(VLMAX);
953 InstrInfo.setAVLVLMAX();
956 InstrInfo.setAVLImm(Imm);
958 InstrInfo.setAVLReg(VLOp.
getReg());
965 InstrInfo.setAVLIgnored();
968 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
969 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
972 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
978 if (InstrInfo.hasAVLReg()) {
981 if (isVectorConfigInstr(*
DefMI)) {
982 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI);
983 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
984 (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
985 InstrInfo.setAVL(DefInstrInfo);
993 const VSETVLIInfo &Info,
994 const VSETVLIInfo &PrevInfo) {
1001 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1003 ++NumInsertedVSETVL;
1004 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1007 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1019 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1022 if (isVectorConfigInstr(*
DefMI)) {
1023 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1024 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1036 if (
Info.hasAVLImm()) {
1044 if (
Info.hasAVLIgnored()) {
1047 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
1048 Info.hasSameVLMAX(PrevInfo)) {
1064 if (
Info.hasAVLVLMAX()) {
1065 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1074 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
1083 return Fractional || LMul == 1;
1089 const VSETVLIInfo &Require,
1090 const VSETVLIInfo &CurInfo)
const {
1093 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1096 DemandedFields
Used = getDemanded(
MI,
MRI, ST);
1106 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1109 Used.VLZeroness =
true;
1111 Used.TailPolicy =
false;
1119 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1122 Used.SEWLMULRatio =
false;
1124 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1125 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1127 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1128 Used.TailPolicy =
false;
1131 if (CurInfo.isCompatible(Used, Require, *
MRI))
1138 if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
1141 if (isVectorConfigInstr(*
DefMI)) {
1142 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1143 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1155 DemandedFields &Demanded) {
1156 VSETVLIInfo
Info = NewInfo;
1158 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1159 !PrevInfo.isUnknown()) {
1161 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1162 Info.setVLMul(*NewVLMul);
1163 Demanded.LMUL =
true;
1172void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1179 assert(NewInfo.isValid() && !NewInfo.isUnknown());
1180 if (
Info.isValid() && !needVSETVLI(
MI, NewInfo, Info))
1183 const VSETVLIInfo PrevInfo =
Info;
1184 if (!
Info.isValid() ||
Info.isUnknown())
1187 DemandedFields Demanded = getDemanded(
MI,
MRI, ST);
1188 const VSETVLIInfo IncomingInfo =
adjustIncoming(PrevInfo, NewInfo, Demanded);
1197 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, *
MRI) &&
1198 IncomingInfo.hasSameVLMAX(PrevInfo);
1199 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1200 Info.setAVL(IncomingInfo);
1203 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1205 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1208 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1209 IncomingInfo.getTailAgnostic(),
1210 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1211 IncomingInfo.getMaskAgnostic());
1215 if (
Info.hasSEWLMULRatioOnly()) {
1216 VSETVLIInfo RatiolessInfo = IncomingInfo;
1217 RatiolessInfo.setAVL(Info);
1218 Info = RatiolessInfo;
1225void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1227 if (isVectorConfigInstr(
MI)) {
1228 Info = getInfoForVSETVLI(
MI);
1234 Info.setAVLReg(
MI.getOperand(1).getReg());
1240 if (
MI.isCall() ||
MI.isInlineAsm() ||
1241 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1242 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1243 Info = VSETVLIInfo::getUnknown();
1247 VSETVLIInfo &Info)
const {
1248 bool HadVectorOp =
false;
1252 transferBefore(Info,
MI);
1257 transferAfter(Info,
MI);
1267 BBInfo.InQueue =
false;
1271 VSETVLIInfo InInfo = BBInfo.
Pred;
1274 InInfo.setUnknown();
1277 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1281 if (!InInfo.isValid())
1285 if (InInfo == BBInfo.
Pred)
1288 BBInfo.
Pred = InInfo;
1290 <<
" changed to " << BBInfo.
Pred <<
"\n");
1296 VSETVLIInfo TmpStatus;
1297 computeVLVTYPEChanges(
MBB, TmpStatus);
1301 if (BBInfo.
Exit == TmpStatus)
1304 BBInfo.
Exit = TmpStatus;
1306 <<
" changed to " << BBInfo.
Exit <<
"\n");
1311 if (!BlockInfo[S->getNumber()].InQueue) {
1312 BlockInfo[S->getNumber()].InQueue =
true;
1320bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1325 if (!Require.hasAVLReg())
1328 Register AVLReg = Require.getAVLReg();
1333 if (
PHI->getOpcode() != RISCV::PHI ||
PHI->getParent() != &
MBB)
1336 for (
unsigned PHIOp = 1, NumOps =
PHI->getNumOperands(); PHIOp != NumOps;
1340 const VSETVLIInfo &PBBExit = BlockInfo[PBB->
getNumber()].Exit;
1349 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1350 if (DefInfo != PBBExit)
1356 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1369 bool PrefixTransparent =
true;
1371 const VSETVLIInfo PrevInfo = CurInfo;
1372 transferBefore(CurInfo,
MI);
1375 if (isVectorConfigInstr(
MI)) {
1377 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1378 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1379 "Unexpected operands where VL and VTYPE should be");
1380 MI.getOperand(3).setIsDead(
false);
1381 MI.getOperand(4).setIsDead(
false);
1382 PrefixTransparent =
false;
1387 if (PrevInfo != CurInfo) {
1395 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1396 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1397 PrefixTransparent =
false;
1407 VLOp.
setReg(RISCV::NoRegister);
1414 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, Reg) &&
1415 MRI->use_nodbg_empty(Reg))
1425 if (
MI.isCall() ||
MI.isInlineAsm() ||
1426 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1427 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1428 PrefixTransparent =
false;
1430 transferAfter(CurInfo,
MI);
1436 const VSETVLIInfo &ExitInfo = BlockInfo[
MBB.
getNumber()].Exit;
1437 if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() &&
1438 CurInfo != ExitInfo) {
1450 if (CurInfo !=
Info.Exit) {
1457 "InsertVSETVLI dataflow invariant violated");
1471 VSETVLIInfo AvailableInfo;
1473 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1474 if (PredInfo.isUnknown()) {
1475 if (UnavailablePred)
1477 UnavailablePred =
P;
1478 }
else if (!AvailableInfo.isValid()) {
1479 AvailableInfo = PredInfo;
1480 }
else if (AvailableInfo != PredInfo) {
1487 if (!UnavailablePred || !AvailableInfo.isValid())
1492 if (AvailableInfo.hasSEWLMULRatioOnly())
1502 if (AvailableInfo.hasAVLReg()) {
1503 MachineInstr *AVLDefMI =
MRI->getUniqueVRegDef(AvailableInfo.getAVLReg());
1507 if (AVLDefMI->
getParent() != UnavailablePred)
1509 for (
auto &TermMI : UnavailablePred->
terminators())
1510 if (&TermMI == AVLDefMI)
1516 if (AvailableInfo.hasAVLIgnored())
1528 VSETVLIInfo CurInfo = AvailableInfo;
1529 int TransitionsRemoved = 0;
1531 const VSETVLIInfo LastInfo = CurInfo;
1532 const VSETVLIInfo LastOldInfo = OldInfo;
1533 transferBefore(CurInfo,
MI);
1534 transferBefore(OldInfo,
MI);
1535 if (CurInfo == LastInfo)
1536 TransitionsRemoved++;
1537 if (LastOldInfo == OldInfo)
1538 TransitionsRemoved--;
1539 transferAfter(CurInfo,
MI);
1540 transferAfter(OldInfo,
MI);
1541 if (CurInfo == OldInfo)
1545 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1552 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1554 << UnavailablePred->
getName() <<
" with state "
1555 << AvailableInfo <<
"\n");
1556 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1562 insertVSETVLI(*UnavailablePred, InsertPt,
1564 AvailableInfo, OldExit);
1571 const DemandedFields &Used,
1576 if (!isVLPreservingConfig(
MI)) {
1580 if (Used.VLZeroness) {
1581 if (isVLPreservingConfig(PrevMI))
1583 if (!getInfoForVSETVLI(PrevMI).hasEquallyZeroAVL(getInfoForVSETVLI(
MI),
1588 auto &AVL =
MI.getOperand(1);
1593 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1594 (!
MRI.hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1595 PrevAVL.getReg() != AVL.getReg()))
1601 auto VType =
MI.getOperand(2).getImm();
1602 return areCompatibleVTYPEs(PriorVType, VType, Used);
1609 DemandedFields
Used;
1615 if (!isVectorConfigInstr(
MI)) {
1617 if (
MI.isCall() ||
MI.isInlineAsm() ||
1618 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1619 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1626 if (RegDef != RISCV::X0 && !
MRI->use_nodbg_empty(RegDef))
1630 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1637 if (!isVLPreservingConfig(*NextMI)) {
1640 MI.getOperand(0).setReg(DefReg);
1641 MI.getOperand(0).setIsDead(
false);
1651 DefVNI->
def = MISlot;
1657 LIS->shrinkToUses(&DefLI);
1661 if (
MI.getOperand(1).isReg())
1662 OldVLReg =
MI.getOperand(1).getReg();
1674 LIS->shrinkToUses(&LIS->getInterval(OldVLReg));
1677 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1678 MRI->use_nodbg_empty(OldVLReg)) {
1680 LIS->removeInterval(OldVLReg);
1694 NumCoalescedVSETVL += ToDelete.
size();
1695 for (
auto *
MI : ToDelete) {
1696 LIS->RemoveMachineInstrFromMaps(*
MI);
1697 MI->eraseFromParent();
1700 return !ToDelete.empty();
1707 Register VLOutput =
MI.getOperand(1).getReg();
1708 if (!
MRI->use_nodbg_empty(VLOutput))
1712 MI.getOperand(1).setReg(RISCV::X0);
1720 if (!
ST->hasVInstructions())
1725 TII =
ST->getInstrInfo();
1728 assert(BlockInfo.empty() &&
"Expect empty block infos");
1731 bool HaveVectorOp =
false;
1735 VSETVLIInfo TmpStatus;
1736 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1739 BBInfo.
Exit = TmpStatus;
1741 <<
" is " << BBInfo.
Exit <<
"\n");
1746 if (!HaveVectorOp) {
1755 WorkList.push(&
MBB);
1758 while (!WorkList.empty()) {
1761 computeIncomingVLVTYPE(
MBB);
1781 return HaveVectorOp;
1786 return new RISCVInsertVSETVLI();
1798 if (!ST->hasVInstructions())
1800 TII = ST->getInstrInfo();
1802 LIS = &getAnalysis<LiveIntervals>();
1804 bool Changed =
false;
1806 Changed |= coalesceVSETVLIs(
MBB);
1812 return new RISCVCoalesceVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
#define RISCV_COALESCE_VSETVLI_NAME
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const MachineRegisterInfo *MRI)
#define RISCV_INSERT_VSETVLI_NAME
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI)
static cl::opt< bool > UseStrictAsserts("riscv-insert-vsetvl-strict-asserts", cl::init(true), cl::Hidden, cl::desc("Enable strict assertion checking for the dataflow algorithm"))
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool isRegSequence() const
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
FunctionPass * createRISCVCoalesceVSETVLIPass()
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.