35#define DEBUG_TYPE "riscv-insert-vsetvli"
36#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
38STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
39STATISTIC(NumRemovedVSETVL,
"Number of VSETVL inst removed");
43 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
47 cl::desc(
"Enable strict assertion checking for the dataflow algorithm"));
60 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
61 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
62 MI.getOpcode() == RISCV::PseudoVSETIVLI;
68 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
70 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
71 return RISCV::X0 ==
MI.getOperand(0).getReg();
74static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
110 case RISCV::VFMV_V_F:
119 case RISCV::VSLIDEDOWN_VX:
120 case RISCV::VSLIDEDOWN_VI:
121 case RISCV::VSLIDEUP_VX:
122 case RISCV::VSLIDEUP_VI:
129static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
139 case RISCV::VLSE16_V:
141 case RISCV::VSSE16_V:
144 case RISCV::VLSE32_V:
146 case RISCV::VSSE32_V:
149 case RISCV::VLSE64_V:
151 case RISCV::VSSE64_V:
157 return MI.getOpcode() == RISCV::ADDI &&
158 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
159 MI.getOperand(1).getReg() == RISCV::X0 &&
160 MI.getOperand(2).getImm() != 0;
168 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
181 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
190 if (UseMO.
getReg() == RISCV::NoRegister)
210struct DemandedFields {
215 bool VLZeroness =
false;
219 SEWGreaterThanOrEqual = 2,
221 SEWGreaterThanOrEqualAndLessThan64 =
228 bool SEWLMULRatio =
false;
229 bool TailPolicy =
false;
230 bool MaskPolicy =
false;
233 bool usedVTYPE()
const {
234 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
239 return VLAny || VLZeroness;
257#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
267 OS <<
"VLAny=" << VLAny <<
", ";
268 OS <<
"VLZeroness=" << VLZeroness <<
", ";
274 case SEWGreaterThanOrEqual:
275 OS <<
"SEWGreaterThanOrEqual";
277 case SEWGreaterThanOrEqualAndLessThan64:
278 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
285 OS <<
"LMUL=" <<
LMUL <<
", ";
286 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
287 OS <<
"TailPolicy=" << TailPolicy <<
", ";
288 OS <<
"MaskPolicy=" << MaskPolicy;
294#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
306 const DemandedFields &Used) {
308 case DemandedFields::SEWNone:
310 case DemandedFields::SEWEqual:
314 case DemandedFields::SEWGreaterThanOrEqual:
318 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
329 if (
Used.SEWLMULRatio) {
334 if (Ratio1 != Ratio2)
359 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.readsRegister(RISCV::VL))
361 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.readsRegister(RISCV::VTYPE))
372 Res.MaskPolicy =
false;
381 if (getEEWForLoadStore(
MI)) {
382 Res.SEW = DemandedFields::SEWNone;
388 Res.TailPolicy =
false;
389 Res.MaskPolicy =
false;
396 if (isMaskRegOp(
MI)) {
397 Res.SEW = DemandedFields::SEWNone;
402 if (isScalarInsertInstr(
MI)) {
404 Res.SEWLMULRatio =
false;
412 if (hasUndefinedMergeOp(
MI, *
MRI)) {
413 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
414 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
416 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
417 Res.TailPolicy =
false;
422 if (isScalarExtractInstr(
MI)) {
425 Res.SEWLMULRatio =
false;
426 Res.TailPolicy =
false;
427 Res.MaskPolicy =
false;
451 uint8_t TailAgnostic : 1;
452 uint8_t MaskAgnostic : 1;
453 uint8_t SEWLMULRatioOnly : 1;
457 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
458 SEWLMULRatioOnly(
false) {}
460 static VSETVLIInfo getUnknown() {
467 void setUnknown() { State =
Unknown; }
468 bool isUnknown()
const {
return State ==
Unknown; }
471 assert(
Reg.isVirtual() || Reg == RISCV::X0 || Reg == RISCV::NoRegister);
476 void setAVLImm(
unsigned Imm) {
481 bool hasAVLImm()
const {
return State == AVLIsImm; }
482 bool hasAVLReg()
const {
return State == AVLIsReg; }
487 unsigned getAVLImm()
const {
492 void setAVL(VSETVLIInfo Info) {
494 if (
Info.isUnknown())
496 else if (
Info.hasAVLReg())
497 setAVLReg(
Info.getAVLReg());
500 setAVLImm(
Info.getAVLImm());
504 unsigned getSEW()
const {
return SEW; }
506 bool getTailAgnostic()
const {
return TailAgnostic; }
507 bool getMaskAgnostic()
const {
return MaskAgnostic; }
511 return getAVLImm() > 0;
513 if (getAVLReg() == RISCV::X0)
516 MI && isNonZeroLoadImmediate(*
MI))
523 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other,
525 if (hasSameAVL(
Other))
527 return (hasNonZeroAVL(
MRI) &&
Other.hasNonZeroAVL(
MRI));
530 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
531 if (hasAVLReg() &&
Other.hasAVLReg())
532 return getAVLReg() ==
Other.getAVLReg();
534 if (hasAVLImm() &&
Other.hasAVLImm())
535 return getAVLImm() ==
Other.getAVLImm();
540 void setVTYPE(
unsigned VType) {
542 "Can't set VTYPE for uninitialized or unknown");
550 "Can't set VTYPE for uninitialized or unknown");
561 "Can't encode VTYPE for uninitialized or unknown");
565 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
567 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
569 "Can't compare invalid VSETVLIInfos");
571 "Can't compare VTYPE in unknown state");
572 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
573 "Can't compare when only LMUL/SEW ratio is valid.");
574 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
581 "Can't use VTYPE for uninitialized or unknown");
589 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
591 "Can't compare invalid VSETVLIInfos");
593 "Can't compare VTYPE in unknown state");
597 bool hasCompatibleVTYPE(
const DemandedFields &Used,
598 const VSETVLIInfo &Require)
const {
599 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
605 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
608 "Can't compare invalid VSETVLIInfos");
609 assert(!Require.SEWLMULRatioOnly &&
610 "Expected a valid VTYPE for instruction!");
612 if (isUnknown() || Require.isUnknown())
616 if (SEWLMULRatioOnly)
619 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
622 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require,
MRI))
625 return hasCompatibleVTYPE(Used, Require);
631 return !
Other.isValid();
632 if (!
Other.isValid())
637 return Other.isUnknown();
638 if (
Other.isUnknown())
641 if (!hasSameAVL(
Other))
645 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
649 if (SEWLMULRatioOnly)
650 return hasSameVLMAX(
Other);
653 return hasSameVTYPE(
Other);
657 return !(*
this ==
Other);
664 if (!
Other.isValid())
672 if (isUnknown() ||
Other.isUnknown())
673 return VSETVLIInfo::getUnknown();
681 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
682 VSETVLIInfo MergeInfo = *
this;
683 MergeInfo.SEWLMULRatioOnly =
true;
688 return VSETVLIInfo::getUnknown();
691#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
703 OS <<
"Uninitialized";
709 OS <<
"AVLImm=" << (
unsigned)AVLImm;
711 <<
"VLMul=" << (
unsigned)VLMul <<
", "
712 <<
"SEW=" << (
unsigned)SEW <<
", "
713 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
714 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
715 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
720#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
738 bool InQueue =
false;
748 std::vector<BlockData> BlockInfo;
749 std::queue<const MachineBasicBlock *> WorkList;
765 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
766 const VSETVLIInfo &CurInfo)
const;
767 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
770 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
773 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
775 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
776 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
778 VSETVLIInfo &Info)
const;
788char RISCVInsertVSETVLI::ID = 0;
797 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
798 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
800 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
801 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
803 assert((AVLReg != RISCV::X0 ||
MI.getOperand(0).getReg() != RISCV::X0) &&
804 "Can't handle X0, X0 vsetvli yet");
805 NewInfo.setAVLReg(AVLReg);
807 NewInfo.setVTYPE(
MI.getOperand(2).getImm());
825 VSETVLIInfo InstrInfo;
827 bool TailAgnostic =
true;
828 bool MaskAgnostic =
true;
829 if (!hasUndefinedMergeOp(
MI, *
MRI)) {
831 TailAgnostic =
false;
832 MaskAgnostic =
false;
839 "Invalid Policy Value");
855 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
857 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
863 int64_t Imm = VLOp.
getImm();
868 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
869 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
870 InstrInfo.setAVLImm(VLMAX);
872 InstrInfo.setAVLReg(RISCV::X0);
875 InstrInfo.setAVLImm(Imm);
877 InstrInfo.setAVLReg(VLOp.
getReg());
881 InstrInfo.setAVLReg(RISCV::NoRegister);
884 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
885 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
888 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
894 if (InstrInfo.hasAVLReg() && InstrInfo.getAVLReg().isVirtual()) {
897 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI);
898 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
899 (DefInstrInfo.hasAVLImm() || DefInstrInfo.getAVLReg() == RISCV::X0)) {
900 InstrInfo.setAVL(DefInstrInfo);
909 const VSETVLIInfo &Info,
910 const VSETVLIInfo &PrevInfo) {
917 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
920 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
923 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
935 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg() &&
936 Info.getAVLReg().isVirtual()) {
938 if (isVectorConfigInstr(*
DefMI)) {
939 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
940 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
953 if (
Info.hasAVLImm()) {
962 if (AVLReg == RISCV::NoRegister) {
965 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
966 Info.hasSameVLMAX(PrevInfo)) {
983 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
989 unsigned Opcode = RISCV::PseudoVSETVLI;
990 if (AVLReg == RISCV::X0) {
991 DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
992 Opcode = RISCV::PseudoVSETVLIX0;
1002 return Fractional || LMul == 1;
1008 const VSETVLIInfo &Require,
1009 const VSETVLIInfo &CurInfo)
const {
1012 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1015 DemandedFields
Used = getDemanded(
MI,
MRI, ST);
1025 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1028 Used.VLZeroness =
true;
1030 Used.TailPolicy =
false;
1038 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1041 Used.SEWLMULRatio =
false;
1043 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1044 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1046 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1047 Used.TailPolicy =
false;
1050 if (CurInfo.isCompatible(Used, Require, *
MRI))
1057 if (Require.hasAVLReg() && Require.getAVLReg().isVirtual() &&
1058 CurInfo.hasCompatibleVTYPE(Used, Require)) {
1060 if (isVectorConfigInstr(*
DefMI)) {
1061 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1062 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1075 DemandedFields &Demanded) {
1076 VSETVLIInfo
Info = NewInfo;
1078 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1079 !PrevInfo.isUnknown()) {
1081 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1082 Info.setVLMul(*NewVLMul);
1083 Demanded.LMUL =
true;
1092void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1099 assert(NewInfo.isValid() && !NewInfo.isUnknown());
1100 if (
Info.isValid() && !needVSETVLI(
MI, NewInfo, Info))
1103 const VSETVLIInfo PrevInfo =
Info;
1104 if (!
Info.isValid() ||
Info.isUnknown())
1107 DemandedFields Demanded = getDemanded(
MI,
MRI, ST);
1108 const VSETVLIInfo IncomingInfo =
adjustIncoming(PrevInfo, NewInfo, Demanded);
1117 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, *
MRI) &&
1118 IncomingInfo.hasSameVLMAX(PrevInfo);
1119 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1120 Info.setAVL(IncomingInfo);
1123 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1125 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1128 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1129 IncomingInfo.getTailAgnostic(),
1130 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1131 IncomingInfo.getMaskAgnostic());
1135 if (
Info.hasSEWLMULRatioOnly()) {
1136 VSETVLIInfo RatiolessInfo = IncomingInfo;
1137 RatiolessInfo.setAVL(Info);
1138 Info = RatiolessInfo;
1145void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1147 if (isVectorConfigInstr(
MI)) {
1148 Info = getInfoForVSETVLI(
MI);
1154 Info.setAVLReg(
MI.getOperand(1).getReg());
1160 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.modifiesRegister(RISCV::VL) ||
1161 MI.modifiesRegister(RISCV::VTYPE))
1162 Info = VSETVLIInfo::getUnknown();
1166 VSETVLIInfo &Info)
const {
1167 bool HadVectorOp =
false;
1171 transferBefore(Info,
MI);
1176 transferAfter(Info,
MI);
1186 BBInfo.InQueue =
false;
1190 VSETVLIInfo InInfo = BBInfo.
Pred;
1193 InInfo.setUnknown();
1196 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1200 if (!InInfo.isValid())
1204 if (InInfo == BBInfo.
Pred)
1207 BBInfo.
Pred = InInfo;
1209 <<
" changed to " << BBInfo.
Pred <<
"\n");
1215 VSETVLIInfo TmpStatus;
1216 computeVLVTYPEChanges(
MBB, TmpStatus);
1220 if (BBInfo.
Exit == TmpStatus)
1223 BBInfo.
Exit = TmpStatus;
1225 <<
" changed to " << BBInfo.
Exit <<
"\n");
1230 if (!BlockInfo[S->getNumber()].InQueue) {
1231 BlockInfo[S->getNumber()].InQueue =
true;
1239bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1244 if (!Require.hasAVLReg())
1247 Register AVLReg = Require.getAVLReg();
1253 if (!
PHI ||
PHI->getOpcode() != RISCV::PHI ||
PHI->getParent() != &
MBB)
1256 for (
unsigned PHIOp = 1, NumOps =
PHI->getNumOperands(); PHIOp != NumOps;
1263 if (PBBInfo.
Exit.isUnknown() || !PBBInfo.
Exit.hasSameVTYPE(Require))
1273 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1274 if (!DefInfo.hasSameAVL(PBBInfo.
Exit) ||
1275 !DefInfo.hasSameVTYPE(PBBInfo.
Exit))
1288 bool PrefixTransparent =
true;
1290 const VSETVLIInfo PrevInfo = CurInfo;
1291 transferBefore(CurInfo,
MI);
1294 if (isVectorConfigInstr(
MI)) {
1296 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1297 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1298 "Unexpected operands where VL and VTYPE should be");
1299 MI.getOperand(3).setIsDead(
false);
1300 MI.getOperand(4).setIsDead(
false);
1301 PrefixTransparent =
false;
1306 if (PrevInfo != CurInfo) {
1314 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1315 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1316 PrefixTransparent =
false;
1326 VLOp.
setReg(RISCV::NoRegister);
1333 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, Reg) &&
1334 MRI->use_nodbg_empty(Reg))
1344 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.modifiesRegister(RISCV::VL) ||
1345 MI.modifiesRegister(RISCV::VTYPE))
1346 PrefixTransparent =
false;
1348 transferAfter(CurInfo,
MI);
1354 const VSETVLIInfo &ExitInfo = BlockInfo[
MBB.
getNumber()].Exit;
1355 if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() &&
1356 CurInfo != ExitInfo) {
1368 if (CurInfo !=
Info.Exit) {
1375 "InsertVSETVLI dataflow invariant violated");
1389 VSETVLIInfo AvailableInfo;
1391 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1392 if (PredInfo.isUnknown()) {
1393 if (UnavailablePred)
1395 UnavailablePred =
P;
1396 }
else if (!AvailableInfo.isValid()) {
1397 AvailableInfo = PredInfo;
1398 }
else if (AvailableInfo != PredInfo) {
1405 if (!UnavailablePred || !AvailableInfo.isValid())
1410 if (AvailableInfo.hasSEWLMULRatioOnly())
1420 if (AvailableInfo.hasAVLReg() && RISCV::X0 != AvailableInfo.getAVLReg()) {
1426 if (AVLDefMI->
getParent() != UnavailablePred)
1428 for (
auto &TermMI : UnavailablePred->
terminators())
1429 if (&TermMI == AVLDefMI)
1442 VSETVLIInfo CurInfo = AvailableInfo;
1443 int TransitionsRemoved = 0;
1445 const VSETVLIInfo LastInfo = CurInfo;
1446 const VSETVLIInfo LastOldInfo = OldInfo;
1447 transferBefore(CurInfo,
MI);
1448 transferBefore(OldInfo,
MI);
1449 if (CurInfo == LastInfo)
1450 TransitionsRemoved++;
1451 if (LastOldInfo == OldInfo)
1452 TransitionsRemoved--;
1453 transferAfter(CurInfo,
MI);
1454 transferAfter(OldInfo,
MI);
1455 if (CurInfo == OldInfo)
1459 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1466 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1468 << UnavailablePred->
getName() <<
" with state "
1469 << AvailableInfo <<
"\n");
1470 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1476 insertVSETVLI(*UnavailablePred, InsertPt,
1478 AvailableInfo, OldExit);
1483 A.VLZeroness |=
B.VLZeroness;
1484 A.SEW = std::max(
A.SEW,
B.SEW);
1486 A.SEWLMULRatio |=
B.SEWLMULRatio;
1487 A.TailPolicy |=
B.TailPolicy;
1488 A.MaskPolicy |=
B.MaskPolicy;
1495 const DemandedFields &Used,
1500 if (!isVLPreservingConfig(
MI)) {
1504 if (Used.VLZeroness) {
1505 if (isVLPreservingConfig(PrevMI))
1507 if (!getInfoForVSETVLI(PrevMI).hasEquallyZeroAVL(getInfoForVSETVLI(
MI),
1512 auto &AVL =
MI.getOperand(1);
1518 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1519 (!PrevAVL.isReg() || PrevAVL.getReg() != AVL.getReg()))
1525 auto VType =
MI.getOperand(2).getImm();
1526 return areCompatibleVTYPEs(PriorVType, VType, Used);
1533 DemandedFields
Used;
1539 if (!isVectorConfigInstr(
MI)) {
1541 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.modifiesRegister(RISCV::VL) ||
1542 MI.modifiesRegister(RISCV::VTYPE))
1549 if (RegDef != RISCV::X0 && !
MRI->use_nodbg_empty(RegDef))
1553 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1560 if (!isVLPreservingConfig(*NextMI)) {
1562 MI.getOperand(0).setIsDead(
false);
1564 if (
MI.getOperand(1).isReg())
1565 OldVLReg =
MI.getOperand(1).getReg();
1572 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1573 MRI->use_nodbg_empty(OldVLReg))
1587 NumRemovedVSETVL += ToDelete.
size();
1588 for (
auto *
MI : ToDelete)
1589 MI->eraseFromParent();
1596 Register VLOutput =
MI.getOperand(1).getReg();
1597 if (!
MRI->use_nodbg_empty(VLOutput))
1601 MI.getOperand(1).setReg(RISCV::X0);
1609 if (!
ST->hasVInstructions())
1614 TII =
ST->getInstrInfo();
1617 assert(BlockInfo.empty() &&
"Expect empty block infos");
1620 bool HaveVectorOp =
false;
1624 VSETVLIInfo TmpStatus;
1625 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1628 BBInfo.
Exit = TmpStatus;
1630 <<
" is " << BBInfo.
Exit <<
"\n");
1635 if (!HaveVectorOp) {
1644 WorkList.push(&
MBB);
1647 while (!WorkList.empty()) {
1650 computeIncomingVLVTYPE(
MBB);
1671 doLocalPostpass(
MBB);
1679 return HaveVectorOp;
1684 return new RISCVInsertVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const MachineRegisterInfo *MRI)
#define RISCV_INSERT_VSETVLI_NAME
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static void doUnion(DemandedFields &A, DemandedFields B)
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI)
static cl::opt< bool > UseStrictAsserts("riscv-insert-vsetvl-strict-asserts", cl::init(true), cl::Hidden, cl::desc("Enable strict assertion checking for the dataflow algorithm"))
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
bool isImplicitDef() const
const MachineBasicBlock * getParent() const
unsigned getNumOperands() const
Retuns the total number of operands.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
bool isRegSequence() const
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const