34 #define DEBUG_TYPE "riscv-insert-vsetvli"
35 #define RISCV_INSERT_VSETVLI_NAME "RISCV Insert VSETVLI pass"
39 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
43 cl::desc(
"Enable strict assertion checking for the dataflow algorithm"));
56 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
57 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
58 MI.getOpcode() == RISCV::PseudoVSETIVLI;
64 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
66 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
67 return RISCV::X0 ==
MI.getOperand(0).getReg();
72 RISCVVPseudosTable::getPseudoInfo(RVVPseudoOpcode);
79 switch (getRVVMCOpcode(
MI.getOpcode())) {
90 static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
91 switch (getRVVMCOpcode(
MI.getOpcode())) {
100 case RISCV::VLSE16_V:
102 case RISCV::VSSE16_V:
105 case RISCV::VLSE32_V:
107 case RISCV::VSSE32_V:
110 case RISCV::VLSE64_V:
112 case RISCV::VSSE64_V:
128 struct DemandedFields {
133 bool VLZeroness =
false;
136 bool SEWLMULRatio =
false;
137 bool TailPolicy =
false;
138 bool MaskPolicy =
false;
141 bool usedVTYPE()
const {
142 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
147 return VLAny || VLZeroness;
165 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
175 OS <<
"VLAny=" << VLAny <<
", ";
176 OS <<
"VLZeroness=" << VLZeroness <<
", ";
177 OS <<
"SEW=" <<
SEW <<
", ";
178 OS <<
"LMUL=" <<
LMUL <<
", ";
179 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
180 OS <<
"TailPolicy=" << TailPolicy <<
", ";
181 OS <<
"MaskPolicy=" << MaskPolicy;
187 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
199 static bool areCompatibleVTYPEs(
uint64_t VType1,
201 const DemandedFields &Used) {
210 if (
Used.SEWLMULRatio) {
215 if (Ratio1 != Ratio2)
219 if (
Used.TailPolicy &&
222 if (
Used.MaskPolicy &&
238 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.readsRegister(RISCV::VL))
240 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.readsRegister(RISCV::VTYPE))
251 Res.MaskPolicy =
false;
260 if (getEEWForLoadStore(
MI)) {
267 Res.TailPolicy =
false;
268 Res.MaskPolicy =
false;
275 if (isMaskRegOp(
MI)) {
281 if (isScalarMoveInstr(
MI)) {
283 Res.SEWLMULRatio =
false;
310 uint8_t SEWLMULRatioOnly : 1;
315 SEWLMULRatioOnly(
false) {}
317 static VSETVLIInfo getUnknown() {
324 void setUnknown() { State =
Unknown; }
325 bool isUnknown()
const {
return State ==
Unknown; }
332 void setAVLImm(
unsigned Imm) {
337 bool hasAVLImm()
const {
return State == AVLIsImm; }
338 bool hasAVLReg()
const {
return State == AVLIsReg; }
343 unsigned getAVLImm()
const {
353 return getAVLImm() > 0;
355 return getAVLReg() == RISCV::X0;
359 bool hasEquallyZeroAVL(
const VSETVLIInfo &Other)
const {
360 if (hasSameAVL(Other))
365 bool hasSameAVL(
const VSETVLIInfo &Other)
const {
366 if (hasAVLReg() &&
Other.hasAVLReg())
367 return getAVLReg() ==
Other.getAVLReg();
369 if (hasAVLImm() &&
Other.hasAVLImm())
370 return getAVLImm() ==
Other.getAVLImm();
377 "Can't set VTYPE for uninitialized or unknown");
385 "Can't set VTYPE for uninitialized or unknown");
394 "Can't encode VTYPE for uninitialized or unknown");
398 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
400 bool hasSameVTYPE(
const VSETVLIInfo &Other)
const {
402 "Can't compare invalid VSETVLIInfos");
404 "Can't compare VTYPE in unknown state");
405 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
406 "Can't compare when only LMUL/SEW ratio is valid.");
414 "Can't use VTYPE for uninitialized or unknown");
422 bool hasSameVLMAX(
const VSETVLIInfo &Other)
const {
424 "Can't compare invalid VSETVLIInfos");
426 "Can't compare VTYPE in unknown state");
430 bool hasCompatibleVTYPE(
const DemandedFields &Used,
431 const VSETVLIInfo &Require)
const {
432 return areCompatibleVTYPEs(
encodeVTYPE(), Require.encodeVTYPE(), Used);
438 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require)
const {
440 "Can't compare invalid VSETVLIInfos");
441 assert(!Require.SEWLMULRatioOnly &&
442 "Expected a valid VTYPE for instruction!");
444 if (isUnknown() || Require.isUnknown())
448 if (SEWLMULRatioOnly)
453 if (Require.hasAVLReg() && Require.AVLReg == RISCV::NoRegister)
454 if (
SEW == Require.SEW)
457 if (
Used.VLAny && !hasSameAVL(Require))
460 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require))
463 return areCompatibleVTYPEs(
encodeVTYPE(), Require.encodeVTYPE(), Used);
466 bool operator==(
const VSETVLIInfo &Other)
const {
469 return !
Other.isValid();
470 if (!
Other.isValid())
475 return Other.isUnknown();
476 if (
Other.isUnknown())
479 if (!hasSameAVL(Other))
483 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
487 if (SEWLMULRatioOnly)
488 return hasSameVLMAX(Other);
491 return hasSameVTYPE(Other);
494 bool operator!=(
const VSETVLIInfo &Other)
const {
495 return !(*
this ==
Other);
500 VSETVLIInfo
intersect(
const VSETVLIInfo &Other)
const {
502 if (!
Other.isValid())
510 if (isUnknown() ||
Other.isUnknown())
511 return VSETVLIInfo::getUnknown();
519 if (hasSameAVL(Other) && hasSameVLMAX(Other)) {
520 VSETVLIInfo MergeInfo = *
this;
521 MergeInfo.SEWLMULRatioOnly =
true;
526 return VSETVLIInfo::getUnknown();
529 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
541 OS <<
"Uninitialized";
545 OS <<
"AVLReg=" << (unsigned)AVLReg;
547 OS <<
"AVLImm=" << (
unsigned)AVLImm;
549 <<
"VLMul=" << (unsigned)
VLMul <<
", "
550 <<
"SEW=" << (
unsigned)
SEW <<
", "
553 <<
"SEWLMULRatioOnly=" << (bool)SEWLMULRatioOnly <<
"}";
558 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
580 bool InQueue =
false;
582 BlockData() =
default;
589 std::vector<BlockData> BlockInfo;
590 std::queue<const MachineBasicBlock *> WorkList;
608 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
609 const VSETVLIInfo &CurInfo)
const;
610 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
613 const VSETVLIInfo &
Info,
const VSETVLIInfo &PrevInfo);
616 const VSETVLIInfo &
Info,
const VSETVLIInfo &PrevInfo);
651 "Invalid Policy Value");
702 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
703 assert(
SEW == EEW &&
"Initial SEW doesn't match expected EEW");
712 const VSETVLIInfo &
Info,
713 const VSETVLIInfo &PrevInfo) {
720 const VSETVLIInfo &
Info,
const VSETVLIInfo &PrevInfo) {
724 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
725 Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
734 if (
Info.hasAVLImm()) {
743 if (AVLReg == RISCV::NoRegister) {
746 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
747 Info.hasSameVLMAX(PrevInfo)) {
764 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
770 unsigned Opcode = RISCV::PseudoVSETVLI;
771 if (AVLReg == RISCV::X0) {
772 DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
773 Opcode = RISCV::PseudoVSETVLIX0;
785 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
786 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
788 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
789 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
791 assert((AVLReg != RISCV::X0 ||
MI.getOperand(0).getReg() != RISCV::X0) &&
792 "Can't handle X0, X0 vsetvli yet");
793 NewInfo.setAVLReg(AVLReg);
795 NewInfo.setVTYPE(
MI.getOperand(2).getImm());
803 const VSETVLIInfo &Require,
804 const VSETVLIInfo &CurInfo)
const {
805 assert(Require == computeInfoForInstr(
MI,
MI.getDesc().TSFlags,
MRI));
807 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
810 DemandedFields
Used = getDemanded(
MI);
812 if (isScalarMoveInstr(
MI)) {
819 auto *VRegDef =
MRI->getVRegDef(
MI.getOperand(1).getReg());
820 if (VRegDef && VRegDef->isImplicitDef() &&
821 CurInfo.getSEW() >= Require.getSEW()) {
823 Used.TailPolicy =
false;
827 if (CurInfo.isCompatible(Used, Require))
834 if (Require.hasAVLReg() && Require.getAVLReg().isVirtual() &&
835 CurInfo.hasCompatibleVTYPE(Used, Require)) {
837 if (isVectorConfigInstr(*
DefMI)) {
839 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
856 const VSETVLIInfo NewInfo = computeInfoForInstr(
MI,
TSFlags,
MRI);
857 if (
Info.isValid() && !needVSETVLI(
MI, NewInfo,
Info))
860 const VSETVLIInfo PrevInfo =
Info;
874 if (isScalarMoveInstr(
MI) && PrevInfo.isValid() &&
875 PrevInfo.hasEquallyZeroAVL(
Info) &&
876 Info.hasSameVLMAX(PrevInfo)) {
877 if (PrevInfo.hasAVLImm())
878 Info.setAVLImm(PrevInfo.getAVLImm());
880 Info.setAVLReg(PrevInfo.getAVLReg());
888 if (!
Info.hasAVLReg() || !
Info.getAVLReg().isVirtual())
895 if (DefInfo.hasSameVLMAX(
Info) &&
896 (DefInfo.hasAVLImm() || DefInfo.getAVLReg() == RISCV::X0)) {
897 if (DefInfo.hasAVLImm())
898 Info.setAVLImm(DefInfo.getAVLImm());
900 Info.setAVLReg(DefInfo.getAVLReg());
909 if (isVectorConfigInstr(
MI)) {
916 Info.setAVLReg(
MI.getOperand(1).getReg());
922 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.modifiesRegister(RISCV::VL) ||
923 MI.modifiesRegister(RISCV::VTYPE))
924 Info = VSETVLIInfo::getUnknown();
928 bool HadVectorOp =
false;
931 BBInfo.Change = BBInfo.Pred;
933 transferBefore(BBInfo.Change,
MI);
938 transferAfter(BBInfo.Change,
MI);
948 BBInfo.InQueue =
false;
952 VSETVLIInfo InInfo = BBInfo.Pred;
958 InInfo = InInfo.intersect(BlockInfo[
P->getNumber()].Exit);
962 if (!InInfo.isValid())
966 if (InInfo == BBInfo.Pred)
969 BBInfo.Pred = InInfo;
971 <<
" changed to " << BBInfo.Pred <<
"\n");
977 computeVLVTYPEChanges(
MBB);
978 VSETVLIInfo TmpStatus = BBInfo.Change;
982 if (BBInfo.Exit == TmpStatus)
985 BBInfo.Exit = TmpStatus;
987 <<
" changed to " << BBInfo.Exit <<
"\n");
992 if (!BlockInfo[
S->getNumber()].InQueue) {
993 BlockInfo[
S->getNumber()].InQueue =
true;
1001 bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1006 if (!Require.hasAVLReg())
1009 Register AVLReg = Require.getAVLReg();
1018 for (
unsigned PHIOp = 1, NumOps =
PHI->getNumOperands(); PHIOp != NumOps;
1022 const BlockData &PBBInfo = BlockInfo[PBB->
getNumber()];
1025 if (PBBInfo.Exit.isUnknown() || !PBBInfo.Exit.hasSameVTYPE(Require))
1036 if (!DefInfo.hasSameAVL(PBBInfo.Exit) ||
1037 !DefInfo.hasSameVTYPE(PBBInfo.Exit))
1050 bool PrefixTransparent =
true;
1052 const VSETVLIInfo PrevInfo = CurInfo;
1053 transferBefore(CurInfo,
MI);
1056 if (isVectorConfigInstr(
MI)) {
1058 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1059 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1060 "Unexpected operands where VL and VTYPE should be");
1061 MI.getOperand(3).setIsDead(
false);
1062 MI.getOperand(4).setIsDead(
false);
1063 PrefixTransparent =
false;
1068 if (PrevInfo != CurInfo) {
1076 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1077 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1078 PrefixTransparent =
false;
1085 VLOp.
setReg(RISCV::NoRegister);
1095 if (
MI.isCall() ||
MI.isInlineAsm() ||
MI.modifiesRegister(RISCV::VL) ||
1096 MI.modifiesRegister(RISCV::VTYPE))
1097 PrefixTransparent =
false;
1099 transferAfter(CurInfo,
MI);
1105 const VSETVLIInfo &ExitInfo = BlockInfo[
MBB.
getNumber()].Exit;
1106 if (CurInfo.isValid() && ExitInfo.isValid() && !ExitInfo.isUnknown() &&
1107 CurInfo != ExitInfo) {
1119 if (CurInfo !=
Info.Exit) {
1126 "InsertVSETVLI dataflow invariant violated");
1132 if (!
Info.hasAVLImm())
1136 return RISCV::X0 ==
Info.getAVLReg();
1138 unsigned AVL =
Info.getAVLImm();
1140 unsigned AVLInBits = AVL *
SEW;
1147 return ST.getRealMinVLen() / LMul >= AVLInBits;
1148 return ST.getRealMinVLen() * LMul >= AVLInBits;
1164 VSETVLIInfo AvailableInfo;
1166 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1167 if (PredInfo.isUnknown()) {
1168 if (UnavailablePred)
1170 UnavailablePred =
P;
1171 }
else if (!AvailableInfo.isValid()) {
1172 AvailableInfo = PredInfo;
1173 }
else if (AvailableInfo != PredInfo) {
1180 if (!UnavailablePred || !AvailableInfo.isValid())
1193 for (
auto &
MI :
MBB) {
1194 if (isVectorConfigInstr(
MI))
1199 if (AvailableInfo != computeInfoForInstr(
MI,
TSFlags,
MRI))
1211 auto OldInfo = BlockInfo[UnavailablePred->
getNumber()].Exit;
1213 << UnavailablePred->
getName() <<
" with state "
1214 << AvailableInfo <<
"\n");
1215 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1221 insertVSETVLI(*UnavailablePred, InsertPt,
1223 AvailableInfo, OldInfo);
1226 static void doUnion(DemandedFields &A, DemandedFields
B) {
1228 A.VLZeroness |=
B.VLZeroness;
1231 A.SEWLMULRatio |=
B.SEWLMULRatio;
1232 A.TailPolicy |=
B.TailPolicy;
1233 A.MaskPolicy |=
B.MaskPolicy;
1238 return RISCV::X0 == MO.
getReg();
1247 const DemandedFields &Used) {
1251 if (!isVLPreservingConfig(
MI)) {
1256 if (isVLPreservingConfig(PrevMI))
1261 if (Used.VLZeroness &&
1268 if (
MI.getOperand(1).isReg() &&
1269 RISCV::X0 !=
MI.getOperand(1).getReg())
1276 MI.getOperand(1).isReg())
1284 auto VType =
MI.getOperand(2).getImm();
1285 return areCompatibleVTYPEs(PriorVType, VType, Used);
1292 DemandedFields
Used;
1298 if (!isVectorConfigInstr(
MI)) {
1303 Register VRegDef =
MI.getOperand(0).getReg();
1304 if (VRegDef != RISCV::X0 &&
1305 !(VRegDef.
isVirtual() &&
MRI->use_nodbg_empty(VRegDef)))
1309 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1310 ToDelete.push_back(&
MI);
1314 if (!isVLPreservingConfig(*NextMI)) {
1322 ToDelete.push_back(NextMI);
1330 for (
auto *
MI : ToDelete)
1331 MI->eraseFromParent();
1338 Register VLOutput =
MI.getOperand(1).getReg();
1339 if (!
MRI->use_nodbg_empty(VLOutput))
1343 MI.getOperand(1).setReg(RISCV::X0);
1351 if (!
ST.hasVInstructions())
1356 TII =
ST.getInstrInfo();
1359 assert(BlockInfo.empty() &&
"Expect empty block infos");
1362 bool HaveVectorOp =
false;
1366 HaveVectorOp |= computeVLVTYPEChanges(
MBB);
1369 BBInfo.Exit = BBInfo.Change;
1371 <<
" is " << BBInfo.Exit <<
"\n");
1376 if (!HaveVectorOp) {
1385 WorkList.push(&
MBB);
1388 while (!WorkList.empty()) {
1391 computeIncomingVLVTYPE(
MBB);
1412 doLocalPostpass(
MBB);
1421 if (
MI.getOpcode() == RISCV::PseudoVSETVLI ||
1422 MI.getOpcode() == RISCV::PseudoVSETIVLI) {
1423 Register VRegDef =
MI.getOperand(0).getReg();
1424 if (VRegDef != RISCV::X0 &&
MRI->use_nodbg_empty(VRegDef))
1425 MI.getOperand(0).setReg(RISCV::X0);
1436 return HaveVectorOp;
1441 return new RISCVInsertVSETVLI();