37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
55 auto &LI =
LIS->getInterval(Reg);
57 VNInfo *VNI = LI.getVNInfoBefore(SI);
71 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
72 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
73 MI.getOpcode() == RISCV::PseudoVSETIVLI;
79 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
81 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
82 return RISCV::X0 ==
MI.getOperand(0).getReg();
85static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
100 case RISCV::VFMV_F_S:
110 case RISCV::VFMV_S_F:
121 case RISCV::VFMV_V_F:
130 case RISCV::VSLIDEDOWN_VX:
131 case RISCV::VSLIDEDOWN_VI:
132 case RISCV::VSLIDEUP_VX:
133 case RISCV::VSLIDEUP_VI:
140static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
150 case RISCV::VLSE16_V:
152 case RISCV::VSSE16_V:
155 case RISCV::VLSE32_V:
157 case RISCV::VSSE32_V:
160 case RISCV::VLSE64_V:
162 case RISCV::VSSE64_V:
168 return MI.getOpcode() == RISCV::ADDI &&
169 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
170 MI.getOperand(1).getReg() == RISCV::X0 &&
171 MI.getOperand(2).getImm() != 0;
179 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
191 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
199 return UseMO.
getReg() == RISCV::NoRegister || UseMO.
isUndef();
203struct DemandedFields {
208 bool VLZeroness =
false;
212 SEWGreaterThanOrEqual = 2,
214 SEWGreaterThanOrEqualAndLessThan64 =
221 bool SEWLMULRatio =
false;
222 bool TailPolicy =
false;
223 bool MaskPolicy =
false;
226 bool usedVTYPE()
const {
227 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
232 return VLAny || VLZeroness;
251 void doUnion(
const DemandedFields &
B) {
253 VLZeroness |=
B.VLZeroness;
254 SEW = std::max(SEW,
B.SEW);
256 SEWLMULRatio |=
B.SEWLMULRatio;
257 TailPolicy |=
B.TailPolicy;
258 MaskPolicy |=
B.MaskPolicy;
261#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
271 OS <<
"VLAny=" << VLAny <<
", ";
272 OS <<
"VLZeroness=" << VLZeroness <<
", ";
278 case SEWGreaterThanOrEqual:
279 OS <<
"SEWGreaterThanOrEqual";
281 case SEWGreaterThanOrEqualAndLessThan64:
282 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
289 OS <<
"LMUL=" <<
LMUL <<
", ";
290 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
291 OS <<
"TailPolicy=" << TailPolicy <<
", ";
292 OS <<
"MaskPolicy=" << MaskPolicy;
298#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
310 const DemandedFields &Used) {
312 case DemandedFields::SEWNone:
314 case DemandedFields::SEWEqual:
318 case DemandedFields::SEWGreaterThanOrEqual:
322 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
333 if (
Used.SEWLMULRatio) {
338 if (Ratio1 != Ratio2)
361 if (
MI.isCall() ||
MI.isInlineAsm() ||
362 MI.readsRegister(RISCV::VL,
nullptr))
364 if (
MI.isCall() ||
MI.isInlineAsm() ||
365 MI.readsRegister(RISCV::VTYPE,
nullptr))
376 Res.MaskPolicy =
false;
385 if (getEEWForLoadStore(
MI)) {
386 Res.SEW = DemandedFields::SEWNone;
392 Res.TailPolicy =
false;
393 Res.MaskPolicy =
false;
400 if (isMaskRegOp(
MI)) {
401 Res.SEW = DemandedFields::SEWNone;
406 if (isScalarInsertInstr(
MI)) {
408 Res.SEWLMULRatio =
false;
416 if (hasUndefinedMergeOp(
MI)) {
417 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
418 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
420 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
421 Res.TailPolicy =
false;
426 if (isScalarExtractInstr(
MI)) {
429 Res.SEWLMULRatio =
false;
430 Res.TailPolicy =
false;
431 Res.MaskPolicy =
false;
462 uint8_t TailAgnostic : 1;
463 uint8_t MaskAgnostic : 1;
464 uint8_t SEWLMULRatioOnly : 1;
468 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
469 SEWLMULRatioOnly(
false) {}
471 static VSETVLIInfo getUnknown() {
478 void setUnknown() { State =
Unknown; }
479 bool isUnknown()
const {
return State ==
Unknown; }
484 AVLRegDef.DefReg =
AVLReg;
488 void setAVLImm(
unsigned Imm) {
493 void setAVLVLMAX() { State = AVLIsVLMAX; }
495 void setAVLIgnored() { State = AVLIsIgnored; }
497 bool hasAVLImm()
const {
return State == AVLIsImm; }
498 bool hasAVLReg()
const {
return State == AVLIsReg; }
499 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
500 bool hasAVLIgnored()
const {
return State == AVLIsIgnored; }
502 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());
503 return AVLRegDef.DefReg;
505 unsigned getAVLImm()
const {
509 const VNInfo *getAVLVNInfo()
const {
511 return AVLRegDef.ValNo;
518 auto *
MI =
LIS->getInstructionFromIndex(getAVLVNInfo()->def);
519 assert(!(getAVLVNInfo()->isPHIDef() &&
MI));
523 void setAVL(VSETVLIInfo Info) {
525 if (
Info.isUnknown())
527 else if (
Info.hasAVLReg())
529 else if (
Info.hasAVLVLMAX())
531 else if (
Info.hasAVLIgnored())
535 setAVLImm(
Info.getAVLImm());
539 unsigned getSEW()
const {
return SEW; }
541 bool getTailAgnostic()
const {
return TailAgnostic; }
542 bool getMaskAgnostic()
const {
return MaskAgnostic; }
546 return getAVLImm() > 0;
549 return isNonZeroLoadImmediate(*
DefMI);
558 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other,
560 if (hasSameAVL(
Other))
562 return (hasNonZeroAVL(
LIS) &&
Other.hasNonZeroAVL(
LIS));
565 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
566 if (hasAVLReg() &&
Other.hasAVLReg())
567 return getAVLVNInfo()->id ==
Other.getAVLVNInfo()->id &&
568 getAVLReg() ==
Other.getAVLReg();
570 if (hasAVLImm() &&
Other.hasAVLImm())
571 return getAVLImm() ==
Other.getAVLImm();
574 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
577 return Other.hasAVLIgnored();
584 "Can't set VTYPE for uninitialized or unknown");
592 "Can't set VTYPE for uninitialized or unknown");
603 "Can't encode VTYPE for uninitialized or unknown");
607 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
609 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
611 "Can't compare invalid VSETVLIInfos");
613 "Can't compare VTYPE in unknown state");
614 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
615 "Can't compare when only LMUL/SEW ratio is valid.");
616 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
623 "Can't use VTYPE for uninitialized or unknown");
631 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
633 "Can't compare invalid VSETVLIInfos");
635 "Can't compare VTYPE in unknown state");
639 bool hasCompatibleVTYPE(
const DemandedFields &Used,
640 const VSETVLIInfo &Require)
const {
641 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
647 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
650 "Can't compare invalid VSETVLIInfos");
651 assert(!Require.SEWLMULRatioOnly &&
652 "Expected a valid VTYPE for instruction!");
654 if (isUnknown() || Require.isUnknown())
658 if (SEWLMULRatioOnly)
661 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
664 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require,
LIS))
667 return hasCompatibleVTYPE(Used, Require);
673 return !
Other.isValid();
674 if (!
Other.isValid())
679 return Other.isUnknown();
680 if (
Other.isUnknown())
683 if (!hasSameAVL(
Other))
687 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
691 if (SEWLMULRatioOnly)
692 return hasSameVLMAX(
Other);
695 return hasSameVTYPE(
Other);
699 return !(*
this ==
Other);
706 if (!
Other.isValid())
714 if (isUnknown() ||
Other.isUnknown())
715 return VSETVLIInfo::getUnknown();
723 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
724 VSETVLIInfo MergeInfo = *
this;
725 MergeInfo.SEWLMULRatioOnly =
true;
730 return VSETVLIInfo::getUnknown();
733#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
745 OS <<
"Uninitialized";
757 <<
"VLMul=" << (
unsigned)VLMul <<
", "
758 <<
"SEW=" << (
unsigned)SEW <<
", "
759 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
760 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
761 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
766#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
784 bool InQueue =
false;
795 std::vector<BlockData> BlockInfo;
796 std::queue<const MachineBasicBlock *> WorkList;
820 bool needVSETVLI(
const MachineInstr &
MI,
const VSETVLIInfo &Require,
821 const VSETVLIInfo &CurInfo)
const;
822 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
825 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
828 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
830 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
831 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
833 VSETVLIInfo &Info)
const;
872char RISCVInsertVSETVLI::ID = 0;
877char RISCVCoalesceVSETVLI::
ID = 0;
887 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
888 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
890 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
891 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
894 "Can't handle X0, X0 vsetvli yet");
918 VSETVLIInfo InstrInfo;
920 bool TailAgnostic =
true;
921 bool MaskAgnostic =
true;
922 if (!hasUndefinedMergeOp(
MI)) {
924 TailAgnostic =
false;
925 MaskAgnostic =
false;
932 "Invalid Policy Value");
948 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
950 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
956 int64_t Imm = VLOp.
getImm();
961 const unsigned VLMAX =
computeVLMAX(ST.getRealMaxVLen(), SEW, VLMul);
962 if (ST.getRealMinVLen() == ST.getRealMaxVLen() && VLMAX <= 31)
963 InstrInfo.setAVLImm(VLMAX);
965 InstrInfo.setAVLVLMAX();
968 InstrInfo.setAVLImm(Imm);
970 InstrInfo.setAVLRegDef(getVNInfoFromReg(VLOp.
getReg(),
MI,
LIS),
978 InstrInfo.setAVLIgnored();
981 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
982 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
985 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
991 if (InstrInfo.hasAVLReg()) {
994 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI,
LIS);
995 if (DefInstrInfo.hasSameVLMAX(InstrInfo) &&
996 (DefInstrInfo.hasAVLImm() || DefInstrInfo.hasAVLVLMAX()))
997 InstrInfo.setAVL(DefInstrInfo);
1005 const VSETVLIInfo &Info,
1006 const VSETVLIInfo &PrevInfo) {
1013 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1015 ++NumInsertedVSETVL;
1016 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1019 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1025 LIS->InsertMachineInstrInMaps(*
MI);
1032 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1035 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1036 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1042 LIS->InsertMachineInstrInMaps(*
MI);
1049 if (
Info.hasAVLImm()) {
1054 LIS->InsertMachineInstrInMaps(*
MI);
1058 if (
Info.hasAVLIgnored()) {
1061 if (PrevInfo.isValid() && !PrevInfo.isUnknown() &&
1062 Info.hasSameVLMAX(PrevInfo)) {
1068 LIS->InsertMachineInstrInMaps(*
MI);
1076 LIS->InsertMachineInstrInMaps(*
MI);
1080 if (
Info.hasAVLVLMAX()) {
1081 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1086 LIS->InsertMachineInstrInMaps(*
MI);
1087 LIS->createAndComputeVirtRegInterval(DestReg);
1092 MRI->constrainRegClass(
AVLReg, &RISCV::GPRNoX0RegClass);
1097 LIS->InsertMachineInstrInMaps(*
MI);
1102 LIS->getMBBStartIdx(&
MBB),
LIS->getInstructionIndex(*MI).getRegSlot());
1107 return Fractional || LMul == 1;
1113 const VSETVLIInfo &Require,
1114 const VSETVLIInfo &CurInfo)
const {
1117 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1120 DemandedFields
Used = getDemanded(
MI, ST);
1130 if (isVSlideInstr(
MI) && Require.hasAVLImm() && Require.getAVLImm() == 1 &&
1133 Used.VLZeroness =
true;
1135 Used.TailPolicy =
false;
1143 if (isScalarSplatInstr(
MI) && Require.hasAVLImm() &&
1145 hasUndefinedMergeOp(
MI)) {
1147 Used.SEWLMULRatio =
false;
1149 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
1150 Used.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
1152 Used.SEW = DemandedFields::SEWGreaterThanOrEqual;
1153 Used.TailPolicy =
false;
1156 if (CurInfo.isCompatible(Used, Require,
LIS))
1163 if (Require.hasAVLReg() && CurInfo.hasCompatibleVTYPE(Used, Require)) {
1166 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1167 if (DefInfo.hasSameAVL(CurInfo) && DefInfo.hasSameVLMAX(CurInfo))
1179 DemandedFields &Demanded) {
1182 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1183 !PrevInfo.isUnknown()) {
1185 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1186 Info.setVLMul(*NewVLMul);
1187 Demanded.LMUL =
true;
1196void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1207 const VSETVLIInfo PrevInfo =
Info;
1208 if (!
Info.isValid() ||
Info.isUnknown())
1211 DemandedFields Demanded = getDemanded(
MI, ST);
1221 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo,
LIS) &&
1222 IncomingInfo.hasSameVLMAX(PrevInfo);
1223 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1224 Info.setAVL(IncomingInfo);
1227 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1229 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1232 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1233 IncomingInfo.getTailAgnostic(),
1234 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1235 IncomingInfo.getMaskAgnostic());
1239 if (
Info.hasSEWLMULRatioOnly()) {
1240 VSETVLIInfo RatiolessInfo = IncomingInfo;
1241 RatiolessInfo.setAVL(Info);
1242 Info = RatiolessInfo;
1249void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1251 if (isVectorConfigInstr(
MI)) {
1258 assert(
MI.getOperand(1).getReg().isVirtual());
1259 auto &LI =
LIS->getInterval(
MI.getOperand(1).getReg());
1260 SlotIndex SI =
LIS->getSlotIndexes()->getInstructionIndex(
MI).getRegSlot();
1261 VNInfo *VNI = LI.getVNInfoAt(SI);
1262 Info.setAVLRegDef(VNI,
MI.getOperand(1).getReg());
1268 if (
MI.isCall() ||
MI.isInlineAsm() ||
1269 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1270 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1271 Info = VSETVLIInfo::getUnknown();
1275 VSETVLIInfo &Info)
const {
1276 bool HadVectorOp =
false;
1280 transferBefore(Info,
MI);
1285 transferAfter(Info,
MI);
1295 BBInfo.InQueue =
false;
1299 VSETVLIInfo InInfo = BBInfo.
Pred;
1302 InInfo.setUnknown();
1305 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1309 if (!InInfo.isValid())
1313 if (InInfo == BBInfo.
Pred)
1316 BBInfo.
Pred = InInfo;
1318 <<
" changed to " << BBInfo.
Pred <<
"\n");
1324 VSETVLIInfo TmpStatus;
1325 computeVLVTYPEChanges(
MBB, TmpStatus);
1329 if (BBInfo.
Exit == TmpStatus)
1332 BBInfo.
Exit = TmpStatus;
1334 <<
" changed to " << BBInfo.
Exit <<
"\n");
1339 if (!BlockInfo[S->getNumber()].InQueue) {
1340 BlockInfo[S->getNumber()].InQueue =
true;
1348bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1353 if (!Require.hasAVLReg())
1357 const VNInfo *Valno = Require.getAVLVNInfo();
1361 const LiveRange &LR =
LIS->getInterval(Require.getAVLReg());
1364 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
1376 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI,
LIS);
1377 if (DefInfo != PBBExit)
1383 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1396 bool PrefixTransparent =
true;
1398 const VSETVLIInfo PrevInfo = CurInfo;
1399 transferBefore(CurInfo,
MI);
1402 if (isVectorConfigInstr(
MI)) {
1404 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1405 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1406 "Unexpected operands where VL and VTYPE should be");
1407 MI.getOperand(3).setIsDead(
false);
1408 MI.getOperand(4).setIsDead(
false);
1409 PrefixTransparent =
false;
1414 if (PrevInfo != CurInfo) {
1422 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1423 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1424 PrefixTransparent =
false;
1434 VLOp.
setReg(RISCV::NoRegister);
1437 LIS->shrinkToUses(&LI, &DeadMIs);
1441 LIS->splitSeparateComponents(LI, SplitLIs);
1448 if (!
TII->isAddImmediate(*DeadMI, Reg))
1450 LIS->RemoveMachineInstrFromMaps(*DeadMI);
1451 DeadMI->eraseFromParent();
1461 if (
MI.isCall() ||
MI.isInlineAsm() ||
1462 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1463 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1464 PrefixTransparent =
false;
1466 transferAfter(CurInfo,
MI);
1470 if (CurInfo !=
Info.Exit) {
1476 assert(CurInfo ==
Info.Exit &&
"InsertVSETVLI dataflow invariant violated");
1489 VSETVLIInfo AvailableInfo;
1491 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1492 if (PredInfo.isUnknown()) {
1493 if (UnavailablePred)
1495 UnavailablePred =
P;
1496 }
else if (!AvailableInfo.isValid()) {
1497 AvailableInfo = PredInfo;
1498 }
else if (AvailableInfo != PredInfo) {
1505 if (!UnavailablePred || !AvailableInfo.isValid())
1510 if (AvailableInfo.hasSEWLMULRatioOnly())
1520 if (AvailableInfo.hasAVLReg()) {
1524 if (
LIS->getMBBFromIndex(SI) != UnavailablePred)
1533 if (AvailableInfo.hasAVLIgnored())
1545 VSETVLIInfo CurInfo = AvailableInfo;
1546 int TransitionsRemoved = 0;
1548 const VSETVLIInfo LastInfo = CurInfo;
1549 const VSETVLIInfo LastOldInfo = OldInfo;
1550 transferBefore(CurInfo,
MI);
1551 transferBefore(OldInfo,
MI);
1552 if (CurInfo == LastInfo)
1553 TransitionsRemoved++;
1554 if (LastOldInfo == OldInfo)
1555 TransitionsRemoved--;
1556 transferAfter(CurInfo,
MI);
1557 transferAfter(OldInfo,
MI);
1558 if (CurInfo == OldInfo)
1562 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1569 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1571 << UnavailablePred->
getName() <<
" with state "
1572 << AvailableInfo <<
"\n");
1573 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1579 insertVSETVLI(*UnavailablePred, InsertPt,
1581 AvailableInfo, OldExit);
1588 const DemandedFields &Used,
1594 if (!isVLPreservingConfig(
MI)) {
1598 if (Used.VLZeroness) {
1599 if (isVLPreservingConfig(PrevMI))
1601 if (!getInfoForVSETVLI(PrevMI,
LIS)
1602 .hasEquallyZeroAVL(getInfoForVSETVLI(
MI,
LIS),
LIS))
1606 auto &AVL =
MI.getOperand(1);
1611 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1612 (!
MRI.hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1613 PrevAVL.getReg() != AVL.getReg()))
1619 auto VType =
MI.getOperand(2).getImm();
1620 return areCompatibleVTYPEs(PriorVType, VType, Used);
1627 DemandedFields
Used;
1633 if (!isVectorConfigInstr(
MI)) {
1634 Used.doUnion(getDemanded(
MI, ST));
1635 if (
MI.isCall() ||
MI.isInlineAsm() ||
1636 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1637 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1642 if (!
MI.getOperand(0).isDead())
1646 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1653 if (!isVLPreservingConfig(*NextMI)) {
1656 MI.getOperand(0).setReg(DefReg);
1657 MI.getOperand(0).setIsDead(
false);
1667 DefVNI->
def = MISlot;
1673 LIS->shrinkToUses(&DefLI);
1677 if (
MI.getOperand(1).isReg())
1678 OldVLReg =
MI.getOperand(1).getReg();
1690 LIS->shrinkToUses(&
LIS->getInterval(OldVLReg));
1693 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1694 MRI->use_nodbg_empty(OldVLReg)) {
1696 LIS->removeInterval(OldVLReg);
1707 Used = getDemanded(
MI, ST);
1710 NumCoalescedVSETVL += ToDelete.
size();
1711 for (
auto *
MI : ToDelete) {
1712 LIS->RemoveMachineInstrFromMaps(*
MI);
1713 MI->eraseFromParent();
1716 return !ToDelete.empty();
1723 Register VLOutput =
MI.getOperand(1).getReg();
1725 if (!
MI.getOperand(1).isDead()) {
1727 TII->get(RISCV::PseudoReadVL), VLOutput);
1730 LIS->InsertMachineInstrInMaps(*ReadVLMI).getRegSlot();
1734 DefVNI->
def = NewDefSI;
1737 MI.getOperand(1).setReg(RISCV::X0);
1745 if (!
ST->hasVInstructions())
1750 TII =
ST->getInstrInfo();
1752 LIS = &getAnalysis<LiveIntervals>();
1754 assert(BlockInfo.empty() &&
"Expect empty block infos");
1757 bool HaveVectorOp =
false;
1761 VSETVLIInfo TmpStatus;
1762 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1765 BBInfo.
Exit = TmpStatus;
1767 <<
" is " << BBInfo.
Exit <<
"\n");
1772 if (!HaveVectorOp) {
1781 WorkList.push(&
MBB);
1784 while (!WorkList.empty()) {
1787 computeIncomingVLVTYPE(
MBB);
1807 return HaveVectorOp;
1812 return new RISCVInsertVSETVLI();
1824 if (!ST->hasVInstructions())
1826 TII = ST->getInstrInfo();
1828 LIS = &getAnalysis<LiveIntervals>();
1830 bool Changed =
false;
1832 Changed |= coalesceVSETVLIs(
MBB);
1838 return new RISCVCoalesceVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
NewInfo setVTYPE(MI.getOperand(2).getImm())
static VSETVLIInfo computeInfoForInstr(const MachineInstr &MI, uint64_t TSFlags, const RISCVSubtarget &ST, const LiveIntervals *LIS)
#define RISCV_COALESCE_VSETVLI_NAME
assert((AVLReg !=RISCV::X0||MI.getOperand(0).getReg() !=RISCV::X0) &&"Can't handle X0, X0 vsetvli yet")
static unsigned computeVLMAX(unsigned VLEN, unsigned SEW, RISCVII::VLMUL VLMul)
#define RISCV_INSERT_VSETVLI_NAME
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
else NewInfo setAVLRegDef(getVNInfoFromReg(AVLReg, MI, LIS), AVLReg)
static bool isLMUL1OrSmaller(RISCVII::VLMUL LMUL)
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
static bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI, const DemandedFields &Used, const MachineRegisterInfo &MRI, const LiveIntervals *LIS)
INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME, false, false) char RISCVCoalesceVSETVLI const LiveIntervals * LIS
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
This class represents the liveness of a register, stack slot, etc.
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx,...
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
void removeSegment(SlotIndex Start, SlotIndex End, bool RemoveDeadValNo=false)
Remove the specified interval from this live range.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
FunctionPass * createRISCVCoalesceVSETVLIPass()
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.