37#define DEBUG_TYPE "riscv-insert-vsetvli"
38#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
39#define RISCV_COALESCE_VSETVLI_NAME "RISC-V Coalesce VSETVLI pass"
41STATISTIC(NumInsertedVSETVL,
"Number of VSETVL inst inserted");
42STATISTIC(NumCoalescedVSETVL,
"Number of VSETVL inst coalesced");
46 cl::desc(
"Disable looking through phis when inserting vsetvlis."));
60 return LI.getVNInfoBefore(SI);
72 return MI.getOpcode() == RISCV::PseudoVSETVLI ||
73 MI.getOpcode() == RISCV::PseudoVSETVLIX0 ||
74 MI.getOpcode() == RISCV::PseudoVSETIVLI;
80 if (
MI.getOpcode() != RISCV::PseudoVSETVLIX0)
82 assert(RISCV::X0 ==
MI.getOperand(1).getReg());
83 return RISCV::X0 ==
MI.getOperand(0).getReg();
86static bool isFloatScalarMoveOrScalarSplatInstr(
const MachineInstr &
MI) {
101 case RISCV::VFMV_F_S:
111 case RISCV::VFMV_S_F:
122 case RISCV::VFMV_V_F:
131 case RISCV::VSLIDEDOWN_VX:
132 case RISCV::VSLIDEDOWN_VI:
133 case RISCV::VSLIDEUP_VX:
134 case RISCV::VSLIDEUP_VI:
141static std::optional<unsigned> getEEWForLoadStore(
const MachineInstr &
MI) {
151 case RISCV::VLSE16_V:
153 case RISCV::VSSE16_V:
156 case RISCV::VLSE32_V:
158 case RISCV::VSSE32_V:
161 case RISCV::VLSE64_V:
163 case RISCV::VSSE64_V:
169 return MI.getOpcode() == RISCV::ADDI &&
170 MI.getOperand(1).isReg() &&
MI.getOperand(2).isImm() &&
171 MI.getOperand(1).getReg() == RISCV::X0 &&
172 MI.getOperand(2).getImm() != 0;
180 const unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
192 if (!
MI.isRegTiedToUseOperand(0, &UseOpIdx))
200 return UseMO.
getReg() == RISCV::NoRegister || UseMO.
isUndef();
204struct DemandedFields {
209 bool VLZeroness =
false;
213 SEWGreaterThanOrEqual = 2,
215 SEWGreaterThanOrEqualAndLessThan64 =
223 LMULLessThanOrEqualToM1 = 1,
226 bool SEWLMULRatio =
false;
227 bool TailPolicy =
false;
228 bool MaskPolicy =
false;
231 bool usedVTYPE()
const {
232 return SEW ||
LMUL || SEWLMULRatio || TailPolicy || MaskPolicy;
237 return VLAny || VLZeroness;
255 static DemandedFields
all() {
263 void doUnion(
const DemandedFields &
B) {
265 VLZeroness |=
B.VLZeroness;
266 SEW = std::max(SEW,
B.SEW);
267 LMUL = std::max(LMUL,
B.LMUL);
268 SEWLMULRatio |=
B.SEWLMULRatio;
269 TailPolicy |=
B.TailPolicy;
270 MaskPolicy |=
B.MaskPolicy;
273#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
283 OS <<
"VLAny=" << VLAny <<
", ";
284 OS <<
"VLZeroness=" << VLZeroness <<
", ";
290 case SEWGreaterThanOrEqual:
291 OS <<
"SEWGreaterThanOrEqual";
293 case SEWGreaterThanOrEqualAndLessThan64:
294 OS <<
"SEWGreaterThanOrEqualAndLessThan64";
306 case LMULLessThanOrEqualToM1:
307 OS <<
"LMULLessThanOrEqualToM1";
314 OS <<
"SEWLMULRatio=" << SEWLMULRatio <<
", ";
315 OS <<
"TailPolicy=" << TailPolicy <<
", ";
316 OS <<
"MaskPolicy=" << MaskPolicy;
322#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
332 return Fractional || LMul == 1;
339 const DemandedFields &Used) {
341 case DemandedFields::SEWNone:
343 case DemandedFields::SEWEqual:
347 case DemandedFields::SEWGreaterThanOrEqual:
351 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
359 case DemandedFields::LMULNone:
361 case DemandedFields::LMULEqual:
365 case DemandedFields::LMULLessThanOrEqualToM1:
371 if (
Used.SEWLMULRatio) {
376 if (Ratio1 != Ratio2)
399 if (
MI.isCall() ||
MI.isInlineAsm() ||
400 MI.readsRegister(RISCV::VL,
nullptr))
402 if (
MI.isCall() ||
MI.isInlineAsm() ||
403 MI.readsRegister(RISCV::VTYPE,
nullptr))
411 !VLOp.isReg() || !VLOp.isUndef())
416 Res.MaskPolicy =
false;
425 if (getEEWForLoadStore(
MI)) {
426 Res.SEW = DemandedFields::SEWNone;
427 Res.LMUL = DemandedFields::LMULNone;
432 Res.TailPolicy =
false;
433 Res.MaskPolicy =
false;
440 if (isMaskRegOp(
MI)) {
441 Res.SEW = DemandedFields::SEWNone;
442 Res.LMUL = DemandedFields::LMULNone;
446 if (isScalarInsertInstr(
MI)) {
447 Res.LMUL = DemandedFields::LMULNone;
448 Res.SEWLMULRatio =
false;
456 if (hasUndefinedMergeOp(
MI)) {
457 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
458 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
460 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
461 Res.TailPolicy =
false;
466 if (isScalarExtractInstr(
MI)) {
468 Res.LMUL = DemandedFields::LMULNone;
469 Res.SEWLMULRatio =
false;
470 Res.TailPolicy =
false;
471 Res.MaskPolicy =
false;
484 if (isVSlideInstr(
MI) && VLOp.
isImm() && VLOp.
getImm() == 1 &&
485 hasUndefinedMergeOp(
MI)) {
487 Res.VLZeroness =
true;
488 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
489 Res.TailPolicy =
false;
498 if (isScalarSplatInstr(
MI) && VLOp.
isImm() && VLOp.
getImm() == 1 &&
499 hasUndefinedMergeOp(
MI)) {
500 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
501 Res.SEWLMULRatio =
false;
503 if (isFloatScalarMoveOrScalarSplatInstr(
MI) && !
ST->hasVInstructionsF64())
504 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
506 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
507 Res.TailPolicy =
false;
539 uint8_t TailAgnostic : 1;
540 uint8_t MaskAgnostic : 1;
541 uint8_t SEWLMULRatioOnly : 1;
545 : AVLImm(0), TailAgnostic(
false), MaskAgnostic(
false),
546 SEWLMULRatioOnly(
false) {}
548 static VSETVLIInfo getUnknown() {
555 void setUnknown() { State =
Unknown; }
556 bool isUnknown()
const {
return State ==
Unknown; }
561 AVLRegDef.DefReg = AVLReg;
565 void setAVLImm(
unsigned Imm) {
570 void setAVLVLMAX() { State = AVLIsVLMAX; }
572 bool hasAVLImm()
const {
return State == AVLIsImm; }
573 bool hasAVLReg()
const {
return State == AVLIsReg; }
574 bool hasAVLVLMAX()
const {
return State == AVLIsVLMAX; }
576 assert(hasAVLReg() && AVLRegDef.DefReg.isVirtual());
577 return AVLRegDef.DefReg;
579 unsigned getAVLImm()
const {
583 const VNInfo *getAVLVNInfo()
const {
585 return AVLRegDef.ValNo;
595 assert(!(getAVLVNInfo()->isPHIDef() &&
MI));
599 void setAVL(VSETVLIInfo Info) {
601 if (
Info.isUnknown())
603 else if (
Info.hasAVLReg())
604 setAVLRegDef(
Info.getAVLVNInfo(),
Info.getAVLReg());
605 else if (
Info.hasAVLVLMAX())
609 setAVLImm(
Info.getAVLImm());
613 unsigned getSEW()
const {
return SEW; }
615 bool getTailAgnostic()
const {
return TailAgnostic; }
616 bool getMaskAgnostic()
const {
return MaskAgnostic; }
620 return getAVLImm() > 0;
622 if (
auto *
DefMI = getAVLDefMI(LIS))
623 return isNonZeroLoadImmediate(*
DefMI);
630 bool hasEquallyZeroAVL(
const VSETVLIInfo &
Other,
632 if (hasSameAVL(
Other))
634 return (hasNonZeroAVL(LIS) &&
Other.hasNonZeroAVL(LIS));
637 bool hasSameAVLLatticeValue(
const VSETVLIInfo &
Other)
const {
638 if (hasAVLReg() &&
Other.hasAVLReg()) {
640 "we either have intervals or we don't");
642 return getAVLReg() ==
Other.getAVLReg();
643 return getAVLVNInfo()->id ==
Other.getAVLVNInfo()->id &&
644 getAVLReg() ==
Other.getAVLReg();
647 if (hasAVLImm() &&
Other.hasAVLImm())
648 return getAVLImm() ==
Other.getAVLImm();
651 return Other.hasAVLVLMAX() && hasSameVLMAX(
Other);
658 bool hasSameAVL(
const VSETVLIInfo &
Other)
const {
662 if (hasAVLReg() &&
Other.hasAVLReg()) {
664 "we either have intervals or we don't");
668 return hasSameAVLLatticeValue(
Other);
671 void setVTYPE(
unsigned VType) {
673 "Can't set VTYPE for uninitialized or unknown");
681 "Can't set VTYPE for uninitialized or unknown");
692 "Can't encode VTYPE for uninitialized or unknown");
696 bool hasSEWLMULRatioOnly()
const {
return SEWLMULRatioOnly; }
698 bool hasSameVTYPE(
const VSETVLIInfo &
Other)
const {
700 "Can't compare invalid VSETVLIInfos");
702 "Can't compare VTYPE in unknown state");
703 assert(!SEWLMULRatioOnly && !
Other.SEWLMULRatioOnly &&
704 "Can't compare when only LMUL/SEW ratio is valid.");
705 return std::tie(VLMul, SEW, TailAgnostic, MaskAgnostic) ==
712 "Can't use VTYPE for uninitialized or unknown");
720 bool hasSameVLMAX(
const VSETVLIInfo &
Other)
const {
722 "Can't compare invalid VSETVLIInfos");
724 "Can't compare VTYPE in unknown state");
728 bool hasCompatibleVTYPE(
const DemandedFields &Used,
729 const VSETVLIInfo &Require)
const {
730 return areCompatibleVTYPEs(Require.encodeVTYPE(),
encodeVTYPE(), Used);
736 bool isCompatible(
const DemandedFields &Used,
const VSETVLIInfo &Require,
739 "Can't compare invalid VSETVLIInfos");
741 if (isUnknown() || Require.isUnknown())
745 if (SEWLMULRatioOnly || Require.SEWLMULRatioOnly)
748 if (
Used.VLAny && !(hasSameAVL(Require) && hasSameVLMAX(Require)))
751 if (
Used.VLZeroness && !hasEquallyZeroAVL(Require, LIS))
754 return hasCompatibleVTYPE(Used, Require);
760 return !
Other.isValid();
761 if (!
Other.isValid())
766 return Other.isUnknown();
767 if (
Other.isUnknown())
770 if (!hasSameAVLLatticeValue(
Other))
774 if (SEWLMULRatioOnly !=
Other.SEWLMULRatioOnly)
778 if (SEWLMULRatioOnly)
779 return hasSameVLMAX(
Other);
782 return hasSameVTYPE(
Other);
786 return !(*
this ==
Other);
793 if (!
Other.isValid())
801 if (isUnknown() ||
Other.isUnknown())
802 return VSETVLIInfo::getUnknown();
810 if (hasSameAVL(
Other) && hasSameVLMAX(
Other)) {
811 VSETVLIInfo MergeInfo = *
this;
812 MergeInfo.SEWLMULRatioOnly =
true;
817 return VSETVLIInfo::getUnknown();
820#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
832 OS <<
"Uninitialized";
842 <<
"VLMul=" << (
unsigned)VLMul <<
", "
843 <<
"SEW=" << (
unsigned)SEW <<
", "
844 <<
"TailAgnostic=" << (
bool)TailAgnostic <<
", "
845 <<
"MaskAgnostic=" << (
bool)MaskAgnostic <<
", "
846 <<
"SEWLMULRatioOnly=" << (
bool)SEWLMULRatioOnly <<
"}";
851#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
869 bool InQueue =
false;
881 std::vector<BlockData> BlockInfo;
882 std::queue<const MachineBasicBlock *> WorkList;
905 bool needVSETVLI(
const DemandedFields &Used,
const VSETVLIInfo &Require,
906 const VSETVLIInfo &CurInfo)
const;
907 bool needVSETVLIPHI(
const VSETVLIInfo &Require,
910 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
913 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo);
915 void transferBefore(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
916 void transferAfter(VSETVLIInfo &Info,
const MachineInstr &
MI)
const;
918 VSETVLIInfo &Info)
const;
925 const DemandedFields &Used)
const;
934char RISCVInsertVSETVLI::ID = 0;
945 if (
MI.getOpcode() == RISCV::PseudoVSETIVLI) {
946 NewInfo.setAVLImm(
MI.getOperand(1).getImm());
948 assert(
MI.getOpcode() == RISCV::PseudoVSETVLI ||
949 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
951 assert((AVLReg != RISCV::X0 ||
MI.getOperand(0).getReg() != RISCV::X0) &&
952 "Can't handle X0, X0 vsetvli yet");
953 if (AVLReg == RISCV::X0)
954 NewInfo.setAVLVLMAX();
955 else if (
MI.getOperand(1).isUndef())
957 NewInfo.setAVLImm(1);
959 VNInfo *VNI = getVNInfoFromReg(AVLReg,
MI, LIS);
960 NewInfo.setAVLRegDef(VNI, AVLReg);
963 NewInfo.setVTYPE(
MI.getOperand(2).getImm());
967 if (NewInfo.hasAVLReg()) {
970 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI);
971 if (DefInstrInfo.hasSameVLMAX(NewInfo))
972 NewInfo.setAVL(DefInstrInfo);
979static unsigned computeVLMAX(
unsigned VLEN,
unsigned SEW,
990RISCVInsertVSETVLI::computeInfoForInstr(
const MachineInstr &
MI)
const {
991 VSETVLIInfo InstrInfo;
992 const uint64_t TSFlags =
MI.getDesc().TSFlags;
994 bool TailAgnostic =
true;
995 bool MaskAgnostic =
true;
996 if (!hasUndefinedMergeOp(
MI)) {
998 TailAgnostic =
false;
999 MaskAgnostic =
false;
1006 "Invalid Policy Value");
1014 TailAgnostic =
true;
1017 MaskAgnostic =
true;
1022 unsigned Log2SEW =
MI.getOperand(getSEWOpNum(
MI)).getImm();
1024 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
1035 const unsigned VLMAX = computeVLMAX(
ST->getRealMaxVLen(), SEW, VLMul);
1036 if (
ST->getRealMinVLen() ==
ST->getRealMaxVLen() && VLMAX <= 31)
1037 InstrInfo.setAVLImm(VLMAX);
1039 InstrInfo.setAVLVLMAX();
1042 InstrInfo.setAVLImm(Imm);
1045 InstrInfo.setAVLImm(1);
1048 InstrInfo.setAVLRegDef(VNI, VLOp.
getReg());
1054 InstrInfo.setAVLImm(1);
1057 if (std::optional<unsigned> EEW = getEEWForLoadStore(
MI)) {
1058 assert(SEW == EEW &&
"Initial SEW doesn't match expected EEW");
1061 InstrInfo.setVTYPE(VLMul, SEW, TailAgnostic, MaskAgnostic);
1065 if (InstrInfo.hasAVLReg()) {
1068 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(*
DefMI);
1069 if (DefInstrInfo.hasSameVLMAX(InstrInfo))
1070 InstrInfo.setAVL(DefInstrInfo);
1078 const VSETVLIInfo &Info,
1079 const VSETVLIInfo &PrevInfo) {
1086 const VSETVLIInfo &Info,
const VSETVLIInfo &PrevInfo) {
1088 ++NumInsertedVSETVL;
1089 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
1092 if (
Info.hasSameAVL(PrevInfo) &&
Info.hasSameVLMAX(PrevInfo)) {
1106 if (
Info.hasSameVLMAX(PrevInfo) &&
Info.hasAVLReg()) {
1109 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1110 if (DefInfo.hasSameAVL(PrevInfo) && DefInfo.hasSameVLMAX(PrevInfo)) {
1124 if (
Info.hasAVLImm()) {
1134 if (
Info.hasAVLVLMAX()) {
1135 Register DestReg =
MRI->createVirtualRegister(&RISCV::GPRRegClass);
1148 MRI->constrainRegClass(AVLReg, &RISCV::GPRNoX0RegClass);
1169bool RISCVInsertVSETVLI::needVSETVLI(
const DemandedFields &Used,
1170 const VSETVLIInfo &Require,
1171 const VSETVLIInfo &CurInfo)
const {
1172 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
1175 if (CurInfo.isCompatible(Used, Require, LIS))
1185 DemandedFields &Demanded) {
1186 VSETVLIInfo
Info = NewInfo;
1188 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
1189 !PrevInfo.isUnknown()) {
1191 PrevInfo.getSEW(), PrevInfo.getVLMUL(),
Info.getSEW()))
1192 Info.setVLMul(*NewVLMul);
1193 Demanded.LMUL = DemandedFields::LMULEqual;
1202void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
1207 DemandedFields Demanded = getDemanded(
MI, ST);
1209 const VSETVLIInfo NewInfo = computeInfoForInstr(
MI);
1210 assert(NewInfo.isValid() && !NewInfo.isUnknown());
1211 if (
Info.isValid() && !needVSETVLI(Demanded, NewInfo, Info))
1214 const VSETVLIInfo PrevInfo =
Info;
1215 if (!
Info.isValid() ||
Info.isUnknown())
1218 const VSETVLIInfo IncomingInfo =
adjustIncoming(PrevInfo, NewInfo, Demanded);
1227 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(PrevInfo, LIS) &&
1228 IncomingInfo.hasSameVLMAX(PrevInfo);
1229 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
1230 Info.setAVL(IncomingInfo);
1233 ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
1235 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info).getSEW(),
1238 (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
1239 IncomingInfo.getTailAgnostic(),
1240 (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
1241 IncomingInfo.getMaskAgnostic());
1245 if (
Info.hasSEWLMULRatioOnly()) {
1246 VSETVLIInfo RatiolessInfo = IncomingInfo;
1247 RatiolessInfo.setAVL(Info);
1248 Info = RatiolessInfo;
1255void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
1257 if (isVectorConfigInstr(
MI)) {
1258 Info = getInfoForVSETVLI(
MI);
1264 assert(
MI.getOperand(1).getReg().isVirtual());
1270 Info.setAVLRegDef(VNI,
MI.getOperand(1).getReg());
1272 Info.setAVLRegDef(
nullptr,
MI.getOperand(1).getReg());
1278 if (
MI.isCall() ||
MI.isInlineAsm() ||
1279 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1280 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1281 Info = VSETVLIInfo::getUnknown();
1285 VSETVLIInfo &Info)
const {
1286 bool HadVectorOp =
false;
1290 transferBefore(Info,
MI);
1295 transferAfter(Info,
MI);
1305 BBInfo.InQueue =
false;
1309 VSETVLIInfo InInfo = BBInfo.
Pred;
1312 InInfo.setUnknown();
1315 InInfo = InInfo.
intersect(BlockInfo[
P->getNumber()].Exit);
1319 if (!InInfo.isValid())
1323 if (InInfo == BBInfo.
Pred)
1326 BBInfo.
Pred = InInfo;
1328 <<
" changed to " << BBInfo.
Pred <<
"\n");
1334 VSETVLIInfo TmpStatus;
1335 computeVLVTYPEChanges(
MBB, TmpStatus);
1339 if (BBInfo.
Exit == TmpStatus)
1342 BBInfo.
Exit = TmpStatus;
1344 <<
" changed to " << BBInfo.
Exit <<
"\n");
1349 if (!BlockInfo[S->getNumber()].InQueue) {
1350 BlockInfo[S->getNumber()].InQueue =
true;
1358bool RISCVInsertVSETVLI::needVSETVLIPHI(
const VSETVLIInfo &Require,
1363 if (!Require.hasAVLReg())
1370 const VNInfo *Valno = Require.getAVLVNInfo();
1377 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
1389 VSETVLIInfo DefInfo = getInfoForVSETVLI(*
DefMI);
1390 if (DefInfo != PBBExit)
1396 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Require))
1409 bool PrefixTransparent =
true;
1411 const VSETVLIInfo PrevInfo = CurInfo;
1412 transferBefore(CurInfo,
MI);
1415 if (isVectorConfigInstr(
MI)) {
1417 assert(
MI.getOperand(3).getReg() == RISCV::VL &&
1418 MI.getOperand(4).getReg() == RISCV::VTYPE &&
1419 "Unexpected operands where VL and VTYPE should be");
1420 MI.getOperand(3).setIsDead(
false);
1421 MI.getOperand(4).setIsDead(
false);
1422 PrefixTransparent =
false;
1427 if (!PrevInfo.isCompatible(DemandedFields::all(), CurInfo, LIS)) {
1435 if (!PrefixTransparent || needVSETVLIPHI(CurInfo,
MBB))
1436 insertVSETVLI(
MBB,
MI, CurInfo, PrevInfo);
1437 PrefixTransparent =
false;
1446 VLOp.
setReg(RISCV::NoRegister);
1462 if (!
TII->isAddImmediate(*DeadMI, Reg))
1465 DeadMI->eraseFromParent();
1476 if (
MI.isCall() ||
MI.isInlineAsm() ||
1477 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1478 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1479 PrefixTransparent =
false;
1481 transferAfter(CurInfo,
MI);
1485 if (CurInfo !=
Info.Exit) {
1491 assert(CurInfo ==
Info.Exit &&
"InsertVSETVLI dataflow invariant violated");
1504 VSETVLIInfo AvailableInfo;
1506 const VSETVLIInfo &PredInfo = BlockInfo[
P->getNumber()].Exit;
1507 if (PredInfo.isUnknown()) {
1508 if (UnavailablePred)
1510 UnavailablePred =
P;
1511 }
else if (!AvailableInfo.isValid()) {
1512 AvailableInfo = PredInfo;
1513 }
else if (AvailableInfo != PredInfo) {
1520 if (!UnavailablePred || !AvailableInfo.isValid())
1528 if (AvailableInfo.hasSEWLMULRatioOnly())
1538 if (AvailableInfo.hasAVLReg()) {
1558 VSETVLIInfo CurInfo = AvailableInfo;
1559 int TransitionsRemoved = 0;
1561 const VSETVLIInfo LastInfo = CurInfo;
1562 const VSETVLIInfo LastOldInfo = OldInfo;
1563 transferBefore(CurInfo,
MI);
1564 transferBefore(OldInfo,
MI);
1565 if (CurInfo == LastInfo)
1566 TransitionsRemoved++;
1567 if (LastOldInfo == OldInfo)
1568 TransitionsRemoved--;
1569 transferAfter(CurInfo,
MI);
1570 transferAfter(OldInfo,
MI);
1571 if (CurInfo == OldInfo)
1575 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
1582 auto OldExit = BlockInfo[UnavailablePred->
getNumber()].Exit;
1584 << UnavailablePred->
getName() <<
" with state "
1585 << AvailableInfo <<
"\n");
1586 BlockInfo[UnavailablePred->
getNumber()].Exit = AvailableInfo;
1592 insertVSETVLI(*UnavailablePred, InsertPt,
1594 AvailableInfo, OldExit);
1599bool RISCVInsertVSETVLI::canMutatePriorConfig(
1601 const DemandedFields &Used)
const {
1605 if (!isVLPreservingConfig(
MI)) {
1609 if (
Used.VLZeroness) {
1610 if (isVLPreservingConfig(PrevMI))
1612 if (!getInfoForVSETVLI(PrevMI).hasEquallyZeroAVL(getInfoForVSETVLI(
MI),
1617 auto &AVL =
MI.getOperand(1);
1622 if (AVL.isReg() && AVL.getReg() != RISCV::X0 &&
1623 (!
MRI->hasOneDef(AVL.getReg()) || !PrevAVL.isReg() ||
1624 PrevAVL.getReg() != AVL.getReg()))
1630 auto VType =
MI.getOperand(2).getImm();
1631 return areCompatibleVTYPEs(PriorVType, VType, Used);
1638 DemandedFields
Used;
1644 if (!isVectorConfigInstr(
MI)) {
1645 Used.doUnion(getDemanded(
MI, ST));
1646 if (
MI.isCall() ||
MI.isInlineAsm() ||
1647 MI.modifiesRegister(RISCV::VL,
nullptr) ||
1648 MI.modifiesRegister(RISCV::VTYPE,
nullptr))
1653 if (!
MI.getOperand(0).isDead())
1657 if (!
Used.usedVL() && !
Used.usedVTYPE()) {
1663 if (canMutatePriorConfig(
MI, *NextMI, Used)) {
1664 if (!isVLPreservingConfig(*NextMI)) {
1667 MI.getOperand(0).setReg(DefReg);
1668 MI.getOperand(0).setIsDead(
false);
1678 DefVNI->
def = MISlot;
1688 if (
MI.getOperand(1).isReg())
1689 OldVLReg =
MI.getOperand(1).getReg();
1705 if (VLOpDef &&
TII->isAddImmediate(*VLOpDef, OldVLReg) &&
1706 MRI->use_nodbg_empty(OldVLReg)) {
1720 Used = getDemanded(
MI, ST);
1723 NumCoalescedVSETVL += ToDelete.
size();
1724 for (
auto *
MI : ToDelete) {
1727 MI->eraseFromParent();
1735 Register VLOutput =
MI.getOperand(1).getReg();
1737 if (!
MI.getOperand(1).isDead()) {
1739 TII->get(RISCV::PseudoReadVL), VLOutput);
1747 DefVNI->
def = NewDefSI;
1751 MI.getOperand(1).setReg(RISCV::X0);
1759 if (!
ST->hasVInstructions())
1764 TII =
ST->getInstrInfo();
1766 LIS = getAnalysisIfAvailable<LiveIntervals>();
1768 assert(BlockInfo.empty() &&
"Expect empty block infos");
1771 bool HaveVectorOp =
false;
1775 VSETVLIInfo TmpStatus;
1776 HaveVectorOp |= computeVLVTYPEChanges(
MBB, TmpStatus);
1779 BBInfo.
Exit = TmpStatus;
1781 <<
" is " << BBInfo.
Exit <<
"\n");
1786 if (!HaveVectorOp) {
1795 WorkList.push(&
MBB);
1798 while (!WorkList.empty()) {
1801 computeIncomingVLVTYPE(
MBB);
1822 coalesceVSETVLIs(
MBB);
1830 return HaveVectorOp;
1835 return new RISCVInsertVSETVLI();
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
#define LLVM_ATTRIBUTE_USED
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
static RegisterPass< DebugifyFunctionPass > DF("debugify-function", "Attach debug info to a function")
std::optional< std::vector< StOtherPiece > > Other
const HexagonInstrInfo * TII
static ValueLatticeElement intersect(const ValueLatticeElement &A, const ValueLatticeElement &B)
Combine two sets of facts about the same value into a single set of facts.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
#define RISCV_INSERT_VSETVLI_NAME
static VSETVLIInfo adjustIncoming(VSETVLIInfo PrevInfo, VSETVLIInfo NewInfo, DemandedFields &Demanded)
static cl::opt< bool > DisableInsertVSETVLPHIOpt("riscv-disable-insert-vsetvl-phi-opt", cl::init(false), cl::Hidden, cl::desc("Disable looking through phis when inserting vsetvlis."))
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Represent the analysis usage information of a pass.
AnalysisUsage & addUsedIfAvailable()
Add the specified Pass class to the set of analyses used by this pass.
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
This class represents an Operation in the Expression.
FunctionPass class - This class is used to implement most global optimizations.
LiveInterval - This class represents the liveness of a register, or stack slot.
void setWeight(float Value)
MachineInstr * getInstructionFromIndex(SlotIndex index) const
Returns the instruction associated with the given index.
SlotIndex InsertMachineInstrInMaps(MachineInstr &MI)
SlotIndexes * getSlotIndexes() const
SlotIndex getInstructionIndex(const MachineInstr &Instr) const
Returns the base index of the given instruction.
void RemoveMachineInstrFromMaps(MachineInstr &MI)
SlotIndex getMBBEndIdx(const MachineBasicBlock *mbb) const
Return the last index in the given basic block.
LiveInterval & getInterval(Register Reg)
void removeInterval(Register Reg)
Interval removal.
bool shrinkToUses(LiveInterval *li, SmallVectorImpl< MachineInstr * > *dead=nullptr)
After removing some uses of a register, shrink its live range to just the remaining uses.
void splitSeparateComponents(LiveInterval &LI, SmallVectorImpl< LiveInterval * > &SplitLIs)
Split separate components in LiveInterval LI into separate intervals.
MachineBasicBlock * getMBBFromIndex(SlotIndex index) const
LiveInterval & createAndComputeVirtRegInterval(Register Reg)
This class represents the liveness of a register, stack slot, etc.
VNInfo * getValNumInfo(unsigned ValNo)
getValNumInfo - Returns pointer to the specified val#.
iterator addSegment(Segment S)
Add the specified Segment to this range, merging segments as appropriate.
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx,...
SlotIndex beginIndex() const
beginIndex - Return the lowest numbered slot covered.
void removeSegment(SlotIndex Start, SlotIndex End, bool RemoveDeadValNo=false)
Remove the specified interval from this live range.
VNInfo * getVNInfoAt(SlotIndex Idx) const
getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
int getNumber() const
MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
DebugLoc findDebugLoc(instr_iterator MBBI)
Find the next valid DebugLoc starting at MBBI, skipping any debug instructions.
iterator_range< iterator > terminators()
iterator_range< succ_iterator > successors()
instr_iterator getFirstInstrTerminator()
Same getFirstTerminator but it ignores bundles and return an instr_iterator instead.
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
StringRef getName() const
Return the name of the corresponding LLVM basic block, or an empty string.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getNumBlockIDs() const
getNumBlockIDs - Return the number of MBB ID's allocated.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Representation of each machine instruction.
const MCInstrDesc & getDesc() const
Returns the target instruction descriptor of this MachineInstr.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setReg(Register Reg)
Change the register this operand corresponds to.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getRegSlot(bool EC=false) const
Returns the register use/def slot in the current instruction for a normal or early-clobber def.
SlotIndex getInstructionIndex(const MachineInstr &MI, bool IgnoreBundle=false) const
Returns the base index for the given instruction.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
TargetInstrInfo - Interface to description of machine instruction set.
VNInfo - Value Number Information.
SlotIndex def
The index of the defining instruction.
bool isPHIDef() const
Returns true if this value is defined by a PHI instruction (or was, PHI instructions may have been el...
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
static bool usesMaskPolicy(uint64_t TSFlags)
static unsigned getVLOpNum(const MCInstrDesc &Desc)
static bool doesForceTailAgnostic(uint64_t TSFlags)
static VLMUL getLMul(uint64_t TSFlags)
static bool hasVLOp(uint64_t TSFlags)
static bool hasVecPolicyOp(uint64_t TSFlags)
static unsigned getSEWOpNum(const MCInstrDesc &Desc)
static bool hasSEWOp(uint64_t TSFlags)
static bool isTailAgnostic(unsigned VType)
static RISCVII::VLMUL getVLMUL(unsigned VType)
std::pair< unsigned, bool > decodeVLMUL(RISCVII::VLMUL VLMUL)
unsigned getSEWLMULRatio(unsigned SEW, RISCVII::VLMUL VLMul)
static bool isMaskAgnostic(unsigned VType)
static bool isValidSEW(unsigned SEW)
unsigned encodeVTYPE(RISCVII::VLMUL VLMUL, unsigned SEW, bool TailAgnostic, bool MaskAgnostic)
static unsigned getSEW(unsigned VType)
std::optional< RISCVII::VLMUL > getSameRatioLMUL(unsigned SEW, RISCVII::VLMUL VLMUL, unsigned EEW)
unsigned getRVVMCOpcode(unsigned RVVPseudoOpcode)
bool isFaultFirstLoad(const MachineInstr &MI)
static constexpr int64_t VLMaxSentinel
@ Implicit
Not emitted register (e.g. carry, or temporary result).
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
bool operator!=(uint64_t V1, const APInt &V2)
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
FunctionPass * createRISCVInsertVSETVLIPass()
Returns an instance of the Insert VSETVLI pass.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
char & RISCVInsertVSETVLIID
Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)
Prints virtual and physical registers with or without a TRI instance.
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
Status intersect(const Status &S) const
This represents a simple continuous liveness interval for a value.