87#define DEBUG_TYPE "si-wqm"
96 StateStrict = StateStrictWWM | StateStrictWQM,
103 explicit PrintState(
int State) : State(State) {}
109 static const std::pair<char, const char *> Mapping[] = {
110 std::pair(StateWQM,
"WQM"), std::pair(StateStrictWWM,
"StrictWWM"),
111 std::pair(StateStrictWQM,
"StrictWQM"), std::pair(StateExact,
"Exact")};
112 char State = PS.State;
113 for (
auto M : Mapping) {
114 if (State & M.first) {
137 char InitialState = 0;
138 bool NeedsLowering =
false;
164 unsigned AndSaveExecOpc;
165 unsigned AndSaveExecTermOpc;
185 std::vector<WorkItem> &Worklist);
187 unsigned SubReg,
char Flag, std::vector<WorkItem> &Worklist);
189 std::vector<WorkItem> &Worklist);
191 std::vector<WorkItem> &Worklist);
192 char scanInstructions(
MachineFunction &MF, std::vector<WorkItem> &Worklist);
193 void propagateInstruction(
MachineInstr &
MI, std::vector<WorkItem> &Worklist);
208 Register SaveOrig,
char StrictStateNeeded);
211 char NonStrictState,
char CurrentStrictState);
222 void lowerLiveMaskQueries();
223 void lowerCopyInstrs();
224 void lowerKillInstrs(
bool IsWQM);
249 MachineFunctionProperties::Property::IsSSA);
255char SIWholeQuadMode::ID = 0;
268 return new SIWholeQuadMode;
273 for (
const auto &BII :
Blocks) {
276 <<
" InNeeds = " << PrintState(BII.second.InNeeds)
277 <<
", Needs = " << PrintState(BII.second.Needs)
278 <<
", OutNeeds = " << PrintState(BII.second.OutNeeds) <<
"\n\n";
281 auto III = Instructions.find(&
MI);
282 if (III != Instructions.end()) {
283 dbgs() <<
" " <<
MI <<
" Needs = " << PrintState(III->second.Needs)
284 <<
", OutNeeds = " << PrintState(III->second.OutNeeds) <<
'\n';
292 std::vector<WorkItem> &Worklist) {
295 assert(!(Flag & StateExact) && Flag != 0);
301 Flag &= ~II.Disabled;
305 if ((
II.Needs & Flag) == Flag)
310 Worklist.push_back(&
MI);
316 std::vector<WorkItem> &Worklist) {
328 : (
Reg.isVirtual() ?
MRI->getMaxLaneMaskForVReg(Reg)
340 :
Phi(
Phi), PredIdx(PredIdx), DefinedLanes(DefinedLanes) {}
342 using VisitKey = std::pair<const VNInfo *, LaneBitmask>;
346 unsigned NextPredIdx = 0;
348 const VNInfo *NextValue =
nullptr;
349 const VisitKey
Key(
Value, DefinedLanes);
351 if (Visited.
insert(Key).second) {
356 if (
Value->isPHIDef()) {
359 assert(
MBB &&
"Phi-def has no defining MBB");
362 unsigned Idx = NextPredIdx;
365 for (; PI != PE && !NextValue; ++PI, ++
Idx) {
367 if (!Visited.
count(VisitKey(VN, DefinedLanes)))
377 assert(
MI &&
"Def has no defining instruction");
379 if (
Reg.isVirtual()) {
383 if (
Op.getReg() != Reg)
389 :
TRI->getSubRegIndexLaneMask(
Op.getSubReg());
393 HasDef |= Overlap.
any();
396 DefinedLanes |= OpLanes;
400 if ((DefinedLanes & UseLanes) != UseLanes) {
404 if (!Visited.
count(VisitKey(VN, DefinedLanes)))
411 markInstruction(*
MI, Flag, Worklist);
414 markInstruction(*
MI, Flag, Worklist);
418 if (!NextValue && !PhiStack.
empty()) {
421 NextValue =
Entry.Phi;
422 NextPredIdx =
Entry.PredIdx;
423 DefinedLanes =
Entry.DefinedLanes;
433 std::vector<WorkItem> &Worklist) {
440 case AMDGPU::EXEC_LO:
448 if (
Reg.isVirtual()) {
450 markDefs(
MI, LR, Reg,
Op.getSubReg(), Flag, Worklist);
459 markDefs(
MI, LR, Unit, AMDGPU::NoSubRegister, Flag, Worklist);
465void SIWholeQuadMode::markInstructionUses(
const MachineInstr &
MI,
char Flag,
466 std::vector<WorkItem> &Worklist) {
467 LLVM_DEBUG(
dbgs() <<
"markInstructionUses " << PrintState(Flag) <<
": "
471 markOperand(
MI,
Use, Flag, Worklist);
477 std::vector<WorkItem> &Worklist) {
478 char GlobalFlags = 0;
482 bool HasImplicitDerivatives =
495 unsigned Opcode =
MI.getOpcode();
498 if (
TII->isWQM(Opcode)) {
503 if (
ST->hasExtendedImageInsts() && HasImplicitDerivatives) {
507 markInstructionUses(
MI, StateWQM, Worklist);
508 GlobalFlags |= StateWQM;
510 }
else if (Opcode == AMDGPU::WQM) {
514 LowerToCopyInstrs.push_back(&
MI);
515 }
else if (Opcode == AMDGPU::SOFT_WQM) {
516 LowerToCopyInstrs.push_back(&
MI);
518 }
else if (Opcode == AMDGPU::STRICT_WWM) {
522 markInstructionUses(
MI, StateStrictWWM, Worklist);
523 GlobalFlags |= StateStrictWWM;
524 LowerToMovInstrs.push_back(&
MI);
525 }
else if (Opcode == AMDGPU::STRICT_WQM ||
526 TII->isDualSourceBlendEXP(
MI)) {
530 markInstructionUses(
MI, StateStrictWQM, Worklist);
531 GlobalFlags |= StateStrictWQM;
533 if (Opcode == AMDGPU::STRICT_WQM) {
534 LowerToMovInstrs.push_back(&
MI);
539 BBI.Needs |= StateExact;
540 if (!(BBI.InNeeds & StateExact)) {
541 BBI.InNeeds |= StateExact;
542 Worklist.push_back(
MBB);
544 GlobalFlags |= StateExact;
545 III.Disabled = StateWQM | StateStrict;
547 }
else if (Opcode == AMDGPU::LDS_PARAM_LOAD ||
548 Opcode == AMDGPU::DS_PARAM_LOAD ||
549 Opcode == AMDGPU::LDS_DIRECT_LOAD ||
550 Opcode == AMDGPU::DS_DIRECT_LOAD) {
554 II.Needs |= StateStrictWQM;
555 GlobalFlags |= StateStrictWQM;
556 }
else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 ||
557 Opcode == AMDGPU::V_SET_INACTIVE_B64) {
558 III.Disabled = StateStrict;
560 if (Inactive.
isReg()) {
562 LowerToCopyInstrs.push_back(&
MI);
564 markOperand(
MI, Inactive, StateStrictWWM, Worklist);
568 }
else if (
TII->isDisableWQM(
MI)) {
569 BBI.Needs |= StateExact;
570 if (!(BBI.InNeeds & StateExact)) {
571 BBI.InNeeds |= StateExact;
572 Worklist.push_back(
MBB);
574 GlobalFlags |= StateExact;
575 III.Disabled = StateWQM | StateStrict;
576 }
else if (Opcode == AMDGPU::SI_PS_LIVE ||
577 Opcode == AMDGPU::SI_LIVE_MASK) {
578 LiveMaskQueries.push_back(&
MI);
579 }
else if (Opcode == AMDGPU::SI_KILL_I1_TERMINATOR ||
580 Opcode == AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR ||
581 Opcode == AMDGPU::SI_DEMOTE_I1) {
582 KillInstrs.push_back(&
MI);
583 BBI.NeedsLowering =
true;
584 }
else if (Opcode == AMDGPU::SI_INIT_EXEC ||
585 Opcode == AMDGPU::SI_INIT_EXEC_FROM_INPUT) {
586 InitExecInstrs.push_back(&
MI);
587 }
else if (WQMOutputs) {
594 if (
Reg.isPhysical() &&
595 TRI->hasVectorRegisters(
TRI->getPhysRegBaseClass(Reg))) {
603 markInstruction(
MI, Flags, Worklist);
604 GlobalFlags |=
Flags;
613 if (GlobalFlags & StateWQM) {
615 markInstruction(*
MI, StateWQM, Worklist);
617 markInstruction(*
MI, StateWQM, Worklist);
624 std::vector<WorkItem>& Worklist) {
631 if ((
II.OutNeeds & StateWQM) && !(
II.Disabled & StateWQM) &&
632 (
MI.isTerminator() || (
TII->usesVM_CNT(
MI) &&
MI.mayStore()))) {
638 if (
II.Needs & StateWQM) {
639 BI.Needs |= StateWQM;
640 if (!(BI.InNeeds & StateWQM)) {
641 BI.InNeeds |= StateWQM;
642 Worklist.push_back(
MBB);
648 char InNeeds = (
II.Needs & ~StateStrict) |
II.OutNeeds;
649 if (!PrevMI->isPHI()) {
651 if ((PrevII.OutNeeds | InNeeds) != PrevII.OutNeeds) {
652 PrevII.OutNeeds |= InNeeds;
653 Worklist.push_back(PrevMI);
662 markInstructionUses(
MI,
II.Needs, Worklist);
666 if (
II.Needs & StateStrictWWM)
667 BI.Needs |= StateStrictWWM;
668 if (
II.Needs & StateStrictWQM)
669 BI.Needs |= StateStrictWQM;
673 std::vector<WorkItem>& Worklist) {
680 if ((LastII.OutNeeds | BI.OutNeeds) != LastII.OutNeeds) {
681 LastII.OutNeeds |= BI.OutNeeds;
682 Worklist.push_back(LastMI);
688 BlockInfo &PredBI =
Blocks[Pred];
689 if ((PredBI.OutNeeds | BI.InNeeds) == PredBI.OutNeeds)
692 PredBI.OutNeeds |= BI.InNeeds;
693 PredBI.InNeeds |= BI.InNeeds;
694 Worklist.push_back(Pred);
699 BlockInfo &SuccBI =
Blocks[Succ];
700 if ((SuccBI.InNeeds | BI.OutNeeds) == SuccBI.InNeeds)
703 SuccBI.InNeeds |= BI.OutNeeds;
704 Worklist.push_back(Succ);
709 std::vector<WorkItem> Worklist;
710 char GlobalFlags = scanInstructions(MF, Worklist);
712 while (!Worklist.empty()) {
717 propagateInstruction(*WI.MI, Worklist);
719 propagateBlock(*WI.MBB, Worklist);
728 Register SaveReg =
MRI->createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass);
737 LIS->InsertMachineInstrInMaps(*Save);
738 LIS->InsertMachineInstrInMaps(*Restore);
739 LIS->createAndComputeVirtRegInterval(SaveReg);
750 BB->
splitAt(*TermMI,
true, LIS);
754 unsigned NewOpcode = 0;
756 case AMDGPU::S_AND_B32:
757 NewOpcode = AMDGPU::S_AND_B32_term;
759 case AMDGPU::S_AND_B64:
760 NewOpcode = AMDGPU::S_AND_B64_term;
762 case AMDGPU::S_MOV_B32:
763 NewOpcode = AMDGPU::S_MOV_B32_term;
765 case AMDGPU::S_MOV_B64:
766 NewOpcode = AMDGPU::S_MOV_B64_term;
779 DTUpdates.
push_back({DomTreeT::Insert, SplitBB, Succ});
780 DTUpdates.
push_back({DomTreeT::Delete, BB, Succ});
782 DTUpdates.
push_back({DomTreeT::Insert, BB, SplitBB});
784 MDT->getBase().applyUpdates(DTUpdates);
786 PDT->applyUpdates(DTUpdates);
792 LIS->InsertMachineInstrInMaps(*
MI);
812 switch (
MI.getOperand(2).getImm()) {
814 Opcode = AMDGPU::V_CMP_LG_F32_e64;
817 Opcode = AMDGPU::V_CMP_GE_F32_e64;
820 Opcode = AMDGPU::V_CMP_GT_F32_e64;
823 Opcode = AMDGPU::V_CMP_LE_F32_e64;
826 Opcode = AMDGPU::V_CMP_LT_F32_e64;
829 Opcode = AMDGPU::V_CMP_EQ_F32_e64;
832 Opcode = AMDGPU::V_CMP_O_F32_e64;
835 Opcode = AMDGPU::V_CMP_U_F32_e64;
839 Opcode = AMDGPU::V_CMP_NEQ_F32_e64;
843 Opcode = AMDGPU::V_CMP_NLT_F32_e64;
847 Opcode = AMDGPU::V_CMP_NLE_F32_e64;
851 Opcode = AMDGPU::V_CMP_NGT_F32_e64;
855 Opcode = AMDGPU::V_CMP_NGE_F32_e64;
859 Opcode = AMDGPU::V_CMP_NLG_F32_e64;
871 Register VCC =
ST->isWave32() ? AMDGPU::VCC_LO : AMDGPU::VCC;
904 LIS->ReplaceMachineInstrInMaps(
MI, *VcmpMI);
907 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);
908 LIS->InsertMachineInstrInMaps(*ExecMaskMI);
909 LIS->InsertMachineInstrInMaps(*EarlyTermMI);
910 LIS->InsertMachineInstrInMaps(*NewTerm);
920 const bool IsDemote = IsWQM && (
MI.getOpcode() == AMDGPU::SI_DEMOTE_I1);
922 int64_t KillVal =
MI.getOperand(1).getImm();
929 if (
Op.getImm() == KillVal) {
937 if (
MI.getOpcode() == AMDGPU::SI_DEMOTE_I1) {
938 LIS->RemoveMachineInstrFromMaps(
MI);
943 LIS->ReplaceMachineInstrInMaps(
MI, *NewTerm);
952 TmpReg =
MRI->createVirtualRegister(
TRI->getBoolRC());
953 ComputeKilledMaskMI =
978 LiveMaskWQM =
MRI->createVirtualRegister(
TRI->getBoolRC());
987 unsigned MovOpc =
ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
994 unsigned Opcode = KillVal ? AndN2Opc : AndOpc;
1001 LIS->RemoveMachineInstrFromMaps(
MI);
1006 if (ComputeKilledMaskMI)
1007 LIS->InsertMachineInstrInMaps(*ComputeKilledMaskMI);
1008 LIS->InsertMachineInstrInMaps(*MaskUpdateMI);
1009 LIS->InsertMachineInstrInMaps(*EarlyTermMI);
1011 LIS->InsertMachineInstrInMaps(*WQMMaskMI);
1012 LIS->InsertMachineInstrInMaps(*NewTerm);
1015 LIS->removeInterval(CndReg);
1016 LIS->createAndComputeVirtRegInterval(CndReg);
1019 LIS->createAndComputeVirtRegInterval(TmpReg);
1021 LIS->createAndComputeVirtRegInterval(LiveMaskWQM);
1034 const BlockInfo &BI = BII->second;
1035 if (!BI.NeedsLowering)
1041 char State = BI.InitialState;
1045 if (StateTransition.count(&
MI))
1046 State = StateTransition[&
MI];
1049 switch (
MI.getOpcode()) {
1050 case AMDGPU::SI_DEMOTE_I1:
1051 case AMDGPU::SI_KILL_I1_TERMINATOR:
1052 SplitPoint = lowerKillI1(
MBB,
MI, State == StateWQM);
1054 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1055 SplitPoint = lowerKillF32(
MBB,
MI);
1065 if (!SplitPoints.
empty()) {
1086 : LIS->getMBBEndIdx(&
MBB);
1088 Last != MBBE ? LIS->getInstructionIndex(*
Last) : LIS->getMBBEndIdx(&
MBB);
1099 if (Next < FirstIdx)
1104 assert(EndMI &&
"Segment does not end on valid instruction");
1108 SlotIndex Next = LIS->getInstructionIndex(*NextI);
1128 bool IsExecDef =
false;
1131 MO.getReg() == AMDGPU::EXEC_LO || MO.getReg() == AMDGPU::EXEC;
1149 if (!IsTerminator) {
1151 if (FirstTerm !=
MBB.
end()) {
1152 SlotIndex FirstTermIdx = LIS->getInstructionIndex(*FirstTerm);
1154 IsTerminator = BeforeIdx > FirstTermIdx;
1161 unsigned Opcode = IsTerminator ? AndSaveExecTermOpc : AndSaveExecOpc;
1165 unsigned Opcode = IsTerminator ? AndTermOpc : AndOpc;
1171 LIS->InsertMachineInstrInMaps(*
MI);
1172 StateTransition[
MI] = StateExact;
1187 LIS->InsertMachineInstrInMaps(*
MI);
1188 StateTransition[
MI] = StateWQM;
1193 Register SaveOrig,
char StrictStateNeeded) {
1196 assert(StrictStateNeeded == StateStrictWWM ||
1197 StrictStateNeeded == StateStrictWQM);
1199 if (StrictStateNeeded == StateStrictWWM) {
1208 LIS->InsertMachineInstrInMaps(*
MI);
1209 StateTransition[
MI] = StrictStateNeeded;
1214 Register SavedOrig,
char NonStrictState,
1215 char CurrentStrictState) {
1219 assert(CurrentStrictState == StateStrictWWM ||
1220 CurrentStrictState == StateStrictWQM);
1222 if (CurrentStrictState == StateStrictWWM) {
1231 LIS->InsertMachineInstrInMaps(*
MI);
1232 StateTransition[
MI] = NonStrictState;
1240 BlockInfo &BI = BII->second;
1244 if (!IsEntry && BI.Needs == StateWQM && BI.OutNeeds != StateExact) {
1245 BI.InitialState = StateWQM;
1254 bool WQMFromExec = IsEntry;
1255 char State = (IsEntry || !(BI.InNeeds & StateWQM)) ? StateExact : StateWQM;
1256 char NonStrictState = 0;
1262 if (
II != IE &&
II->getOpcode() == AMDGPU::COPY &&
1263 II->getOperand(1).getReg() ==
TRI->getExec())
1278 BI.InitialState = State;
1282 char Needs = StateExact | StateWQM;
1288 if (FirstStrict == IE)
1296 if (
MI.isTerminator() ||
TII->mayReadEXEC(*
MRI,
MI)) {
1299 if (III->second.Needs & StateStrictWWM)
1300 Needs = StateStrictWWM;
1301 else if (III->second.Needs & StateStrictWQM)
1302 Needs = StateStrictWQM;
1303 else if (III->second.Needs & StateWQM)
1306 Needs &= ~III->second.Disabled;
1307 OutNeeds = III->second.OutNeeds;
1312 Needs = StateExact | StateWQM | StateStrict;
1316 if (
MI.isBranch() && OutNeeds == StateExact)
1322 if (BI.OutNeeds & StateWQM)
1324 else if (BI.OutNeeds == StateExact)
1327 Needs = StateWQM | StateExact;
1331 if (!(Needs & State)) {
1333 if (State == StateStrictWWM || Needs == StateStrictWWM ||
1334 State == StateStrictWQM || Needs == StateStrictWQM) {
1336 First = FirstStrict;
1343 bool SaveSCC =
false;
1346 case StateStrictWWM:
1347 case StateStrictWQM:
1351 SaveSCC = (Needs & StateStrict) || ((Needs & StateWQM) && WQMFromExec);
1355 SaveSCC = !(Needs & StateWQM);
1362 prepareInsertion(
MBB,
First,
II, Needs == StateWQM, SaveSCC);
1364 if (State & StateStrict) {
1365 assert(State == StateStrictWWM || State == StateStrictWQM);
1366 assert(SavedNonStrictReg);
1367 fromStrictMode(
MBB,
Before, SavedNonStrictReg, NonStrictState, State);
1369 LIS->createAndComputeVirtRegInterval(SavedNonStrictReg);
1370 SavedNonStrictReg = 0;
1371 State = NonStrictState;
1374 if (Needs & StateStrict) {
1375 NonStrictState = State;
1376 assert(Needs == StateStrictWWM || Needs == StateStrictWQM);
1377 assert(!SavedNonStrictReg);
1378 SavedNonStrictReg =
MRI->createVirtualRegister(BoolRC);
1380 toStrictMode(
MBB,
Before, SavedNonStrictReg, Needs);
1384 if (State == StateWQM && (Needs & StateExact) && !(Needs & StateWQM)) {
1385 if (!WQMFromExec && (OutNeeds & StateWQM)) {
1387 SavedWQMReg =
MRI->createVirtualRegister(BoolRC);
1392 }
else if (State == StateExact && (Needs & StateWQM) &&
1393 !(Needs & StateExact)) {
1394 assert(WQMFromExec == (SavedWQMReg == 0));
1399 LIS->createAndComputeVirtRegInterval(SavedWQMReg);
1412 if (Needs != (StateExact | StateWQM | StateStrict)) {
1413 if (Needs != (StateExact | StateWQM))
1424 assert(!SavedNonStrictReg);
1427void SIWholeQuadMode::lowerLiveMaskQueries() {
1436 LIS->ReplaceMachineInstrInMaps(*
MI, *Copy);
1437 MI->eraseFromParent();
1441void SIWholeQuadMode::lowerCopyInstrs() {
1443 assert(
MI->getNumExplicitOperands() == 2);
1448 TRI->getRegClassForOperandReg(*
MRI,
MI->getOperand(0));
1449 if (
TRI->isVGPRClass(regClass)) {
1450 const unsigned MovOp =
TII->getMovOpcode(regClass);
1451 MI->setDesc(
TII->get(MovOp));
1456 return MO.isUse() && MO.getReg() == AMDGPU::EXEC;
1462 if (
MI->getOperand(0).isEarlyClobber()) {
1463 LIS->removeInterval(Reg);
1464 MI->getOperand(0).setIsEarlyClobber(
false);
1465 LIS->createAndComputeVirtRegInterval(Reg);
1467 int Index =
MI->findRegisterUseOperandIdx(AMDGPU::EXEC,
nullptr);
1468 while (
Index >= 0) {
1470 Index =
MI->findRegisterUseOperandIdx(AMDGPU::EXEC,
nullptr);
1472 MI->setDesc(
TII->get(AMDGPU::COPY));
1477 if (
MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32 ||
1478 MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B64) {
1479 assert(
MI->getNumExplicitOperands() == 3);
1483 assert(
MI->getOperand(2).isUndef());
1484 MI->removeOperand(2);
1485 MI->untieRegOperand(1);
1487 assert(
MI->getNumExplicitOperands() == 2);
1490 unsigned CopyOp =
MI->getOperand(1).isReg()
1492 :
TII->getMovOpcode(
TRI->getRegClassForOperandReg(
1493 *
MRI,
MI->getOperand(0)));
1494 MI->setDesc(
TII->get(CopyOp));
1498void SIWholeQuadMode::lowerKillInstrs(
bool IsWQM) {
1502 switch (
MI->getOpcode()) {
1503 case AMDGPU::SI_DEMOTE_I1:
1504 case AMDGPU::SI_KILL_I1_TERMINATOR:
1505 SplitPoint = lowerKillI1(*
MBB, *
MI, IsWQM);
1507 case AMDGPU::SI_KILL_F32_COND_IMM_TERMINATOR:
1508 SplitPoint = lowerKillF32(*
MBB, *
MI);
1518 bool IsWave32 =
ST->isWave32();
1520 if (
MI.getOpcode() == AMDGPU::SI_INIT_EXEC) {
1524 TII->get(IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64),
1526 .
addImm(
MI.getOperand(0).getImm());
1528 LIS->RemoveMachineInstrFromMaps(
MI);
1529 LIS->InsertMachineInstrInMaps(*InitMI);
1531 MI.eraseFromParent();
1542 Register InputReg =
MI.getOperand(0).getReg();
1548 if (DefInstr != FirstMI) {
1554 LIS->handleMove(*DefInstr);
1566 Register CountReg =
MRI->createVirtualRegister(&AMDGPU::SGPR_32RegClass);
1567 auto BfeMI =
BuildMI(*
MBB, FirstMI,
DL,
TII->get(AMDGPU::S_BFE_U32), CountReg)
1569 .
addImm((
MI.getOperand(1).getImm() & Mask) | 0x70000);
1572 TII->get(IsWave32 ? AMDGPU::S_BFM_B32 : AMDGPU::S_BFM_B64), Exec)
1575 auto CmpMI =
BuildMI(*
MBB, FirstMI,
DL,
TII->get(AMDGPU::S_CMP_EQ_U32))
1580 TII->get(IsWave32 ? AMDGPU::S_CMOV_B32 : AMDGPU::S_CMOV_B64),
1585 MI.eraseFromParent();
1589 LIS->RemoveMachineInstrFromMaps(
MI);
1590 MI.eraseFromParent();
1592 LIS->InsertMachineInstrInMaps(*BfeMI);
1593 LIS->InsertMachineInstrInMaps(*BfmMI);
1594 LIS->InsertMachineInstrInMaps(*CmpMI);
1595 LIS->InsertMachineInstrInMaps(*CmovMI);
1597 LIS->removeInterval(InputReg);
1598 LIS->createAndComputeVirtRegInterval(InputReg);
1599 LIS->createAndComputeVirtRegInterval(CountReg);
1612 if (
MI->getParent() == &Entry)
1613 InsertPt = std::next(
MI->getIterator());
1623 <<
" ------------- \n");
1628 LiveMaskQueries.clear();
1629 LowerToCopyInstrs.clear();
1630 LowerToMovInstrs.clear();
1632 InitExecInstrs.clear();
1633 StateTransition.clear();
1637 TII =
ST->getInstrInfo();
1638 TRI = &
TII->getRegisterInfo();
1640 LIS = &getAnalysis<LiveIntervals>();
1641 auto *MDTWrapper = getAnalysisIfAvailable<MachineDominatorTreeWrapperPass>();
1642 MDT = MDTWrapper ? &MDTWrapper->getDomTree() :
nullptr;
1644 getAnalysisIfAvailable<MachinePostDominatorTreeWrapperPass>();
1645 PDT = PDTWrapper ? &PDTWrapper->getPostDomTree() :
nullptr;
1647 if (
ST->isWave32()) {
1648 AndOpc = AMDGPU::S_AND_B32;
1649 AndTermOpc = AMDGPU::S_AND_B32_term;
1650 AndN2Opc = AMDGPU::S_ANDN2_B32;
1651 XorOpc = AMDGPU::S_XOR_B32;
1652 AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B32;
1653 AndSaveExecTermOpc = AMDGPU::S_AND_SAVEEXEC_B32_term;
1654 WQMOpc = AMDGPU::S_WQM_B32;
1655 Exec = AMDGPU::EXEC_LO;
1657 AndOpc = AMDGPU::S_AND_B64;
1658 AndTermOpc = AMDGPU::S_AND_B64_term;
1659 AndN2Opc = AMDGPU::S_ANDN2_B64;
1660 XorOpc = AMDGPU::S_XOR_B64;
1661 AndSaveExecOpc = AMDGPU::S_AND_SAVEEXEC_B64;
1662 AndSaveExecTermOpc = AMDGPU::S_AND_SAVEEXEC_B64_term;
1663 WQMOpc = AMDGPU::S_WQM_B64;
1664 Exec = AMDGPU::EXEC;
1668 const bool NeedsLiveMask = !(KillInstrs.empty() && LiveMaskQueries.empty());
1676 if (!(GlobalFlags & (StateWQM | StateStrict)) && LowerToCopyInstrs.empty() &&
1677 LowerToMovInstrs.empty() && KillInstrs.empty()) {
1678 lowerLiveMaskQueries();
1679 return !InitExecInstrs.empty() || !LiveMaskQueries.empty();
1683 if (NeedsLiveMask || (GlobalFlags & StateWQM)) {
1684 LiveMaskReg =
MRI->createVirtualRegister(
TRI->getBoolRC());
1688 LIS->InsertMachineInstrInMaps(*
MI);
1693 lowerLiveMaskQueries();
1697 if (GlobalFlags == StateWQM) {
1700 LIS->InsertMachineInstrInMaps(*
MI);
1701 lowerKillInstrs(
true);
1704 processBlock(*BII.first, BII.first == &Entry);
1707 lowerBlock(*BII.first);
1711 if (LiveMaskReg != Exec)
1712 LIS->createAndComputeVirtRegInterval(LiveMaskReg);
1717 LIS->removeAllRegUnitsForPhysReg(AMDGPU::SCC);
1720 if (!KillInstrs.empty())
1721 LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Provides AMDGPU specific target descriptions.
static void analyzeFunction(Function &Fn, const DataLayout &Layout, FunctionVarLocsBuilder *FnVarLocs)
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
AMD GCN specific subclass of TargetSubtarget.
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
This file implements a map that provides insertion order iteration.
uint64_t IntrinsicInst * II
#define INITIALIZE_PASS_DEPENDENCY(depName)
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI, MachineDominatorTree *MDT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Represent the analysis usage information of a pass.
AnalysisUsage & addRequired()
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
This class represents an Operation in the Expression.
Core dominator tree base class.
FunctionPass class - This class is used to implement most global optimizations.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Result of a LiveRange query.
VNInfo * valueIn() const
Return the value that is live-in to the instruction.
This class represents the liveness of a register, stack slot, etc.
const Segment * getSegmentContaining(SlotIndex Idx) const
Return the segment that contains the specified index, or null if there is none.
LiveQueryResult Query(SlotIndex Idx) const
Query Liveness at Idx.
VNInfo * getVNInfoBefore(SlotIndex Idx) const
getVNInfoBefore - Return the VNInfo that is live up to but not necessarilly including Idx,...
static MCRegister from(unsigned Val)
Check the provided unsigned value is a valid MCRegister.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
succ_iterator succ_begin()
MachineInstr * remove(MachineInstr *I)
Remove the unbundled instruction from the instruction list without deleting it.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned succ_size() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
pred_iterator pred_begin()
MachineBasicBlock * splitAt(MachineInstr &SplitInst, bool UpdateLiveIns=true, LiveIntervals *LIS=nullptr)
Split a basic block into 2 pieces at SplitPoint.
iterator_range< succ_iterator > successors()
reverse_iterator rbegin()
iterator_range< pred_iterator > predecessors()
Analysis pass which computes a MachineDominatorTree.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual MachineFunctionProperties getClearedProperties() const
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
StringRef getName() const
getName - Return the name of the corresponding LLVM function.
void dump() const
dump - Print the current MachineFunction to cerr, useful for debugger use.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
MachineInstr * removeFromParent()
Unlink 'this' from the containing basic block, and return it without deleting it.
const MachineBasicBlock * getParent() const
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Register getReg() const
getReg - Returns the register number.
MachinePostDominatorTree - an analysis pass wrapper for DominatorTree used to compute the post-domina...
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements a map that also provides access to all stored values in a deterministic order.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
SlotIndex - An opaque wrapper around machine indexes.
SlotIndex getBaseIndex() const
Returns the base index for associated with this index.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
A Use represents the edge between a Value definition and its users.
VNInfo - Value Number Information.
LLVM Value Representation.
self_iterator getIterator()
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char WavefrontSize[]
Key for Kernel::CodeProps::Metadata::mWavefrontSize.
LLVM_READONLY int getVOPe32(uint16_t Opcode)
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_PS
Used for Mesa/AMDPAL pixel shaders.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
@ Define
Register definition.
@ Kill
The last use of a register.
Reg
All possible values of the reg field in the ModR/M byte.
NodeAddr< PhiNode * > Phi
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
FunctionPass * createSIWholeQuadModePass()
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Printable printMBBReference(const MachineBasicBlock &MBB)
Prints a machine basic block reference.
static constexpr LaneBitmask getAll()
constexpr bool any() const
static constexpr LaneBitmask getNone()
This represents a simple continuous liveness interval for a value.