44#define DEBUG_TYPE "gi-combiner"
47using namespace MIPatternMatch;
53 cl::desc(
"Force all indexed operations to be "
54 "legal for the GlobalISel combiner"));
60 : Builder(
B),
MRI(Builder.getMF().getRegInfo()), Observer(Observer), KB(KB),
61 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
62 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
63 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
76 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
95 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
96 return ByteWidth -
I - 1;
116static std::optional<bool>
120 unsigned Width = MemOffset2Idx.
size();
123 bool BigEndian =
true, LittleEndian =
true;
124 for (
unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
125 auto MemOffsetAndIdx = MemOffset2Idx.
find(MemOffset);
126 if (MemOffsetAndIdx == MemOffset2Idx.
end())
128 const int64_t
Idx = MemOffsetAndIdx->second - LowestIdx;
129 assert(
Idx >= 0 &&
"Expected non-negative byte offset?");
132 if (!BigEndian && !LittleEndian)
136 assert((BigEndian != LittleEndian) &&
137 "Pattern cannot be both big and little endian!");
144 assert(
LI &&
"Must have LegalizerInfo to query isLegal!");
160 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
161 isLegal({TargetOpcode::G_CONSTANT, {EltTy}});
188 unsigned ToOpcode)
const {
213 if (
MI.getOpcode() != TargetOpcode::COPY)
222 MI.eraseFromParent();
228 assert(
MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
229 "Invalid instruction");
239 assert(Def &&
"Operand not defined");
242 switch (Def->getOpcode()) {
243 case TargetOpcode::G_BUILD_VECTOR:
250 case TargetOpcode::G_IMPLICIT_DEF: {
259 "All undefs should have the same type");
263 EltIdx != EltEnd; ++EltIdx)
264 Ops.
push_back(Undef->getOperand(0).getReg());
275 {TargetOpcode::G_BUILD_VECTOR, {DstTy,
MRI.
getType(Ops[0])}})) {
302 MI.eraseFromParent();
313 if (!ConcatMI1 || !ConcatMI2)
317 if (
MRI.
getType(ConcatMI1->getSourceReg(0)) !=
324 for (
unsigned i = 0; i < Mask.size(); i += ConcatSrcNumElt) {
328 for (
unsigned j = 1; j < ConcatSrcNumElt; j++) {
329 if (i + j >= Mask.size())
331 if (Mask[i + j] != -1)
335 {TargetOpcode::G_IMPLICIT_DEF, {ConcatSrcTy}}))
338 }
else if (Mask[i] % ConcatSrcNumElt == 0) {
339 for (
unsigned j = 1; j < ConcatSrcNumElt; j++) {
340 if (i + j >= Mask.size())
342 if (Mask[i + j] != Mask[i] +
static_cast<int>(j))
348 Ops.
push_back(ConcatMI1->getSourceReg(Mask[i] / ConcatSrcNumElt));
350 Ops.
push_back(ConcatMI2->getSourceReg(Mask[i] / ConcatSrcNumElt -
351 ConcatMI1->getNumSources()));
359 {TargetOpcode::G_CONCAT_VECTORS,
360 {
MRI.
getType(
MI.getOperand(0).getReg()), ConcatSrcTy}}))
371 for (
unsigned i = 0; i < Ops.
size(); i++) {
380 MI.eraseFromParent();
394 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
395 "Invalid instruction kind");
420 if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1)
425 if (DstNumElts % SrcNumElts != 0)
431 unsigned NumConcat = DstNumElts / SrcNumElts;
434 for (
unsigned i = 0; i != DstNumElts; ++i) {
441 if ((
Idx % SrcNumElts != (i % SrcNumElts)) ||
442 (ConcatSrcs[i / SrcNumElts] >= 0 &&
443 ConcatSrcs[i / SrcNumElts] != (
int)(
Idx / SrcNumElts)))
446 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
453 for (
auto Src : ConcatSrcs) {
479 MI.eraseFromParent();
484 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
485 "Invalid instruction kind");
488 return Mask.size() == 1;
495 int I =
MI.getOperand(3).getShuffleMask()[0];
500 if (
I >= Src1NumElts) {
501 SrcReg =
MI.getOperand(2).getReg();
513 MI.eraseFromParent();
522 const LLT TyForCandidate,
523 unsigned OpcodeForCandidate,
528 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
539 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
542 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ANYEXT &&
543 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
544 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
550 if (!isa<GZExtLoad>(LoadMI) && CurrentUse.
Ty == TyForCandidate) {
552 OpcodeForCandidate == TargetOpcode::G_ZEXT)
554 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ZEXT &&
555 OpcodeForCandidate == TargetOpcode::G_SEXT)
556 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
565 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
576static void InsertInsnsWithoutSideEffectsBeforeUse(
588 InsertBB = PredBB->
getMBB();
593 if (InsertBB ==
DefMI.getParent()) {
595 Inserter(InsertBB, std::next(InsertPt), UseMO);
614 unsigned CandidateLoadOpc;
616 case TargetOpcode::G_ANYEXT:
617 CandidateLoadOpc = TargetOpcode::G_LOAD;
619 case TargetOpcode::G_SEXT:
620 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
622 case TargetOpcode::G_ZEXT:
623 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
628 return CandidateLoadOpc;
659 if (!llvm::has_single_bit<uint32_t>(LoadValueTy.
getSizeInBits()))
667 unsigned PreferredOpcode =
669 ? TargetOpcode::G_ANYEXT
670 : isa<GSExtLoad>(&
MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
671 Preferred = {
LLT(), PreferredOpcode,
nullptr};
673 if (
UseMI.getOpcode() == TargetOpcode::G_SEXT ||
674 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
675 (
UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
676 const auto &MMO = LoadMI->
getMMO();
686 if (
LI->
getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
690 Preferred = ChoosePreferredUse(
MI, Preferred,
701 assert(Preferred.Ty != LoadValueTy &&
"Extending to same type?");
719 if (PreviouslyEmitted) {
729 EmittedInsns[InsertIntoBB] = NewMI;
738 auto &LoadValue =
MI.getOperand(0);
741 Uses.push_back(&UseMO);
743 for (
auto *UseMO :
Uses) {
753 if (UseDstReg != ChosenDstReg) {
754 if (Preferred.
Ty == UseDstTy) {
791 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO,
806 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO, InsertTruncAt);
809 MI.getOperand(0).setReg(ChosenDstReg);
815 assert(
MI.getOpcode() == TargetOpcode::G_AND);
834 APInt MaskVal = MaybeMask->Value;
855 if (MaskSizeBits > LoadSizeBits.
getValue())
875 else if (LoadSizeBits.
getValue() > MaskSizeBits ||
881 {TargetOpcode::G_ZEXTLOAD, {RegTy,
MRI.
getType(PtrReg)}, {MemDesc}}))
885 B.setInstrAndDebugLoc(*LoadMI);
886 auto &MF =
B.getMF();
888 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, MemDesc.
MemoryTy);
889 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);
898 "shouldn't consider debug uses");
906 if (DefOrUse ==
MBB.
end())
908 return &*DefOrUse == &
DefMI;
914 "shouldn't consider debug uses");
917 else if (
DefMI.getParent() !=
UseMI.getParent())
924 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
935 uint64_t SizeInBits =
MI.getOperand(2).getImm();
938 if (
auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser,
MRI)) {
940 auto LoadSizeBits = LoadMI->getMemSizeInBits();
944 if (LoadSizeBits == SizeInBits)
951 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
953 MI.eraseFromParent();
958 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
968 auto *LoadDef = getOpcodeDef<GLoad>(SrcReg,
MRI);
972 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
977 unsigned NewSizeBits = std::min((
uint64_t)
MI.getOperand(2).getImm(), MemBits);
992 if (LoadDef->isSimple())
994 else if (MemBits > NewSizeBits || MemBits == RegTy.
getSizeInBits())
1004 MatchInfo = std::make_tuple(LoadDef->getDstReg(), NewSizeBits);
1010 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
1012 unsigned ScalarSizeBits;
1013 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;
1022 auto &MMO = LoadDef->
getMMO();
1025 auto PtrInfo = MMO.getPointerInfo();
1029 MI.eraseFromParent();
1044 auto *MF =
MI->getMF();
1045 auto *
Addr = getOpcodeDef<GPtrAdd>(
MI->getPointerReg(),
MRI);
1051 AM.
BaseOffs = CstOff->getSExtValue();
1056 MF->getDataLayout(), AM,
1058 MF->getFunction().getContext()),
1059 MI->getMMO().getAddrSpace());
1064 case TargetOpcode::G_LOAD:
1065 return TargetOpcode::G_INDEXED_LOAD;
1066 case TargetOpcode::G_STORE:
1067 return TargetOpcode::G_INDEXED_STORE;
1068 case TargetOpcode::G_ZEXTLOAD:
1069 return TargetOpcode::G_INDEXED_ZEXTLOAD;
1070 case TargetOpcode::G_SEXTLOAD:
1071 return TargetOpcode::G_INDEXED_SEXTLOAD;
1077bool CombinerHelper::isIndexedLoadStoreLegal(
GLoadStore &LdSt)
const {
1086 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1087 OpTys = {PtrTy, Ty, Ty};
1089 OpTys = {Ty, PtrTy};
1097 cl::desc(
"Number of uses of a base pointer to check before it is no longer "
1098 "considered for post-indexing."));
1102 bool &RematOffset) {
1115 if (!isIndexedLoadStoreLegal(LdSt))
1124 unsigned NumUsesChecked = 0;
1129 auto *PtrAdd = dyn_cast<GPtrAdd>(&
Use);
1137 if (StoredValDef == &
Use)
1140 Offset = PtrAdd->getOffsetReg();
1142 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(),
Offset,
1148 RematOffset =
false;
1152 if (OffsetDef->
getOpcode() != TargetOpcode::G_CONSTANT)
1158 if (&BasePtrUse == PtrDef)
1163 auto *BasePtrLdSt = dyn_cast<GLoadStore>(&BasePtrUse);
1164 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1166 isIndexedLoadStoreLegal(*BasePtrLdSt))
1171 if (
auto *BasePtrUseDef = dyn_cast<GPtrAdd>(&BasePtrUse)) {
1172 Register PtrAddDefReg = BasePtrUseDef->getReg(0);
1176 if (BaseUseUse.getParent() != LdSt.
getParent())
1179 if (
auto *UseUseLdSt = dyn_cast<GLoadStore>(&BaseUseUse))
1188 Addr = PtrAdd->getReg(0);
1189 Base = PtrAdd->getBaseReg();
1210 if (!isIndexedLoadStoreLegal(LdSt))
1214 if (BaseDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
1217 if (
auto *St = dyn_cast<GStore>(&LdSt)) {
1219 if (
Base == St->getValueReg())
1224 if (St->getValueReg() ==
Addr)
1230 if (AddrUse.getParent() != LdSt.
getParent())
1235 bool RealUse =
false;
1242 if (
auto *UseLdSt = dyn_cast<GLoadStore>(&AddrUse)) {
1254 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1257 auto *LoadMI = getOpcodeDef<GLoad>(
MI.getOperand(1).getReg(),
MRI);
1271 if (!LoadMI->isSimple())
1283 const unsigned MaxIter = 20;
1285 for (
auto II = LoadMI->
getIterator(), IE =
MI.getIterator(); II != IE; ++II) {
1286 if (II->isLoadFoldBarrier())
1288 if (Iter++ == MaxIter)
1304 int Elt = CVal->getZExtValue();
1317 Register VecPtr = LoadMI->getPointerReg();
1325 LegalityQuery Q = {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}};
1350 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);
1360 auto &LdSt = cast<GLoadStore>(
MI);
1365 MatchInfo.
IsPre = findPreIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1367 if (!MatchInfo.
IsPre &&
1368 !findPostIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1378 unsigned Opcode =
MI.getOpcode();
1379 bool IsStore = Opcode == TargetOpcode::G_STORE;
1387 *OldCst->getOperand(1).getCImm());
1388 MatchInfo.
Offset = NewCst.getReg(0);
1394 MIB.
addUse(
MI.getOperand(0).getReg());
1396 MIB.
addDef(
MI.getOperand(0).getReg());
1404 MI.eraseFromParent();
1412 unsigned Opcode =
MI.getOpcode();
1413 bool IsDiv, IsSigned;
1418 case TargetOpcode::G_SDIV:
1419 case TargetOpcode::G_UDIV: {
1421 IsSigned = Opcode == TargetOpcode::G_SDIV;
1424 case TargetOpcode::G_SREM:
1425 case TargetOpcode::G_UREM: {
1427 IsSigned = Opcode == TargetOpcode::G_SREM;
1433 unsigned DivOpcode, RemOpcode, DivremOpcode;
1435 DivOpcode = TargetOpcode::G_SDIV;
1436 RemOpcode = TargetOpcode::G_SREM;
1437 DivremOpcode = TargetOpcode::G_SDIVREM;
1439 DivOpcode = TargetOpcode::G_UDIV;
1440 RemOpcode = TargetOpcode::G_UREM;
1441 DivremOpcode = TargetOpcode::G_UDIVREM;
1460 if (
MI.getParent() ==
UseMI.getParent() &&
1461 ((IsDiv &&
UseMI.getOpcode() == RemOpcode) ||
1462 (!IsDiv &&
UseMI.getOpcode() == DivOpcode)) &&
1475 unsigned Opcode =
MI.getOpcode();
1476 assert(OtherMI &&
"OtherMI shouldn't be empty.");
1479 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1480 DestDivReg =
MI.getOperand(0).getReg();
1484 DestRemReg =
MI.getOperand(0).getReg();
1488 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1498 : TargetOpcode::G_UDIVREM,
1499 {DestDivReg, DestRemReg},
1501 MI.eraseFromParent();
1507 assert(
MI.getOpcode() == TargetOpcode::G_BR);
1526 assert(std::next(BrIt) ==
MBB->
end() &&
"expected G_BR to be a terminator");
1528 BrCond = &*std::prev(BrIt);
1529 if (BrCond->
getOpcode() != TargetOpcode::G_BRCOND)
1535 return BrCondTarget !=
MI.getOperand(0).getMBB() &&
1553 MI.getOperand(0).setMBB(FallthroughBB);
1569 return Helper.lowerMemcpyInline(
MI) ==
1585 switch (
MI.getOpcode()) {
1588 case TargetOpcode::G_FNEG: {
1589 Result.changeSign();
1592 case TargetOpcode::G_FABS: {
1596 case TargetOpcode::G_FPTRUNC: {
1598 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1603 case TargetOpcode::G_FSQRT: {
1607 Result =
APFloat(sqrt(Result.convertToDouble()));
1610 case TargetOpcode::G_FLOG2: {
1631 MI.eraseFromParent();
1642 if (
MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1652 if (!Add2Def || Add2Def->
getOpcode() != TargetOpcode::G_PTR_ADD)
1665 Type *AccessTy =
nullptr;
1666 auto &MF = *
MI.getMF();
1668 if (
auto *LdSt = dyn_cast<GLoadStore>(&
UseMI)) {
1670 MF.getFunction().getContext());
1675 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1680 AMOld.
BaseOffs = MaybeImmVal->Value.getSExtValue();
1683 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1684 if (TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&
1685 !TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))
1698 assert(
MI.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
1704 MI.getOperand(1).setReg(MatchInfo.
Base);
1705 MI.getOperand(2).setReg(NewOffset.getReg(0));
1718 unsigned Opcode =
MI.getOpcode();
1719 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1720 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1721 Opcode == TargetOpcode::G_USHLSAT) &&
1722 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1742 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1747 if (Opcode == TargetOpcode::G_USHLSAT &&
1756 unsigned Opcode =
MI.getOpcode();
1757 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1758 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1759 Opcode == TargetOpcode::G_USHLSAT) &&
1760 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1764 auto Imm = MatchInfo.
Imm;
1766 if (Imm >= ScalarSizeInBits) {
1768 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1770 MI.eraseFromParent();
1775 Imm = ScalarSizeInBits - 1;
1781 MI.getOperand(1).setReg(MatchInfo.
Reg);
1782 MI.getOperand(2).setReg(NewImm);
1798 unsigned ShiftOpcode =
MI.getOpcode();
1799 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1800 ShiftOpcode == TargetOpcode::G_ASHR ||
1801 ShiftOpcode == TargetOpcode::G_LSHR ||
1802 ShiftOpcode == TargetOpcode::G_USHLSAT ||
1803 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
1804 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1807 Register LogicDest =
MI.getOperand(1).getReg();
1812 unsigned LogicOpcode = LogicMI->
getOpcode();
1813 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1814 LogicOpcode != TargetOpcode::G_XOR)
1818 const Register C1 =
MI.getOperand(2).getReg();
1820 if (!MaybeImmVal || MaybeImmVal->Value == 0)
1823 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
1827 if (
MI->getOpcode() != ShiftOpcode ||
1837 ShiftVal = MaybeImmVal->Value.getSExtValue();
1848 if (matchFirstShift(LogicMIOp1, C0Val)) {
1850 MatchInfo.
Shift2 = LogicMIOp1;
1851 }
else if (matchFirstShift(LogicMIOp2, C0Val)) {
1853 MatchInfo.
Shift2 = LogicMIOp2;
1857 MatchInfo.
ValSum = C0Val + C1Val;
1863 MatchInfo.
Logic = LogicMI;
1869 unsigned Opcode =
MI.getOpcode();
1870 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1871 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
1872 Opcode == TargetOpcode::G_SSHLSAT) &&
1873 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1891 Register Shift2Const =
MI.getOperand(2).getReg();
1903 MI.eraseFromParent();
1907 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
"Expected G_SHL");
1910 auto &Shl = cast<GenericMachineInstr>(
MI);
1930 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD ||
1931 SrcDef->getOpcode() == TargetOpcode::G_OR) &&
"Unexpected op");
1934 auto S1 =
B.buildShl(SrcTy,
X, ShiftReg);
1935 auto S2 =
B.buildShl(SrcTy, C1, ShiftReg);
1936 B.buildInstr(SrcDef->getOpcode(), {DstReg}, {
S1, S2});
1942 unsigned &ShiftVal) {
1943 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1949 ShiftVal = MaybeImmVal->Value.exactLogBase2();
1950 return (
static_cast<int32_t
>(ShiftVal) != -1);
1954 unsigned &ShiftVal) {
1955 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1960 MI.setDesc(MIB.
getTII().
get(TargetOpcode::G_SHL));
1961 MI.getOperand(2).setReg(ShiftCst.getReg(0));
1968 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
KB);
1983 if (!MaybeShiftAmtVal)
1997 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
1998 MatchData.
Reg = ExtSrc;
1999 MatchData.
Imm = ShiftAmt;
2003 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
2009 int64_t ShiftAmtVal = MatchData.
Imm;
2016 MI.eraseFromParent();
2023 for (
unsigned I = 0;
I <
Merge.getNumSources(); ++
I)
2026 auto *Unmerge = getOpcodeDef<GUnmerge>(MergedValues[0],
MRI);
2027 if (!Unmerge || Unmerge->getNumDefs() !=
Merge.getNumSources())
2030 for (
unsigned I = 0;
I < MergedValues.
size(); ++
I)
2031 if (MergedValues[
I] != Unmerge->getReg(
I))
2034 MatchInfo = Unmerge->getSourceReg();
2048 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2049 "Expected an unmerge");
2050 auto &Unmerge = cast<GUnmerge>(
MI);
2053 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(SrcReg,
MRI);
2061 if (SrcMergeTy != Dst0Ty && !SameSize)
2065 for (
unsigned Idx = 0;
Idx < SrcInstr->getNumSources(); ++
Idx)
2072 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2073 "Expected an unmerge");
2075 "Not enough operands to replace all defs");
2076 unsigned NumElems =
MI.getNumOperands() - 1;
2080 bool CanReuseInputDirectly = DstTy == SrcTy;
2081 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2093 if (CanReuseInputDirectly)
2098 MI.eraseFromParent();
2103 unsigned SrcIdx =
MI.getNumOperands() - 1;
2104 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2106 if (SrcInstr->
getOpcode() != TargetOpcode::G_CONSTANT &&
2107 SrcInstr->
getOpcode() != TargetOpcode::G_FCONSTANT)
2118 for (
unsigned Idx = 0;
Idx != SrcIdx; ++
Idx) {
2120 Val = Val.
lshr(ShiftAmt);
2128 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2129 "Expected an unmerge");
2131 "Not enough operands to replace all defs");
2132 unsigned NumElems =
MI.getNumOperands() - 1;
2133 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2138 MI.eraseFromParent();
2143 unsigned SrcIdx =
MI.getNumOperands() - 1;
2144 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2146 unsigned NumElems =
MI.getNumOperands() - 1;
2147 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2149 B.buildUndef(DstReg);
2156 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2157 "Expected an unmerge");
2162 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2170 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2171 Register Dst0Reg =
MI.getOperand(0).getReg();
2173 MI.eraseFromParent();
2177 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2178 "Expected an unmerge");
2179 Register Dst0Reg =
MI.getOperand(0).getReg();
2186 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2203 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2204 "Expected an unmerge");
2206 Register Dst0Reg =
MI.getOperand(0).getReg();
2211 "Expecting a G_ZEXT");
2221 "ZExt src doesn't fit in destination");
2226 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2231 MI.eraseFromParent();
2235 unsigned TargetShiftSize,
2236 unsigned &ShiftVal) {
2237 assert((
MI.getOpcode() == TargetOpcode::G_SHL ||
2238 MI.getOpcode() == TargetOpcode::G_LSHR ||
2239 MI.getOpcode() == TargetOpcode::G_ASHR) &&
"Expected a shift");
2247 if (
Size <= TargetShiftSize)
2255 ShiftVal = MaybeImmVal->Value.getSExtValue();
2256 return ShiftVal >=
Size / 2 && ShiftVal <
Size;
2260 const unsigned &ShiftVal) {
2265 unsigned HalfSize =
Size / 2;
2266 assert(ShiftVal >= HalfSize);
2271 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2273 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
2274 Register Narrowed = Unmerge.getReg(1);
2281 if (NarrowShiftAmt != 0) {
2288 }
else if (
MI.getOpcode() == TargetOpcode::G_SHL) {
2289 Register Narrowed = Unmerge.getReg(0);
2294 if (NarrowShiftAmt != 0) {
2302 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
2304 HalfTy, Unmerge.getReg(1),
2307 if (ShiftVal == HalfSize) {
2311 }
else if (ShiftVal ==
Size - 1) {
2319 HalfTy, Unmerge.getReg(1),
2328 MI.eraseFromParent();
2332 unsigned TargetShiftAmount) {
2343 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2352 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2355 MI.eraseFromParent();
2359 assert(
MI.getOpcode() == TargetOpcode::G_PTRTOINT &&
"Expected a G_PTRTOINT");
2362 MI.eraseFromParent();
2367 assert(
MI.getOpcode() == TargetOpcode::G_ADD);
2374 PtrReg.second =
false;
2384 PtrReg.second =
true;
2396 const bool DoCommute = PtrReg.second;
2405 MI.eraseFromParent();
2410 auto &PtrAdd = cast<GPtrAdd>(
MI);
2421 NewCst += RHSCst->
sextOrTrunc(DstTy.getSizeInBits());
2431 auto &PtrAdd = cast<GPtrAdd>(
MI);
2435 PtrAdd.eraseFromParent();
2439 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT &&
"Expected a G_ANYEXT");
2444 SrcReg = OriginalSrcReg;
2451 assert(
MI.getOpcode() == TargetOpcode::G_ZEXT &&
"Expected a G_ZEXT");
2466 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2467 MI.getOpcode() == TargetOpcode::G_SEXT ||
2468 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2469 "Expected a G_[ASZ]EXT");
2473 SrcReg = OriginalSrcReg;
2476 unsigned Opc =
MI.getOpcode();
2478 if (Opc == SrcOpc ||
2479 (Opc == TargetOpcode::G_ANYEXT &&
2480 (SrcOpc == TargetOpcode::G_SEXT || SrcOpc == TargetOpcode::G_ZEXT)) ||
2481 (Opc == TargetOpcode::G_SEXT && SrcOpc == TargetOpcode::G_ZEXT)) {
2490 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2491 MI.getOpcode() == TargetOpcode::G_SEXT ||
2492 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2493 "Expected a G_[ASZ]EXT");
2495 Register Reg = std::get<0>(MatchInfo);
2496 unsigned SrcExtOp = std::get<1>(MatchInfo);
2499 if (
MI.getOpcode() == SrcExtOp) {
2501 MI.getOperand(1).setReg(Reg);
2509 if (
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2510 (
MI.getOpcode() == TargetOpcode::G_SEXT &&
2511 SrcExtOp == TargetOpcode::G_ZEXT)) {
2514 MI.eraseFromParent();
2520 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2524 if (SrcOpc == TargetOpcode::G_ANYEXT || SrcOpc == TargetOpcode::G_SEXT ||
2525 SrcOpc == TargetOpcode::G_ZEXT) {
2534 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2536 unsigned SrcExtOp = MatchInfo.second;
2540 if (SrcTy == DstTy) {
2541 MI.eraseFromParent();
2549 MI.eraseFromParent();
2557 if (ShiftSize > 32 && TruncSize < 32)
2571 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2588 case TargetOpcode::G_SHL: {
2597 case TargetOpcode::G_LSHR:
2598 case TargetOpcode::G_ASHR: {
2605 if (
User.getOpcode() == TargetOpcode::G_STORE)
2609 if (NewShiftTy == SrcTy)
2623 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))
2626 MatchInfo = std::make_pair(SrcMI, NewShiftTy);
2633 LLT NewShiftTy = MatchInfo.second;
2647 if (NewShiftTy == DstTy)
2657 return MO.isReg() &&
2658 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2664 return !MO.isReg() ||
2665 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2670 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2672 return all_of(Mask, [](
int Elt) {
return Elt < 0; });
2676 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
2677 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(0).getReg(),
2682 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2683 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(1).getReg(),
2688 assert((
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2689 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2690 "Expected an insert/extract element op");
2693 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2706 OpIdx = Cst->isZero() ? 3 : 2;
2751 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2778 return MO.isReg() && MO.getReg().isPhysical();
2788 return I1->isIdenticalTo(*I2);
2803 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg,
nullptr) ==
2814 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&
2815 MaybeCst->getSExtValue() ==
C;
2821 std::optional<FPValueAndVReg> MaybeCst;
2825 return MaybeCst->Value.isExactlyValue(
C);
2830 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2832 Register Replacement =
MI.getOperand(OpIdx).getReg();
2834 MI.eraseFromParent();
2840 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2843 MI.eraseFromParent();
2848 unsigned ConstIdx) {
2849 Register ConstReg =
MI.getOperand(ConstIdx).getReg();
2862 assert((
MI.getOpcode() == TargetOpcode::G_FSHL ||
2863 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2864 "This is not a funnel shift operation");
2866 Register ConstReg =
MI.getOperand(3).getReg();
2871 assert((VRegAndVal) &&
"Value is not a constant");
2874 APInt NewConst = VRegAndVal->Value.
urem(
2879 MI.getOpcode(), {MI.getOperand(0)},
2880 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});
2882 MI.eraseFromParent();
2886 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2907 return MO.
isReg() &&
2918 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2920 MI.eraseFromParent();
2924 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2926 MI.eraseFromParent();
2930 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2932 MI.eraseFromParent();
2937 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2939 MI.eraseFromParent();
2943 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2945 MI.eraseFromParent();
2952 Register &NewLHS = std::get<0>(MatchInfo);
2953 Register &NewRHS = std::get<1>(MatchInfo);
2961 NewLHS = MaybeNewLHS;
2970 assert(
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2979 TargetOpcode::G_INSERT_VECTOR_ELT)
2985 MatchInfo.
resize(NumElts);
2989 if (IntImm >= NumElts || IntImm < 0)
2991 if (!MatchInfo[IntImm])
2992 MatchInfo[IntImm] = TmpReg;
2996 if (CurrInst->
getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2998 if (TmpInst->
getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
3007 return TmpInst->
getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
3014 auto GetUndef = [&]() {
3021 for (
unsigned I = 0;
I < MatchInfo.
size(); ++
I) {
3023 MatchInfo[
I] = GetUndef();
3026 MI.eraseFromParent();
3032 std::tie(SubLHS, SubRHS) = MatchInfo;
3034 MI.eraseFromParent();
3045 unsigned LogicOpcode =
MI.getOpcode();
3046 assert(LogicOpcode == TargetOpcode::G_AND ||
3047 LogicOpcode == TargetOpcode::G_OR ||
3048 LogicOpcode == TargetOpcode::G_XOR);
3061 if (!LeftHandInst || !RightHandInst)
3063 unsigned HandOpcode = LeftHandInst->
getOpcode();
3064 if (HandOpcode != RightHandInst->
getOpcode())
3076 if (!XTy.
isValid() || XTy != YTy)
3081 switch (HandOpcode) {
3084 case TargetOpcode::G_ANYEXT:
3085 case TargetOpcode::G_SEXT:
3086 case TargetOpcode::G_ZEXT: {
3090 case TargetOpcode::G_AND:
3091 case TargetOpcode::G_ASHR:
3092 case TargetOpcode::G_LSHR:
3093 case TargetOpcode::G_SHL: {
3098 ExtraHandOpSrcReg = ZOp.
getReg();
3120 if (ExtraHandOpSrcReg.
isValid())
3132 "Expected at least one instr to build?");
3134 assert(InstrToBuild.Opcode &&
"Expected a valid opcode?");
3135 assert(InstrToBuild.OperandFns.size() &&
"Expected at least one operand?");
3137 for (
auto &OperandFn : InstrToBuild.OperandFns)
3140 MI.eraseFromParent();
3145 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3146 int64_t ShlCst, AshrCst;
3152 if (ShlCst != AshrCst)
3155 {TargetOpcode::G_SEXT_INREG, {
MRI.
getType(Src)}}))
3157 MatchInfo = std::make_tuple(Src, ShlCst);
3163 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3166 std::tie(Src, ShiftAmt) = MatchInfo;
3169 MI.eraseFromParent();
3175 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3190 B.buildAnd(Dst, R,
B.buildConstant(Ty, C1 & C2));
3193 auto Zero =
B.buildConstant(Ty, 0);
3216 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3233 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3240 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3256 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3273 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3280 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3291 unsigned ExtBits =
MI.getOperand(2).getImm();
3297 int64_t Cst,
bool IsVector,
bool IsFP) {
3299 return (ScalarSizeBits == 1 && Cst == -1) ||
3305 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3325 for (
unsigned I = 0;
I < RegsToNegate.
size(); ++
I) {
3330 switch (Def->getOpcode()) {
3335 case TargetOpcode::G_ICMP:
3341 case TargetOpcode::G_FCMP:
3347 case TargetOpcode::G_AND:
3348 case TargetOpcode::G_OR:
3354 RegsToNegate.
push_back(Def->getOperand(1).getReg());
3355 RegsToNegate.
push_back(Def->getOperand(2).getReg());
3382 for (
Register Reg : RegsToNegate) {
3387 switch (Def->getOpcode()) {
3390 case TargetOpcode::G_ICMP:
3391 case TargetOpcode::G_FCMP: {
3398 case TargetOpcode::G_AND:
3401 case TargetOpcode::G_OR:
3409 MI.eraseFromParent();
3415 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3419 Register SharedReg =
MI.getOperand(2).getReg();
3440 return Y == SharedReg;
3447 std::tie(
X,
Y) = MatchInfo;
3451 MI.getOperand(1).setReg(Not->getOperand(0).getReg());
3452 MI.getOperand(2).setReg(
Y);
3457 auto &PtrAdd = cast<GPtrAdd>(
MI);
3458 Register DstReg = PtrAdd.getReg(0);
3467 return ConstVal && *ConstVal == 0;
3476 auto &PtrAdd = cast<GPtrAdd>(
MI);
3478 PtrAdd.eraseFromParent();
3485 Register Pow2Src1 =
MI.getOperand(2).getReg();
3492 MI.eraseFromParent();
3496 unsigned &SelectOpNo) {
3506 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3508 OtherOperandReg =
LHS;
3511 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3528 unsigned BinOpcode =
MI.getOpcode();
3533 bool CanFoldNonConst =
3534 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3539 if (CanFoldNonConst)
3550 const unsigned &SelectOperand) {
3561 unsigned BinOpcode =
MI.getOpcode();
3568 if (SelectOperand == 1) {
3582 MI.eraseFromParent();
3585std::optional<SmallVector<Register, 8>>
3586CombinerHelper::findCandidatesForLoadOrCombine(
const MachineInstr *Root)
const {
3587 assert(Root->
getOpcode() == TargetOpcode::G_OR &&
"Expected G_OR only!");
3616 const unsigned MaxIter =
3618 for (
unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3627 return std::nullopt;
3643 if (RegsToVisit.
empty() || RegsToVisit.
size() % 2 != 0)
3644 return std::nullopt;
3656static std::optional<std::pair<GZExtLoad *, int64_t>>
3660 "Expected Reg to only have one non-debug use?");
3669 if (Shift % MemSizeInBits != 0)
3670 return std::nullopt;
3673 auto *Load = getOpcodeDef<GZExtLoad>(MaybeLoad,
MRI);
3675 return std::nullopt;
3677 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3678 return std::nullopt;
3680 return std::make_pair(Load, Shift / MemSizeInBits);
3683std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3684CombinerHelper::findLoadOffsetsForLoadOrCombine(
3718 for (
auto Reg : RegsToVisit) {
3723 return std::nullopt;
3726 std::tie(Load, DstPos) = *LoadAndPos;
3734 return std::nullopt;
3737 auto &LoadMMO =
Load->getMMO();
3741 return std::nullopt;
3748 LoadPtr =
Load->getOperand(1).getReg();
3754 return std::nullopt;
3761 if (BasePtr != LoadPtr)
3762 return std::nullopt;
3764 if (
Idx < LowestIdx) {
3766 LowestIdxLoad =
Load;
3774 return std::nullopt;
3782 if (!EarliestLoad ||
dominates(*Load, *EarliestLoad))
3783 EarliestLoad =
Load;
3784 if (!LatestLoad ||
dominates(*LatestLoad, *Load))
3791 "Expected to find a load for each register?");
3792 assert(EarliestLoad != LatestLoad && EarliestLoad &&
3793 LatestLoad &&
"Expected at least two loads?");
3802 const unsigned MaxIter = 20;
3808 if (
MI.isLoadFoldBarrier())
3809 return std::nullopt;
3810 if (Iter++ == MaxIter)
3811 return std::nullopt;
3814 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);
3819 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3839 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
3843 auto RegsToVisit = findCandidatesForLoadOrCombine(&
MI);
3850 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
3851 if (NarrowMemSizeInBits % 8 != 0)
3864 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
3865 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);
3868 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;
3875 std::optional<bool> IsBigEndian =
isBigEndian(MemOffset2Idx, LowestIdx);
3878 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
3890 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
3891 const unsigned ZeroByteOffset =
3895 auto ZeroOffsetIdx = MemOffset2Idx.
find(ZeroByteOffset);
3896 if (ZeroOffsetIdx == MemOffset2Idx.
end() ||
3897 ZeroOffsetIdx->second != LowestIdx)
3921 MIB.setInstrAndDebugLoc(*LatestLoad);
3923 MIB.buildLoad(LoadDst,
Ptr, *NewMMO);
3925 MIB.buildBSwap(Dst, LoadDst);
3932 auto &
PHI = cast<GPhi>(
MI);
3945 case TargetOpcode::G_ANYEXT:
3947 case TargetOpcode::G_ZEXT:
3948 case TargetOpcode::G_SEXT:
3962 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3965 case TargetOpcode::G_LOAD:
3966 case TargetOpcode::G_TRUNC:
3967 case TargetOpcode::G_SEXT:
3968 case TargetOpcode::G_ZEXT:
3969 case TargetOpcode::G_ANYEXT:
3970 case TargetOpcode::G_CONSTANT:
3974 if (InSrcs.
size() > 2)
3986 auto &
PHI = cast<GPhi>(
MI);
3995 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3996 auto SrcReg =
PHI.getIncomingValue(
I);
3998 if (!SrcMIs.
insert(SrcMI))
4004 if (InsertPt !=
MBB->
end() && InsertPt->isPHI())
4010 OldToNewSrcMap[SrcMI] = NewExt;
4019 NewPhi.
addMBB(MO.getMBB());
4023 NewPhi.addUse(NewSrc->getOperand(0).getReg());
4031 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4041 unsigned VecIdx = Cst->Value.getZExtValue();
4046 if (SrcVecMI->
getOpcode() == TargetOpcode::G_TRUNC) {
4050 if (SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4051 SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4071 if (ScalarTy != DstTy) {
4074 MI.eraseFromParent();
4083 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4106 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4111 unsigned Idx = Cst->getZExtValue();
4115 SrcDstPairs.emplace_back(
4116 std::make_pair(
MI.getOperand(
Idx + 1).getReg(), &II));
4119 return ExtractedElts.
all();
4125 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4126 for (
auto &Pair : SrcDstPairs) {
4127 auto *ExtMI = Pair.second;
4129 ExtMI->eraseFromParent();
4131 MI.eraseFromParent();
4137 MI.eraseFromParent();
4155 assert(
MI.getOpcode() == TargetOpcode::G_OR);
4161 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4162 unsigned FshOpc = 0;
4173 int64_t CstShlAmt, CstLShrAmt;
4176 CstShlAmt + CstLShrAmt ==
BitWidth) {
4177 FshOpc = TargetOpcode::G_FSHR;
4184 FshOpc = TargetOpcode::G_FSHL;
4190 FshOpc = TargetOpcode::G_FSHR;
4201 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});
4208 unsigned Opc =
MI.getOpcode();
4209 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4214 unsigned RotateOpc =
4215 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
4220 unsigned Opc =
MI.getOpcode();
4221 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4222 bool IsFSHL = Opc == TargetOpcode::G_FSHL;
4225 : TargetOpcode::G_ROTR));
4226 MI.removeOperand(2);
4232 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4233 MI.getOpcode() == TargetOpcode::G_ROTR);
4237 bool OutOfRange =
false;
4238 auto MatchOutOfRange = [Bitsize, &OutOfRange](
const Constant *
C) {
4239 if (
auto *CI = dyn_cast<ConstantInt>(
C))
4240 OutOfRange |= CI->getValue().uge(Bitsize);
4247 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4248 MI.getOpcode() == TargetOpcode::G_ROTR);
4256 MI.getOperand(2).setReg(Amt);
4261 int64_t &MatchInfo) {
4262 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4266 std::optional<bool> KnownVal;
4315 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4340 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4346 unsigned Op = TargetOpcode::COPY;
4347 if (DstSize != LHSSize)
4348 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4358 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4368 int64_t AndMaskBits;
4376 if (AndMaskBits & OrMaskBits)
4382 if (
MI.getOperand(1).getReg() == AndMaskReg)
4383 MI.getOperand(2).setReg(AndMaskReg);
4384 MI.getOperand(1).setReg(Src);
4393 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4400 int64_t Width =
MI.getOperand(2).getImm();
4412 auto Cst1 =
B.buildConstant(ExtractTy, ShiftImm);
4413 auto Cst2 =
B.buildConstant(ExtractTy, Width);
4414 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);
4422 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4429 int64_t AndImm, LSBImm;
4438 auto MaybeMask =
static_cast<uint64_t>(AndImm);
4439 if (MaybeMask & (MaybeMask + 1))
4448 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4449 auto LSBCst =
B.buildConstant(ExtractTy, LSBImm);
4450 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});
4457 const unsigned Opcode =
MI.getOpcode();
4458 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4460 const Register Dst =
MI.getOperand(0).getReg();
4462 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4463 ? TargetOpcode::G_SBFX
4464 : TargetOpcode::G_UBFX;
4485 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >=
Size)
4489 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4493 const int64_t Pos = ShrAmt - ShlAmt;
4494 const int64_t Width =
Size - ShrAmt;
4497 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4498 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4499 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});
4506 const unsigned Opcode =
MI.getOpcode();
4507 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4509 const Register Dst =
MI.getOperand(0).getReg();
4526 if (ShrAmt < 0 || ShrAmt >=
Size)
4530 if (0 == (SMask >> ShrAmt)) {
4532 B.buildConstant(Dst, 0);
4539 UMask |= maskTrailingOnes<uint64_t>(ShrAmt);
4540 UMask &= maskTrailingOnes<uint64_t>(
Size);
4545 const int64_t Pos = ShrAmt;
4550 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt ==
Size)
4554 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4555 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4556 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});
4561bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4563 auto &PtrAdd = cast<GPtrAdd>(
MI);
4565 Register Src1Reg = PtrAdd.getBaseReg();
4566 auto *Src1Def = getOpcodeDef<GPtrAdd>(Src1Reg,
MRI);
4570 Register Src2Reg = PtrAdd.getOffsetReg();
4582 const APInt &C1APIntVal = *C1;
4583 const APInt &C2APIntVal = *C2;
4584 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4590 unsigned ConvUseOpc = ConvUseMI->
getOpcode();
4591 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4592 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4599 auto *LdStMI = dyn_cast<GLoadStore>(ConvUseMI);
4610 PtrAdd.getMF()->getFunction().getContext());
4611 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4612 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4618 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4630 Register Src1Reg =
MI.getOperand(1).getReg();
4631 if (
RHS->getOpcode() != TargetOpcode::G_ADD)
4643 MI.getOperand(1).setReg(NewBase.getReg(0));
4644 MI.getOperand(2).setReg(
RHS->getOperand(2).getReg());
4647 return !reassociationCanBreakAddressingModePattern(
MI);
4657 std::optional<ValueAndVReg> LHSCstOff;
4662 auto *LHSPtrAdd = cast<GPtrAdd>(
LHS);
4667 LHSPtrAdd->moveBefore(&
MI);
4670 auto NewCst =
B.buildConstant(
MRI.
getType(RHSReg), LHSCstOff->Value);
4672 MI.getOperand(2).setReg(NewCst.getReg(0));
4675 LHSPtrAdd->getOperand(2).setReg(RHSReg);
4678 return !reassociationCanBreakAddressingModePattern(
MI);
4686 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(
LHS);
4690 Register Src2Reg =
MI.getOperand(2).getReg();
4691 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4692 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
4701 auto NewCst =
B.buildConstant(
MRI.
getType(Src2Reg), *C1 + *C2);
4703 MI.getOperand(1).setReg(LHSSrc1);
4704 MI.getOperand(2).setReg(NewCst.getReg(0));
4707 return !reassociationCanBreakAddressingModePattern(
MI);
4712 auto &PtrAdd = cast<GPtrAdd>(
MI);
4764 auto NewCst =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});
4765 B.buildInstr(Opc, {DstReg}, {OpLHSLHS, NewCst});
4773 auto NewLHSLHS =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});
4774 B.buildInstr(Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});
4787 unsigned Opc =
MI.getOpcode();
4804 MatchInfo = *MaybeCst;
4817 MatchInfo = *MaybeCst;
4828 ConstantFP::get(
MI.getMF()->getFunction().getContext(), *MaybeCst);
4834 assert(
MI.getOpcode() == TargetOpcode::G_FMA ||
4835 MI.getOpcode() == TargetOpcode::G_FMAD);
4836 auto [
_, Op1, Op2, Op3] =
MI.getFirst4Regs();
4853 MatchInfo = ConstantFP::get(
MI.getMF()->getFunction().getContext(), Op1F);
4875 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4899 case TargetOpcode::G_ADD:
4900 case TargetOpcode::G_SUB:
4901 case TargetOpcode::G_MUL:
4902 case TargetOpcode::G_AND:
4903 case TargetOpcode::G_OR:
4904 case TargetOpcode::G_XOR:
4912 auto Mask = Cst->Value;
4917 unsigned NarrowWidth = Mask.countr_one();
4923 auto &MF = *
MI.getMF();
4926 auto &
DL = MF.getDataLayout();
4927 if (!TLI.isTruncateFree(WideTy, NarrowTy,
DL, Ctx) ||
4928 !TLI.isZExtFree(NarrowTy, WideTy,
DL, Ctx))
4942 MI.getOperand(1).setReg(Ext.getReg(0));
4949 unsigned Opc =
MI.getOpcode();
4950 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
4957 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
4958 : TargetOpcode::G_SADDO;
4960 MI.getOperand(3).setReg(
MI.getOperand(2).getReg());
4968 assert(
MI.getOpcode() == TargetOpcode::G_UMULO ||
4969 MI.getOpcode() == TargetOpcode::G_SMULO);
4978 B.buildConstant(Dst, 0);
4979 B.buildConstant(Carry, 0);
4987 assert(
MI.getOpcode() == TargetOpcode::G_UADDE ||
4988 MI.getOpcode() == TargetOpcode::G_SADDE ||
4989 MI.getOpcode() == TargetOpcode::G_USUBE ||
4990 MI.getOpcode() == TargetOpcode::G_SSUBE);
4995 switch (
MI.getOpcode()) {
4996 case TargetOpcode::G_UADDE:
4997 NewOpcode = TargetOpcode::G_UADDO;
4999 case TargetOpcode::G_SADDE:
5000 NewOpcode = TargetOpcode::G_SADDO;
5002 case TargetOpcode::G_USUBE:
5003 NewOpcode = TargetOpcode::G_USUBO;
5005 case TargetOpcode::G_SSUBE:
5006 NewOpcode = TargetOpcode::G_SSUBO;
5010 MI.setDesc(
B.getTII().get(NewOpcode));
5011 MI.removeOperand(4);
5019 assert(
MI.getOpcode() == TargetOpcode::G_SUB);
5053 B.buildSub(Dst, Zero, ReplaceReg);
5062 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
5063 auto &UDiv = cast<GenericMachineInstr>(
MI);
5073 unsigned KnownLeadingZeros =
5077 bool UseNPQ =
false;
5080 auto BuildUDIVPattern = [&](
const Constant *
C) {
5081 auto *CI = cast<ConstantInt>(
C);
5082 const APInt &Divisor = CI->getValue();
5084 bool SelNPQ =
false;
5086 unsigned PreShift = 0, PostShift = 0;
5091 if (!Divisor.
isOne()) {
5097 Divisor, std::min(KnownLeadingZeros, Divisor.
countl_zero()));
5099 Magic = std::move(magics.
Magic);
5102 "We shouldn't generate an undefined shift!");
5104 "We shouldn't generate an undefined shift!");
5108 SelNPQ = magics.
IsAdd;
5112 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));
5113 MagicFactors.
push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));
5115 MIB.buildConstant(ScalarTy,
5120 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));
5128 assert(Matched &&
"Expected unary predicate match to succeed");
5130 Register PreShift, PostShift, MagicFactor, NPQFactor;
5131 auto *RHSDef = getOpcodeDef<GBuildVector>(
RHS,
MRI);
5133 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);
5134 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5135 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);
5136 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);
5139 "Non-build_vector operation should have been a scalar");
5140 PreShift = PreShifts[0];
5141 MagicFactor = MagicFactors[0];
5142 PostShift = PostShifts[0];
5146 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);
5149 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);
5152 Register NPQ = MIB.buildSub(Ty,
LHS, Q).getReg(0);
5157 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);
5159 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);
5161 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);
5164 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);
5165 auto One = MIB.buildConstant(Ty, 1);
5166 auto IsOne = MIB.buildICmp(
5169 return MIB.buildSelect(Ty, IsOne,
LHS, Q);
5173 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
5181 auto &MF = *
MI.getMF();
5185 auto &
DL = MF.getDataLayout();
5191 if (MF.getFunction().hasMinSize())
5201 {TargetOpcode::G_ICMP,
5217 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5222 auto &MF = *
MI.getMF();
5226 auto &
DL = MF.getDataLayout();
5232 if (MF.getFunction().hasMinSize())
5251 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5252 auto &SDiv = cast<GenericMachineInstr>(
MI);
5262 bool UseSRA =
false;
5268 auto BuildSDIVPattern = [&](
const Constant *
C) {
5270 if (IsSplat && !Factors.
empty()) {
5276 auto *CI = cast<ConstantInt>(
C);
5277 APInt Divisor = CI->getValue();
5287 Shifts.
push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5288 Factors.
push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5295 assert(Matched &&
"Expected unary predicate match to succeed");
5299 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5300 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5303 Factor = Factors[0];
5311 return MIB.buildMul(Ty, Res, Factor);
5315 assert((
MI.getOpcode() == TargetOpcode::G_SDIV ||
5316 MI.getOpcode() == TargetOpcode::G_UDIV) &&
5317 "Expected SDIV or UDIV");
5318 auto &Div = cast<GenericMachineInstr>(
MI);
5320 auto MatchPow2 = [&](
const Constant *
C) {
5321 auto *CI = dyn_cast<ConstantInt>(
C);
5322 return CI && (CI->getValue().isPowerOf2() ||
5323 (IsSigned && CI->getValue().isNegatedPowerOf2()));
5329 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5330 auto &SDiv = cast<GenericMachineInstr>(
MI);
5384 MI.eraseFromParent();
5388 assert(
MI.getOpcode() == TargetOpcode::G_UDIV &&
"Expected UDIV");
5389 auto &UDiv = cast<GenericMachineInstr>(
MI);
5398 MI.eraseFromParent();
5402 assert(
MI.getOpcode() == TargetOpcode::G_UMULH);
5407 auto MatchPow2ExceptOne = [&](
const Constant *
C) {
5408 if (
auto *CI = dyn_cast<ConstantInt>(
C))
5409 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
5430 MI.eraseFromParent();
5435 unsigned Opc =
MI.getOpcode();
5436 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||
5437 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5438 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);
5450 Opc = TargetOpcode::G_FSUB;
5455 Opc = TargetOpcode::G_FADD;
5461 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5462 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&
5471 MI.setDesc(
B.getTII().get(Opc));
5472 MI.getOperand(1).setReg(
X);
5473 MI.getOperand(2).setReg(
Y);
5480 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5483 MatchInfo =
MI.getOperand(2).getReg();
5493 if (LHSCst->Value.isNegZero())
5497 if (LHSCst->Value.isPosZero())
5513 if (
MI.getOpcode() != TargetOpcode::G_FMUL)
5521 MRI.use_instr_nodbg_end()) >
5523 MRI.use_instr_nodbg_end());
5527 bool &AllowFusionGlobally,
5529 bool CanReassociate) {
5531 auto *MF =
MI.getMF();
5532 const auto &TLI = *MF->getSubtarget().getTargetLowering();
5536 if (CanReassociate &&
5543 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&
5546 if (!HasFMAD && !HasFMA)
5550 Options.UnsafeFPMath || HasFMAD;
5555 Aggressive = TLI.enableAggressiveFMAFusion(DstType);
5561 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5563 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5571 unsigned PreferredFusedOpcode =
5572 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5586 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5587 {
LHS.MI->getOperand(1).getReg(),
5588 LHS.MI->getOperand(2).getReg(),
RHS.Reg});
5597 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5598 {
RHS.MI->getOperand(1).getReg(),
5599 RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5609 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5611 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5615 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5622 unsigned PreferredFusedOpcode =
5623 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5637 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5642 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5643 {FpExtX.getReg(0), FpExtY.getReg(0),
RHS.Reg});
5652 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5657 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5658 {FpExtX.getReg(0), FpExtY.getReg(0),
LHS.Reg});
5668 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5670 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5680 unsigned PreferredFusedOpcode =
5681 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5694 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5696 TargetOpcode::G_FMUL) &&
5703 else if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5705 TargetOpcode::G_FMUL) &&
5714 Register X = FMA->getOperand(1).getReg();
5715 Register Y = FMA->getOperand(2).getReg();
5721 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z});
5722 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5733 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5735 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5742 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5749 unsigned PreferredFusedOpcode =
5750 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5763 Register FpExtU =
B.buildFPExt(DstType, U).getReg(0);
5764 Register FpExtV =
B.buildFPExt(DstType, V).getReg(0);
5766 B.buildInstr(PreferredFusedOpcode, {DstType}, {FpExtU, FpExtV, Z})
5768 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5775 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5779 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5784 LHS.MI->getOperand(1).getReg(),
5785 LHS.MI->getOperand(2).getReg(),
B);
5796 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5799 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5804 X =
B.buildFPExt(DstType,
X).getReg(0);
5805 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5816 if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5820 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5825 RHS.MI->getOperand(1).getReg(),
5826 RHS.MI->getOperand(2).getReg(),
B);
5837 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5840 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5845 X =
B.buildFPExt(DstType,
X).getReg(0);
5846 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5859 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5861 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5873 int FirstMulHasFewerUses =
true;
5877 FirstMulHasFewerUses =
false;
5879 unsigned PreferredFusedOpcode =
5880 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5883 if (FirstMulHasFewerUses &&
5888 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5889 {
LHS.MI->getOperand(1).getReg(),
5890 LHS.MI->getOperand(2).getReg(), NegZ});
5899 B.buildFNeg(DstTy,
RHS.MI->getOperand(1).getReg()).
getReg(0);
5900 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5901 {NegY,
RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5911 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5913 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5921 unsigned PreferredFusedOpcode =
5922 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5933 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5934 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5946 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5958 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5960 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5968 unsigned PreferredFusedOpcode =
5969 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5981 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5982 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5983 {FpExtX, FpExtY, NegZ});
5995 Register NegY =
B.buildFNeg(DstTy, FpExtY).getReg(0);
5998 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5999 {NegY, FpExtZ, LHSReg});
6009 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
6011 bool AllowFusionGlobally, HasFMAD,
Aggressive;
6015 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
6020 unsigned PreferredFusedOpcode =
6021 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
6025 Register FpExtX =
B.buildFPExt(DstTy,
X).getReg(0);
6026 Register FpExtY =
B.buildFPExt(DstTy,
Y).getReg(0);
6027 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z});
6038 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
6044 B.buildFNeg(
MI.getOperand(0).getReg(), FMAReg);
6054 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
6067 unsigned &IdxToPropagate) {
6069 switch (
MI.getOpcode()) {
6072 case TargetOpcode::G_FMINNUM:
6073 case TargetOpcode::G_FMAXNUM:
6074 PropagateNaN =
false;
6076 case TargetOpcode::G_FMINIMUM:
6077 case TargetOpcode::G_FMAXIMUM:
6078 PropagateNaN =
true;
6082 auto MatchNaN = [&](
unsigned Idx) {
6087 IdxToPropagate = PropagateNaN ?
Idx : (
Idx == 1 ? 2 : 1);
6091 return MatchNaN(1) || MatchNaN(2);
6095 assert(
MI.getOpcode() == TargetOpcode::G_ADD &&
"Expected a G_ADD");
6105 Reg == MaybeSameReg;
6140 std::optional<ValueAndVReg> ShiftAmount;
6171 std::optional<ValueAndVReg> ShiftAmt;
6178 return ShiftAmt->Value.getZExtValue() == MatchTy.
getSizeInBits() &&
6182unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6184 SelectPatternNaNBehaviour VsNaNRetVal)
const {
6185 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6186 "Expected a NaN behaviour?");
6196 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6197 return TargetOpcode::G_FMAXNUM;
6198 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6199 return TargetOpcode::G_FMAXIMUM;
6200 if (
isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))
6201 return TargetOpcode::G_FMAXNUM;
6202 if (
isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))
6203 return TargetOpcode::G_FMAXIMUM;
6209 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6210 return TargetOpcode::G_FMINNUM;
6211 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6212 return TargetOpcode::G_FMINIMUM;
6213 if (
isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))
6214 return TargetOpcode::G_FMINNUM;
6215 if (!
isLegal({TargetOpcode::G_FMINIMUM, {DstTy}}))
6217 return TargetOpcode::G_FMINIMUM;
6221CombinerHelper::SelectPatternNaNBehaviour
6223 bool IsOrderedComparison)
const {
6227 if (!LHSSafe && !RHSSafe)
6228 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
6229 if (LHSSafe && RHSSafe)
6230 return SelectPatternNaNBehaviour::RETURNS_ANY;
6233 if (IsOrderedComparison)
6234 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
6235 : SelectPatternNaNBehaviour::RETURNS_OTHER;
6238 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
6239 : SelectPatternNaNBehaviour::RETURNS_NAN;
6261 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
6263 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
6265 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
6268 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
6269 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
6270 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
6271 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
6273 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
6276 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, ResWithKnownNaNInfo);
6277 if (!Opc || !
isLegal({Opc, {DstTy}}))
6281 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
6286 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
6288 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
6293 B.buildInstr(Opc, {Dst}, {CmpLHS, CmpRHS});
6301 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
6308 Register TrueVal =
MI.getOperand(2).getReg();
6309 Register FalseVal =
MI.getOperand(3).getReg();
6310 return matchFPSelectToMinMax(Dst,
Cond, TrueVal, FalseVal, MatchInfo);
6315 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
6328 if (MatchedSub &&
X != OpLHS)
6336 Y =
X == OpLHS ? OpRHS :
X == OpRHS ? OpLHS :
Register();
6340 B.buildICmp(Pred, Dst,
Y, Zero);
6346 Register ShiftReg =
MI.getOperand(2).getReg();
6348 auto IsShiftTooBig = [&](
const Constant *
C) {
6349 auto *CI = dyn_cast<ConstantInt>(
C);
6356 unsigned LHSOpndIdx = 1;
6357 unsigned RHSOpndIdx = 2;
6358 switch (
MI.getOpcode()) {
6359 case TargetOpcode::G_UADDO:
6360 case TargetOpcode::G_SADDO:
6361 case TargetOpcode::G_UMULO:
6362 case TargetOpcode::G_SMULO:
6376 TargetOpcode::G_CONSTANT_FOLD_BARRIER)
6381 TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
6388 std::optional<FPValueAndVReg> ValAndVReg;
6396 unsigned LHSOpndIdx = 1;
6397 unsigned RHSOpndIdx = 2;
6398 switch (
MI.getOpcode()) {
6399 case TargetOpcode::G_UADDO:
6400 case TargetOpcode::G_SADDO:
6401 case TargetOpcode::G_UMULO:
6402 case TargetOpcode::G_SMULO:
6409 Register LHSReg =
MI.getOperand(LHSOpndIdx).getReg();
6410 Register RHSReg =
MI.getOperand(RHSOpndIdx).getReg();
6411 MI.getOperand(LHSOpndIdx).setReg(RHSReg);
6412 MI.getOperand(RHSOpndIdx).setReg(LHSReg);
6416bool CombinerHelper::isOneOrOneSplat(
Register Src,
bool AllowUndefs) {
6419 return isConstantSplatVector(Src, 1, AllowUndefs);
6421 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6424 return IConstant && IConstant->Value == 1;
6429bool CombinerHelper::isZeroOrZeroSplat(
Register Src,
bool AllowUndefs) {
6432 return isConstantSplatVector(Src, 0, AllowUndefs);
6434 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6437 return IConstant && IConstant->Value == 0;
6444bool CombinerHelper::isConstantSplatVector(
Register Src, int64_t SplatValue,
6451 for (
unsigned I = 0;
I < NumSources; ++
I) {
6454 if (ImplicitDef && AllowUndefs)
6456 if (ImplicitDef && !AllowUndefs)
6458 std::optional<ValueAndVReg> IConstant =
6460 if (IConstant && IConstant->Value == SplatValue)
6470CombinerHelper::getConstantOrConstantSplatVector(
Register Src) {
6473 return IConstant->Value;
6477 return std::nullopt;
6480 std::optional<APInt>
Value = std::nullopt;
6481 for (
unsigned I = 0;
I < NumSources; ++
I) {
6482 std::optional<ValueAndVReg> IConstant =
6485 return std::nullopt;
6489 return std::nullopt;
6495bool CombinerHelper::isConstantOrConstantVectorI(
Register Src)
const {
6505 for (
unsigned I = 0;
I < NumSources; ++
I) {
6506 std::optional<ValueAndVReg> IConstant =
6515bool CombinerHelper::tryFoldSelectOfConstants(
GSelect *
Select,
6533 std::optional<ValueAndVReg> TrueOpt =
6535 std::optional<ValueAndVReg> FalseOpt =
6538 if (!TrueOpt || !FalseOpt)
6541 APInt TrueValue = TrueOpt->Value;
6542 APInt FalseValue = FalseOpt->Value;
6547 B.setInstrAndDebugLoc(*
Select);
6548 B.buildZExtOrTrunc(Dest,
Cond);
6556 B.setInstrAndDebugLoc(*
Select);
6557 B.buildSExtOrTrunc(Dest,
Cond);
6565 B.setInstrAndDebugLoc(*
Select);
6567 B.buildNot(Inner,
Cond);
6568 B.buildZExtOrTrunc(Dest, Inner);
6576 B.setInstrAndDebugLoc(*
Select);
6578 B.buildNot(Inner,
Cond);
6579 B.buildSExtOrTrunc(Dest, Inner);
6585 if (TrueValue - 1 == FalseValue) {
6587 B.setInstrAndDebugLoc(*
Select);
6589 B.buildZExtOrTrunc(Inner,
Cond);
6590 B.buildAdd(Dest, Inner, False);
6596 if (TrueValue + 1 == FalseValue) {
6598 B.setInstrAndDebugLoc(*
Select);
6600 B.buildSExtOrTrunc(Inner,
Cond);
6601 B.buildAdd(Dest, Inner, False);
6609 B.setInstrAndDebugLoc(*
Select);
6611 B.buildZExtOrTrunc(Inner,
Cond);
6614 auto ShAmtC =
B.buildConstant(ShiftTy, TrueValue.
exactLogBase2());
6615 B.buildShl(Dest, Inner, ShAmtC, Flags);
6622 B.setInstrAndDebugLoc(*
Select);
6624 B.buildSExtOrTrunc(Inner,
Cond);
6625 B.buildOr(Dest, Inner, False, Flags);
6633 B.setInstrAndDebugLoc(*
Select);
6635 B.buildNot(Not,
Cond);
6637 B.buildSExtOrTrunc(Inner, Not);
6638 B.buildOr(Dest, Inner, True, Flags);
6647bool CombinerHelper::tryFoldBoolSelectToLogic(
GSelect *
Select,
6664 if (CondTy != TrueTy)
6669 if ((
Cond == True) || isOneOrOneSplat(True,
true)) {
6671 B.setInstrAndDebugLoc(*
Select);
6673 B.buildZExtOrTrunc(Ext,
Cond);
6674 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6675 B.buildOr(DstReg, Ext, FreezeFalse, Flags);
6682 if ((
Cond == False) || isZeroOrZeroSplat(False,
true)) {
6684 B.setInstrAndDebugLoc(*
Select);
6686 B.buildZExtOrTrunc(Ext,
Cond);
6687 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6688 B.buildAnd(DstReg, Ext, FreezeTrue);
6694 if (isOneOrOneSplat(False,
true)) {
6696 B.setInstrAndDebugLoc(*
Select);
6699 B.buildNot(Inner,
Cond);
6702 B.buildZExtOrTrunc(Ext, Inner);
6703 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6704 B.buildOr(DstReg, Ext, FreezeTrue, Flags);
6710 if (isZeroOrZeroSplat(True,
true)) {
6712 B.setInstrAndDebugLoc(*
Select);
6715 B.buildNot(Inner,
Cond);
6718 B.buildZExtOrTrunc(Ext, Inner);
6719 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6720 B.buildAnd(DstReg, Ext, FreezeFalse);
6728bool CombinerHelper::tryFoldSelectToIntMinMax(
GSelect *
Select,
6758 if (True == CmpRHS && False == CmpLHS) {
6766 if (True == CmpLHS && False == CmpRHS) {
6773 B.buildUMax(DstReg, True, False);
6782 B.buildSMax(DstReg, True, False);
6791 B.buildUMin(DstReg, True, False);
6800 B.buildSMin(DstReg, True, False);
6815 if (tryFoldSelectOfConstants(
Select, MatchInfo))
6818 if (tryFoldBoolSelectToLogic(
Select, MatchInfo))
6821 if (tryFoldSelectToIntMinMax(
Select, MatchInfo))
6831bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
GLogicalBinOp *Logic,
6833 assert(Logic->
getOpcode() != TargetOpcode::G_XOR &&
"unexpected xor");
6834 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6838 unsigned Flags = Logic->
getFlags();
6841 GICmp *Cmp1 = getOpcodeDef<GICmp>(LHS,
MRI);
6846 GICmp *Cmp2 = getOpcodeDef<GICmp>(RHS,
MRI);
6857 std::optional<ValueAndVReg> MaybeC1 =
6861 C1 = MaybeC1->Value;
6863 std::optional<ValueAndVReg> MaybeC2 =
6867 C2 = MaybeC2->Value;
6888 std::optional<APInt> Offset1;
6889 std::optional<APInt> Offset2;
6891 if (
GAdd *
Add = getOpcodeDef<GAdd>(R1,
MRI)) {
6892 std::optional<ValueAndVReg> MaybeOffset1 =
6895 R1 =
Add->getLHSReg();
6896 Offset1 = MaybeOffset1->Value;
6900 std::optional<ValueAndVReg> MaybeOffset2 =
6903 R2 =
Add->getLHSReg();
6904 Offset2 = MaybeOffset2->Value;
6923 bool CreateMask =
false;
6936 if (!LowerDiff.
isPowerOf2() || LowerDiff != UpperDiff ||
6949 CR->getEquivalentICmp(NewPred, NewC,
Offset);
6959 if (CreateMask &&
Offset != 0) {
6960 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6961 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6962 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6963 auto Add =
B.buildAdd(CmpOperandTy,
And, OffsetC, Flags);
6964 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6965 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6966 B.buildZExtOrTrunc(DstReg, ICmp);
6967 }
else if (CreateMask &&
Offset == 0) {
6968 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6969 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6970 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6971 auto ICmp =
B.buildICmp(NewPred, CmpTy,
And, NewCon);
6972 B.buildZExtOrTrunc(DstReg, ICmp);
6973 }
else if (!CreateMask &&
Offset != 0) {
6974 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6975 auto Add =
B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);
6976 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6977 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6978 B.buildZExtOrTrunc(DstReg, ICmp);
6979 }
else if (!CreateMask &&
Offset == 0) {
6980 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6981 auto ICmp =
B.buildICmp(NewPred, CmpTy, R1, NewCon);
6982 B.buildZExtOrTrunc(DstReg, ICmp);
6990bool CombinerHelper::tryFoldLogicOfFCmps(
GLogicalBinOp *Logic,
6996 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6999 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(LHS,
MRI);
7004 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(RHS,
MRI);
7014 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
7028 if (LHS0 == RHS1 && LHS1 == RHS0) {
7034 if (LHS0 == RHS0 && LHS1 == RHS1) {
7038 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
7045 auto False =
B.buildConstant(CmpTy, 0);
7046 B.buildZExtOrTrunc(DestReg, False);
7053 B.buildZExtOrTrunc(DestReg, True);
7055 auto Cmp =
B.buildFCmp(Pred, CmpTy, LHS0, LHS1, Flags);
7056 B.buildZExtOrTrunc(DestReg, Cmp);
7068 if (tryFoldAndOrOrICmpsUsingRanges(
And, MatchInfo))
7071 if (tryFoldLogicOfFCmps(
And, MatchInfo))
7080 if (tryFoldAndOrOrICmpsUsingRanges(
Or, MatchInfo))
7083 if (tryFoldLogicOfFCmps(
Or, MatchInfo))
7097 bool IsSigned =
Add->isSigned();
7106 B.buildUndef(Carry);
7112 if (isConstantOrConstantVectorI(
LHS) && !isConstantOrConstantVectorI(
RHS)) {
7115 B.buildSAddo(Dst, Carry,
RHS,
LHS);
7121 B.buildUAddo(Dst, Carry,
RHS,
LHS);
7126 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(
LHS);
7127 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(
RHS);
7133 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)
7134 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);
7136 B.buildConstant(Dst, Result);
7137 B.buildConstant(Carry, Overflow);
7145 B.buildCopy(Dst,
LHS);
7146 B.buildConstant(Carry, 0);
7158 std::optional<APInt> MaybeAddRHS =
7159 getConstantOrConstantSplatVector(AddLHS->
getRHSReg());
7162 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)
7163 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);
7167 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7168 B.buildSAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7174 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7175 B.buildUAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7200 B.buildConstant(Carry, 0);
7208 B.buildConstant(Carry, 1);
7223 B.buildConstant(Carry, 0);
7239 B.buildConstant(Carry, 0);
7247 B.buildConstant(Carry, 1);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo &MRI)
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally)
Checks if MI is TargetOpcode::G_FMUL and contractable either due to global flags or MachineInstr flag...
static unsigned getIndexedOpc(unsigned LdStOpc)
static APFloat constantFoldFpUnary(const MachineInstr &MI, const MachineRegisterInfo &MRI, const APFloat &Val)
static std::optional< std::pair< GZExtLoad *, int64_t > > matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits, const MachineRegisterInfo &MRI)
Helper function for findLoadOffsetsForLoadOrCombine.
static Register peekThroughBitcast(Register Reg, const MachineRegisterInfo &MRI)
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static cl::opt< bool > ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false), cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner"))
static cl::opt< unsigned > PostIndexUseThreshold("post-index-use-threshold", cl::Hidden, cl::init(32), cl::desc("Number of uses of a base pointer to check before it is no longer " "considered for post-indexing."))
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static unsigned getExtLoadOpcForExtend(unsigned ExtOpc)
static Type * getTypeForLLT(LLT Ty, LLVMContext &C)
static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits, int64_t Cst, bool IsVector, bool IsFP)
static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy)
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Register buildLogBase2(Register V, MachineIRBuilder &MIB)
Determines the LogBase2 value for a non-null input value using the transform: LogBase2(V) = (EltBits ...
This contains common combine transformations that may be used in a combine pass,or by the target else...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Implement a low-level type suitable for MachineInstr level instruction selection.
Contains matchers for matching SSA Machine Instructions.
mir Rename Register Operands
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file describes how to lower LLVM code to machine code.
const fltSemantics & getSemantics() const
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
uint64_t getZExtValue() const
Get zero extended value.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
int32_t exactLogBase2() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
unsigned countl_zero() const
The APInt version of std::countl_zero.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
unsigned countl_one() const
Count the number of leading one bits.
APInt multiplicativeInverse() const
bool isMask(unsigned numBits) const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isOne() const
Determine if this is a value of 1.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
AttributeSet getAttributes(unsigned Index) const
The attributes for the specified index are returned.
bool isEquality() const
Determine if this is an equals/not equals predicate.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
static bool isOrdered(Predicate predicate)
Determine if the predicate is an ordered operation.
void applyUDivByConst(MachineInstr &MI)
void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
bool matchCombineShuffleVector(MachineInstr &MI, SmallVectorImpl< Register > &Ops)
Check if the G_SHUFFLE_VECTOR MI can be replaced by a concat_vectors.
bool matchPtrAddZero(MachineInstr &MI)
}
bool matchAllExplicitUsesAreUndef(MachineInstr &MI)
Return true if all register explicit use operands on MI are defined by a G_IMPLICIT_DEF.
void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx)
Delete MI and replace all of its uses with its OpIdx-th operand.
const RegisterBank * getRegBank(Register Reg) const
Get the register bank of Reg.
bool matchReassocPtrAdd(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate pointer calculations with G_ADD involved, to allow better addressing mode usage.
bool matchUDivByConst(MachineInstr &MI)
Combine G_UDIV by constant into a multiply by magic constant.
void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI)
Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
bool matchShiftsTooBig(MachineInstr &MI)
Match shifts greater or equal to the bitwidth of the operation.
bool tryCombineCopy(MachineInstr &MI)
If MI is COPY, try to combine it.
bool matchTruncLshrBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
bool matchUndefStore(MachineInstr &MI)
Return true if a G_STORE instruction MI is storing an undef value.
bool matchRedundantBinOpInEquality(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (X + Y) == X -> Y == 0 (X - Y) == X -> Y == 0 (X ^ Y) == X -> Y == 0 (X + Y) !...
bool matchRedundantSExtInReg(MachineInstr &MI)
bool matchCombineFAddFpExtFMulToFMadOrFMAAggressive(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchReassocConstantInnerRHS(GPtrAdd &MI, MachineInstr *RHS, BuildFnTy &MatchInfo)
bool matchSubAddSameReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (x + y) - y -> x (x + y) - x -> y x - (y + x) -> 0 - y x - (x + z) -> 0 - z.
bool matchConstantFoldFPBinOp(MachineInstr &MI, ConstantFP *&MatchInfo)
Do constant FP folding when opportunities are exposed after MIR building.
void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal)
void applyCombineUnmergeZExtToZExt(MachineInstr &MI)
void applyCommuteBinOpOperands(MachineInstr &MI)
bool matchBinOpSameVal(MachineInstr &MI)
Optimize (x op x) -> x.
void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
bool matchCombineFSubFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) (fsub (fneg (fmul,...
bool matchCombineCopy(MachineInstr &MI)
bool matchConstantSelectCmp(MachineInstr &MI, unsigned &OpIdx)
Return true if a G_SELECT instruction MI has a constant comparison.
void eraseInst(MachineInstr &MI)
Erase MI.
void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const
MachineRegisterInfo::replaceRegWith() and inform the observer of the changes.
void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, Register ToReg) const
Replace a single register operand with a new register and inform the observer of the changes.
bool matchCombineFAddFMAFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) (fadd (fmad x,...
void applySimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
bool matchSimplifySelectToMinMax(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
If MI is G_CONCAT_VECTORS, try to combine it.
bool matchAddSubSameReg(MachineInstr &MI, Register &Src)
Transform G_ADD(x, G_SUB(y, x)) to y.
void applyRotateOutOfRange(MachineInstr &MI)
bool matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_UMULO x, 2) -> (G_UADDO x, x) (G_SMULO x, 2) -> (G_SADDO x, x)
bool matchRotateOutOfRange(MachineInstr &MI)
void applyCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void applyCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
void applyCombineShuffleVector(MachineInstr &MI, const ArrayRef< Register > Ops)
Replace MI with a concat_vectors with Ops.
const TargetLowering & getTargetLowering() const
void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
void applyPtrAddZero(MachineInstr &MI)
bool matchTruncBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
void setRegBank(Register Reg, const RegisterBank *RegBank)
Set the register bank of Reg.
bool matchRedundantAnd(MachineInstr &MI, Register &Replacement)
void replaceInstWithConstant(MachineInstr &MI, int64_t C)
Replace an instruction with a G_CONSTANT with value C.
bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
Match ashr (shl x, C), C -> sext_inreg (C)
bool tryCombineExtendingLoads(MachineInstr &MI)
If MI is extend that consumes the result of a load, try to combine it.
bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount)
bool matchCombineUnmergeUndef(MachineInstr &MI, std::function< void(MachineIRBuilder &)> &MatchInfo)
Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ...
void applySDivByConst(MachineInstr &MI)
bool matchUndefSelectCmp(MachineInstr &MI)
Return true if a G_SELECT instruction MI has an undef comparison.
void replaceInstWithUndef(MachineInstr &MI)
Replace an instruction with a G_IMPLICIT_DEF.
bool matchRedundantOr(MachineInstr &MI, Register &Replacement)
bool matchOperandIsUndef(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is undef.
void applyBuildFn(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
bool matchCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void replaceInstWithFConstant(MachineInstr &MI, double C)
Replace an instruction with a G_FCONSTANT with value C.
bool matchBitfieldExtractFromSExtInReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2)
Return true if MOP1 and MOP2 are register operands are defined by equivalent instructions.
bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
Fold (shift (shift base, x), y) -> (shift base (x+y))
bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*MULO x, 0) -> 0 + no carry out.
void replaceSingleDefInstWithReg(MachineInstr &MI, Register Replacement)
Delete MI and replace all of its uses with Replacement.
bool matchFunnelShiftToRotate(MachineInstr &MI)
Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
bool matchNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
Combine inverting a result of a compare into the opposite cond code.
void applyCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
void replaceOpcodeWith(MachineInstr &FromMI, unsigned ToOpcode) const
Replace the opcode in instruction with a new opcode and inform the observer of the changes.
bool matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is known to be a power of 2.
void applyCombineCopy(MachineInstr &MI)
void applyCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
bool matchAnyExplicitUseIsUndef(MachineInstr &MI)
Return true if any explicit use operand on MI is defined by a G_IMPLICIT_DEF.
bool matchFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
void applyCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
bool matchSextTruncSextLoad(MachineInstr &MI)
bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
bool matchExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
MachineInstr * buildSDivUsingMul(MachineInstr &MI)
Given an G_SDIV MI expressing a signed divide by constant, return an expression that implements it by...
void applySDivByPow2(MachineInstr &MI)
void applyFunnelShiftConstantModulo(MachineInstr &MI)
Replaces the shift amount in MI with ShiftAmt % BW.
bool matchConstantFoldBinOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool isPreLegalize() const
bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo)
Match (and (load x), mask) -> zextload x.
bool matchConstantOp(const MachineOperand &MOP, int64_t C)
Return true if MOP is defined by a G_CONSTANT or splat with a value equal to C.
bool matchCombineFSubFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fmul x, y), z) -> (fma x, y, -z) (fsub (fmul x, y), z) -> (fmad x,...
bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ands.
void applyCombineI2PToP2I(MachineInstr &MI, Register &Reg)
void applyNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool matchConstantFPOp(const MachineOperand &MOP, double C)
Return true if MOP is defined by a G_FCONSTANT or splat with a value exactly equal to C.
bool matchSimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
Return true if MI is a G_ADD which can be simplified to a G_SUB.
bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Optimize memcpy intrinsics et al, e.g.
bool matchSelectSameVal(MachineInstr &MI)
Optimize (cond ? x : x) -> x.
void applyCombineConstantFoldFpUnary(MachineInstr &MI, const ConstantFP *Cst)
Transform fp_instr(cst) to constant result of the fp operation.
bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool tryReassocBinOp(unsigned Opc, Register DstReg, Register Op0, Register Op1, BuildFnTy &MatchInfo)
Try to reassociate to reassociate operands of a commutative binop.
bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const
bool tryEmitMemcpyInline(MachineInstr &MI)
Emit loads and stores that perform the given memcpy.
void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
Fold (xor (and x, y), y) -> (and (not x), y) {.
bool matchCombineFSubFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) (fsub (fpext (fmul x,...
bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info)
bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData)
bool matchConstantFoldFMA(MachineInstr &MI, ConstantFP *&MatchInfo)
Constant fold G_FMA/G_FMAD.
bool matchBitfieldExtractFromAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: and (lshr x, cst), mask -> ubfx x, cst, width.
void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
bool isLegal(const LegalityQuery &Query) const
bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine selects.
bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
Transform G_UNMERGE Constant -> Constant1, Constant2, ...
bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo)
bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg)
Transform anyext(trunc(x)) to x.
void applySimplifyURemByPow2(MachineInstr &MI)
Combine G_UREM x, (known power of 2) to an add and bitmasking.
bool matchReassocFoldConstantsInSubTree(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
void applyCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops)
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
MachineRegisterInfo & MRI
void applyUMulHToLShr(MachineInstr &MI)
bool matchLoadOrCombine(MachineInstr &MI, BuildFnTy &MatchInfo)
Match expression trees of the form.
bool matchShuffleToExtract(MachineInstr &MI)
bool matchUndefShuffleVectorMask(MachineInstr &MI)
Return true if a G_SHUFFLE_VECTOR instruction MI has an undef mask.
bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const
bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
bool matchAndOrDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineExtractedVectorLoad(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine a G_EXTRACT_VECTOR_ELT of a load into a narrowed load.
bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
Transform a multiply by a power-of-2 value to a left shift.
bool matchBitfieldExtractFromShr(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (shl x, n), k -> sbfx/ubfx x, pos, width.
void applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo)
SelectOperand is the operand in binary operator MI that is the select to fold.
bool matchBuildVectorIdentityFold(MachineInstr &MI, Register &MatchInfo)
bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fmul x, y), z) -> (fma x, y, z) (fadd (fmul x, y), z) -> (fmad x,...
bool matchRedundantNegOperands(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd x, fneg(y)) -> (fsub x, y) (fadd fneg(x), y) -> (fsub y, x) (fsub x,...
bool matchCombineMergeUnmerge(MachineInstr &MI, Register &MatchInfo)
Fold away a merge of an unmerge of the corresponding values.
void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineUnmergeZExtToZExt(MachineInstr &MI)
Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0.
bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z.
bool matchConstantLargerBitWidth(MachineInstr &MI, unsigned ConstIdx)
Checks if constant at ConstIdx is larger than MI 's bitwidth.
CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, GISelKnownBits *KB=nullptr, MachineDominatorTree *MDT=nullptr, const LegalizerInfo *LI=nullptr)
bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM when their source operands are iden...
bool matchCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x).
bool isPredecessor(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI precedes UseMI or they are the same instruction.
bool matchDivByPow2(MachineInstr &MI, bool IsSigned)
Given an G_SDIV MI expressing a signed divided by a pow2 constant, return expressions that implements...
bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchUMulHToLShr(MachineInstr &MI)
bool dominates(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI dominates UseMI.
MachineInstr * buildUDivUsingMul(MachineInstr &MI)
Given an G_UDIV MI expressing a divide by constant, return an expression that implements it by multip...
bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg)
Transform zext(trunc(x)) to x.
void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData)
bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally, bool &HasFMAD, bool &Aggressive, bool CanReassociate=false)
void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
void applyShuffleToExtract(MachineInstr &MI)
MachineDominatorTree * MDT
bool matchSDivByConst(MachineInstr &MI)
void applySextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z.
void applyExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
void applyBuildFnMO(const MachineOperand &MO, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
bool matchCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
Transform trunc (shl x, K) to shl (trunc x), K if K < VT.getScalarSizeInBits().
const RegisterBankInfo * RBI
bool matchCommuteShift(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) (fadd (fpext (fmul x,...
bool matchReassocConstantInnerLHS(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
const TargetRegisterInfo * TRI
bool tryCombineShuffleVector(MachineInstr &MI)
Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
bool matchCombineI2PToP2I(MachineInstr &MI, Register &Reg)
Transform IntToPtr(PtrToInt(x)) to x if cast is in the same address space.
bool matchICmpToLHSKnownBits(MachineInstr &MI, BuildFnTy &MatchInfo)
GISelChangeObserver & Observer
bool matchCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Transform [asz]ext([asz]ext(x)) to [asz]ext x.
bool matchOverlappingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0.
bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Match sext_inreg(load p), imm -> sextload p.
bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
Transform G_ADD (G_PTRTOINT x), y -> G_PTRTOINT (G_PTR_ADD x, y) Transform G_ADD y,...
bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ors.
void applyFunnelShiftToRotate(MachineInstr &MI)
void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
If a brcond's true block is not the fallthrough, make it so by inverting the condition and swapping o...
bool matchAddEToAddO(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*ADDE x, y, 0) -> (G_*ADDO x, y) (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine addos.
void applyCombineP2IToI2P(MachineInstr &MI, Register &Reg)
Transform PtrToInt(IntToPtr(x)) to x.
bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal)
Reduce a shift by a constant to an unmerge and a shift on a half sized type.
bool matchCommuteConstantToRHS(MachineInstr &MI)
Match constant LHS ops that should be commuted.
void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
void applyFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Replace MI with a series of instructions described in MatchInfo.
bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fneg (fmul x, y))), z) -> (fneg (fma (fpext x), (fpext y),...
MachineIRBuilder & Builder
bool matchBitfieldExtractFromShrAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (and x, n), k -> ubfx x, pos, width.
bool matchCombineShuffleConcat(MachineInstr &MI, SmallVector< Register > &Ops)
bool matchReassocCommBinOp(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate commutative binary operations like G_ADD.
bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo)
Push a binary operator through a select on constants.
bool matchConstantFoldCastOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool matchOperandIsZero(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is zero.
bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo)
void applyUDivByPow2(MachineInstr &MI)
Given an G_UDIV MI expressing an unsigned divided by a pow2 constant, return expressions that impleme...
bool matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
void applyAshShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
void applySextTruncSextLoad(MachineInstr &MI)
bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
bool matchCommuteFPConstantToRHS(MachineInstr &MI)
Match constant LHS FP ops that should be commuted.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValue() const
const APFloat & getValueAPF() const
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
const APInt & getLower() const
Return the lower value for this range.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
const APInt & getUpper() const
Return the upper value for this range.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
OverflowResult signedAddMayOverflow(const ConstantRange &Other) const
Return whether signed add of the two ranges always/never overflows.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Represents overflowing add operations.
Represents an integer addition.
Represents a logical and.
CmpInst::Predicate getCond() const
Register getLHSReg() const
Register getRHSReg() const
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Register getLHSReg() const
Register getRHSReg() const
Represents a G_BUILD_VECTOR.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
virtual void erasingInstr(MachineInstr &MI)=0
An instruction is about to be erased.
void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
unsigned computeNumSignBits(Register R, const APInt &DemandedElts, unsigned Depth=0)
KnownBits getKnownBits(Register R)
APInt getKnownZeroes(Register R)
Simple wrapper observer that takes several observers, and calls each one for each event.
Represents a G_IMPLICIT_DEF.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
Represents a logical binary operation.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_MERGE_VALUES.
Register getCondReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
@ Legalized
Instruction has been legalized and the MachineFunction changed.
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
bool isLegalOrCustom(const LegalityQuery &Query) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
LLVMContext & getContext() const
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildCTTZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTTZ Op0, Src0.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert integer negation Zero = G_CONSTANT 0 Res = G_SUB Zero, Op0.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
bool isDereferenceableInvariantLoad() const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI)
Clone another MachineInstr's memory reference descriptor list and replace ours with it.
unsigned getNumOperands() const
Retuns the total number of operands.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
int findRegisterDefOperandIdx(Register Reg, const TargetRegisterInfo *TRI, bool isDead=false, bool Overlap=false) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
unsigned getAddrSpace() const
const MachinePointerInfo & getPointerInfo() const
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setMBB(MachineBasicBlock *MBB)
void setPredicate(unsigned Predicate)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
void setRegClassOrRegBank(Register Reg, const RegClassOrRegBank &RCOrRB)
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
bool constrainRegAttrs(Register Reg, Register ConstrainingReg, unsigned MinNumRegs=0)
Constrain the register class or the register bank of the virtual register Reg (and low-level type) to...
iterator_range< use_iterator > use_operands(Register Reg) const
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const
Given the generic extension instruction ExtMI, returns true if this extension is a likely candidate f...
virtual bool produceSameValue(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo *MRI=nullptr) const
Return true if two machine instructions would produce identical values.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
virtual const TargetLowering * getTargetLowering() const
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Value(Type *Ty, unsigned scid)
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
operand_type_match m_Reg()
SpecificConstantOrSplatMatch m_SpecificICstOrSplat(int64_t RequestedValue)
Matches a RequestedValue constant or a constant splat of RequestedValue.
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR, false > m_GBuildVector(const LHS &L, const RHS &R)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
UnaryOp_match< SrcTy, TargetOpcode::G_INTTOPTR > m_GIntToPtr(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CheckType m_SpecificType(LLT Ty)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FADD, true > m_GFAdd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_PTRTOINT > m_GPtrToInt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FSUB, false > m_GFSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR_TRUNC, false > m_GBuildVectorTrunc(const LHS &L, const RHS &R)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP, true > m_c_GICmp(const Pred &P, const LHS &L, const RHS &R)
G_ICMP matcher that also matches commuted compares.
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_INSERT_VECTOR_ELT > m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
And< Preds... > m_all_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
static double log2(double V)
const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
std::optional< APInt > getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI)
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
std::optional< APFloat > ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
std::optional< APInt > isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a splat vector of constant integers.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
std::function< void(MachineIRBuilder &)> BuildFnTy
std::optional< APInt > ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
std::optional< FPValueAndVReg > getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef=true)
Returns a floating point scalar constant of a build vector splat if it exists.
std::optional< APInt > ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0, const MachineRegisterInfo &MRI)
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
constexpr unsigned BitWidth
int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP)
Returns an integer representing true, as defined by the TargetBooleanContents.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
std::optional< int64_t > getIConstantSplatSExtVal(const Register Reg, const MachineRegisterInfo &MRI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
Simple struct used to hold a Register value and the instruction which defines it.
SmallVector< InstructionBuildSteps, 2 > InstrsToBuild
Describes instructions to be built during a combine.
static std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
static std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
static std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
static std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
const RegisterBank * Bank
Register LogicNonShiftReg
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
static UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
unsigned PostShift
post-shift amount